diff --git a/test/libc/release/clang.sh b/test/libc/release/clang.sh index 5cb899b4e..898bdce39 100755 --- a/test/libc/release/clang.sh +++ b/test/libc/release/clang.sh @@ -3,6 +3,7 @@ #───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘ if CLANG=$(command -v clang); then + mkdir -p o/$MODE/test/libc/release $CLANG \ -o o/$MODE/test/libc/release/smokeclang.com.dbg \ -Os \ diff --git a/test/libc/release/lld.sh b/test/libc/release/lld.sh index 28a43faa3..997b58923 100755 --- a/test/libc/release/lld.sh +++ b/test/libc/release/lld.sh @@ -5,6 +5,7 @@ # TODO(jart): implement me if CLANG=$(command -v clang); then + mkdir -p o/$MODE/test/libc/release $CLANG \ -o o/$MODE/test/libc/release/smokeclang.com.dbg \ -Os \ diff --git a/test/libc/release/metal.sh b/test/libc/release/metal.sh index 42e381364..8ffbc9385 100755 --- a/test/libc/release/metal.sh +++ b/test/libc/release/metal.sh @@ -8,6 +8,8 @@ if [ "$MODE" = opt ]; then exit fi +mkdir -p o/$MODE/test/libc/release/ + # smoke test booting on bare metal and printing data to serial uart CMD="o/$MODE/tool/build/blinkenlights.com.dbg -r o/$MODE/examples/hello.com" if OUTPUT="$($CMD)"; then diff --git a/third_party/python/.azure-pipelines/ci.yml b/third_party/python/.azure-pipelines/ci.yml deleted file mode 100644 index 8678ff765..000000000 --- a/third_party/python/.azure-pipelines/ci.yml +++ /dev/null @@ -1,137 +0,0 @@ -variables: - manylinux: false - coverage: false - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-16.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-16.04 - - steps: - - template: ./docs-steps.yml - parameters: - upload: true - - -- job: macOS_CI_Tests - displayName: macOS CI Tests - dependsOn: Prebuild - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - - variables: - testRunTitle: '$(build.sourceBranchName)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.14 - - steps: - - template: ./macos-steps.yml - - -- job: Ubuntu_CI_Tests - displayName: Ubuntu CI Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-16.04 - - variables: - testRunTitle: '$(build.sourceBranchName)-linux' - testRunPlatform: linux - openssl_version: 1.1.0g - - steps: - - template: ./posix-steps.yml - - -- job: ManyLinux1_CI_Tests - displayName: ManyLinux1 CI Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - variables: - testRunTitle: '$(build.sourceBranchName)-manylinux1' - testRunPlatform: manylinux1 - imageName: 'dockcross/manylinux-x64' - - steps: - - template: ./docker-steps.yml - - -- job: Ubuntu_Coverage_CI_Tests - displayName: Ubuntu CI Tests (coverage) - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['coverage'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-coverage' - testRunPlatform: linux-coverage - openssl_version: 1.1.0g - - steps: - - template: ./posix-steps.yml - parameters: - coverage: true - - -- job: Windows_CI_Tests - displayName: Windows CI Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: vs2017-win2016 - - strategy: - matrix: - win32: - arch: win32 - buildOpt: - testRunTitle: '$(Build.SourceBranchName)-win32' - testRunPlatform: win32 - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(Build.SourceBranchName)-win64' - testRunPlatform: win64 - maxParallel: 2 - - steps: - - template: ./windows-steps.yml diff --git a/third_party/python/.azure-pipelines/docker-steps.yml b/third_party/python/.azure-pipelines/docker-steps.yml deleted file mode 100644 index ba4dfd72d..000000000 --- a/third_party/python/.azure-pipelines/docker-steps.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- ${{ if ne(parameters.targetBranch, '') }}: - - script: | - git fetch -q origin ${{ parameters.targetbranch }} - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - -- task: docker@0 - displayName: 'Configure CPython (debug)' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: './configure --with-pydebug' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Build CPython' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make -s -j4' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Display build info' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make pythoninfo' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Tests' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/third_party/python/.azure-pipelines/docs-steps.yml b/third_party/python/.azure-pipelines/docs-steps.yml deleted file mode 100644 index 492e4e34b..000000000 --- a/third_party/python/.azure-pipelines/docs-steps.yml +++ /dev/null @@ -1,46 +0,0 @@ -parameters: - latex: false - upload: false - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - -- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme - displayName: 'Install build dependencies' - -- ${{ if ne(parameters.latex, 'true') }}: - - script: make check suspicious html PYTHON=python - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.latex, 'true') }}: - - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full - displayName: 'Install LaTeX' - - - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- ${{ if eq(parameters.upload, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish docs' - - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: docs - publishLocation: Container - - - ${{ if eq(parameters.latex, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: 'Publish dist' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/dist' - ArtifactName: docs_dist - publishLocation: Container diff --git a/third_party/python/.azure-pipelines/macos-steps.yml b/third_party/python/.azure-pipelines/macos-steps.yml deleted file mode 100644 index fa38a0df8..000000000 --- a/third_party/python/.azure-pipelines/macos-steps.yml +++ /dev/null @@ -1,27 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - continueOnError: true - timeoutInMinutes: 30 - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/third_party/python/.azure-pipelines/posix-deps.sh b/third_party/python/.azure-pipelines/posix-deps.sh deleted file mode 100755 index a57210756..000000000 --- a/third_party/python/.azure-pipelines/posix-deps.sh +++ /dev/null @@ -1,26 +0,0 @@ -sudo apt-get update - -sudo apt-get -yq install \ - build-essential \ - zlib1g-dev \ - libbz2-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libgdbm-dev \ - tk-dev \ - lzma \ - lzma-dev \ - liblzma-dev \ - libffi-dev \ - uuid-dev \ - xvfb - -if [ ! -z "$1" ] -then - echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 - echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 - python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux -fi diff --git a/third_party/python/.azure-pipelines/posix-steps.yml b/third_party/python/.azure-pipelines/posix-steps.yml deleted file mode 100644 index 9fec9be80..000000000 --- a/third_party/python/.azure-pipelines/posix-steps.yml +++ /dev/null @@ -1,63 +0,0 @@ -parameters: - coverage: false - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: ./.azure-pipelines/posix-deps.sh $(openssl_version) - displayName: 'Install dependencies' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -s -j4 - displayName: 'Build CPython' - -- ${{ if eq(parameters.coverage, 'true') }}: - - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage - displayName: 'Set up virtual environment' - - - script: ./venv/bin/python -m test.pythoninfo - displayName: 'Display build info' - - - script: | - xvfb-run ./venv/bin/python -m coverage run --pylib -m test \ - --fail-env-changed \ - -uall,-cpu \ - --junit-xml=$(build.binariesDirectory)/test-results.xml" \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures - displayName: 'Tests with coverage' - - - script: ./venv/bin/python -m coverage xml - displayName: 'Generate coverage.xml' - - - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) - displayName: 'Publish code coverage results' - - -- ${{ if ne(parameters.coverage, 'true') }}: - - script: make pythoninfo - displayName: 'Display build info' - - - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - - -- script: ./python Tools/scripts/patchcheck.py --travis true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/third_party/python/.azure-pipelines/pr.yml b/third_party/python/.azure-pipelines/pr.yml deleted file mode 100644 index 6db2a0459..000000000 --- a/third_party/python/.azure-pipelines/pr.yml +++ /dev/null @@ -1,88 +0,0 @@ -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-16.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Docs_PR - displayName: Docs PR - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) - - pool: - vmImage: ubuntu-16.04 - - steps: - - template: ./docs-steps.yml - - -- job: macOS_PR_Tests - displayName: macOS PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.14 - - steps: - - template: ./macos-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) - - -- job: Ubuntu_PR_Tests - displayName: Ubuntu PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-16.04 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-linux' - testRunPlatform: linux - openssl_version: 1.1.0g - - steps: - - template: ./posix-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) - - -- job: Windows_PR_Tests - displayName: Windows PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: vs2017-win2016 - - strategy: - matrix: - win32: - arch: win32 - buildOpt: - testRunTitle: '$(System.PullRequest.TargetBranch)-win32' - testRunPlatform: win32 - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(System.PullRequest.TargetBranch)-win64' - testRunPlatform: win64 - maxParallel: 2 - - steps: - - template: ./windows-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) diff --git a/third_party/python/.azure-pipelines/prebuild-checks.yml b/third_party/python/.azure-pipelines/prebuild-checks.yml deleted file mode 100644 index 30ff642d1..000000000 --- a/third_party/python/.azure-pipelines/prebuild-checks.yml +++ /dev/null @@ -1,36 +0,0 @@ -steps: -- checkout: self - fetchDepth: 5 - -- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1" - displayName: Set default diff target - -- script: | - git fetch -q origin $(System.PullRequest.TargetBranch) - echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)" - displayName: Fetch comparison tree - condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) - -- script: | - if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)' - then - echo "No docs were updated: docs.run=false" - echo "##vso[task.setvariable variable=run;isOutput=true]false" - else - echo "Docs were updated: docs.run=true" - echo "##vso[task.setvariable variable=run;isOutput=true]true" - fi - displayName: Detect documentation changes - name: docs - -- script: | - if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated: tests.run=false" - echo "##vso[task.setvariable variable=run;isOutput=true]false" - else - echo "Code was updated: tests.run=true" - echo "##vso[task.setvariable variable=run;isOutput=true]true" - fi - displayName: Detect source changes - name: tests diff --git a/third_party/python/.azure-pipelines/windows-steps.yml b/third_party/python/.azure-pipelines/windows-steps.yml deleted file mode 100644 index d8d5f1753..000000000 --- a/third_party/python/.azure-pipelines/windows-steps.yml +++ /dev/null @@ -1,32 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- powershell: | - # Relocate build outputs outside of source directory to make cleaning faster - Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' - # UNDONE: Do not build to a different directory because of broken tests - Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' - Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals' - displayName: Update build locations - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" - displayName: 'Tests' - env: - PREFIX: $(Py_OutDir)\$(arch) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/third_party/python/.bzrignore b/third_party/python/.bzrignore deleted file mode 100644 index 584711092..000000000 --- a/third_party/python/.bzrignore +++ /dev/null @@ -1,42 +0,0 @@ -.purify -autom4te.cache -config.log -config.cache -config.status -config.status.lineno -db_home -Makefile -buildno -python -build -Makefile.pre -platform -pybuilddir.txt -pyconfig.h -libpython*.a -libpython*.so* -python.exe -python-gdb.py -reflog.txt -tags -TAGS -.gdb_history -Doc/tools/sphinx -Doc/tools/jinja -Doc/tools/jinja2 -Doc/tools/pygments -Doc/tools/docutils -Misc/python.pc -Modules/Setup -Modules/Setup.config -Modules/Setup.local -Modules/config.c -Modules/ld_so_aix -Parser/pgen -Lib/test/data/* -Lib/lib2to3/Grammar*.pickle -Lib/lib2to3/PatternGrammar*.pickle -__pycache__ -.coverage -coverage/* -htmlcov/* diff --git a/third_party/python/.github/PULL_REQUEST_TEMPLATE.md b/third_party/python/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 4ce80d872..000000000 --- a/third_party/python/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -## CPython Mirror - -https://github.com/python/cpython is a cpython mirror repository. Pull requests -are not accepted on this repo and will be automatically closed. - -### Submit patches at https://bugs.python.org - -For additional information about contributing to CPython, see the -[developer's guide](https://docs.python.org/devguide/#contributing). diff --git a/third_party/python/.github/appveyor.yml b/third_party/python/.github/appveyor.yml deleted file mode 100644 index 3e0f588a2..000000000 --- a/third_party/python/.github/appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 3.6build{build} -clone_depth: 5 -branches: - only: - - master - - /\d\.\d/ - - buildbot-custom -cache: - - externals -> PCbuild\* -before_build: - - ps: |+ - if ($env:APPVEYOR_RE_BUILD) { - echo 'Doing full build due to re-build request.' - } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { - echo 'Not a PR, doing full build.' - } else { - git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH - $mergebase = git merge-base HEAD FETCH_HEAD - $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)' - If (!$changes) { - echo 'Only docs were updated, stopping build process.' - Exit-AppveyorBuild - } else { - echo 'Doing full build due to non-doc changes in these files:' - echo $changes - } - } - - -build_script: - - cmd: PCbuild\build.bat -e - - cmd: PCbuild\win32\python.exe -m test.pythoninfo -test_script: - - cmd: PCbuild\rt.bat -q -uall -u-cpu -u-largefile -rwW --slowest --timeout=1200 -j0 -environment: - HOST_PYTHON: C:\Python36\python.exe -image: - - Visual Studio 2015 diff --git a/third_party/python/.github/codecov.yml b/third_party/python/.github/codecov.yml deleted file mode 100644 index ea504f486..000000000 --- a/third_party/python/.github/codecov.yml +++ /dev/null @@ -1,30 +0,0 @@ -codecov: - strict_yaml_branch: master - notify: - require_ci_to_pass: true -comment: off -ignore: - - "Doc/**/*" - - "Misc/**/*" - - "Mac/**/*" - - "PC/**/*" - - "PCbuild/**/*" - - "Tools/**/*" - - "Grammar/*" -coverage: - precision: 2 - range: 70...90 - round: down - status: - changes: off - project: off - patch: off -parsers: - gcov: - branch_detection: - conditional: true - loop: true - macro: false - method: false - javascript: - enable_partials: false diff --git a/third_party/python/.hgignore b/third_party/python/.hgignore deleted file mode 100644 index 92896b7bc..000000000 --- a/third_party/python/.hgignore +++ /dev/null @@ -1,105 +0,0 @@ -.gdb_history -.purify -.svn/ -^.idea/ -^.vscode/ -.DS_Store -Makefile$ -Makefile.pre$ -TAGS$ -autom4te.cache$ -^build/ -^Doc/build/ -^Doc/venv/ -buildno$ -config.cache -config.log -config.status -config.status.lineno -db_home -platform$ -pyconfig.h$ -python$ -python.bat$ -python.exe$ -python-config$ -python-config.py$ -reflog.txt$ -tags$ -Misc/python.pc -Misc/python-config.sh$ -Modules/Setup$ -Modules/Setup.config -Modules/Setup.local -Modules/config.c -Modules/ld_so_aix$ -Parser/pgen$ -^lcov-report/ -^core -^python-gdb.py -^python.exe-gdb.py -^pybuilddir.txt - -syntax: glob -libpython*.a -libpython*.so* -libpython*.dylib -*.swp -*.o -*.pyc -*.pyo -*.pyd -*.cover -*~ -*.gc?? -*.profclang? -*.profraw -*.dyn -Include/pydtrace_probes.h -Lib/distutils/command/*.pdb -Lib/lib2to3/*.pickle -Lib/test/data/* -Misc/*.wpu -PC/python_nt*.h -PC/pythonnt_rc*.h -PC/*/*.exe -PC/*/*.exp -PC/*/*.lib -PC/*/*.bsc -PC/*/*.dll -PC/*/*.pdb -PC/*/*.user -PC/*/*.ncb -PC/*/*.suo -PC/*/Win32-temp-* -PC/*/x64-temp-* -PC/*/amd64 -PCbuild/*.user -PCbuild/*.suo -PCbuild/*.*sdf -PCbuild/*-pgi -PCbuild/*-pgo -PCbuild/.vs -PCbuild/amd64 -PCbuild/obj -PCbuild/win32 -Tools/unicode/build/ -Tools/unicode/MAPPINGS/ -BuildLog.htm -__pycache__ -Programs/_freeze_importlib -Programs/_testembed -.coverage -coverage/ -externals/ -htmlcov/ -*.gcda -*.gcno -*.gcov -ipch/ -coverage.info -Tools/msi/obj -Tools/ssl/amd64 -Tools/ssl/win32 -.vs/ -.vscode/ diff --git a/third_party/python/.travis.yml b/third_party/python/.travis.yml deleted file mode 100644 index 0c4c28e82..000000000 --- a/third_party/python/.travis.yml +++ /dev/null @@ -1,135 +0,0 @@ -language: c -dist: xenial - -# To cache doc-building dependencies and C compiler output. -cache: - - pip - - ccache - -env: - global: - # Use -O3 because we don't use debugger on Travis-CI - - CFLAGS="-O3" - -branches: - only: - - master - - /^\d\.\d$/ - - buildbot-custom - -matrix: - fast_finish: true - allow_failures: - - env: OPTIONAL=true - include: - - os: linux - language: c - compiler: clang - # gcc also works, but to keep the # of concurrent builds down, we use one C - # compiler here and the other to run the coverage build. Clang is preferred - # in this instance for its better error messages. - env: TESTING=cpython - addons: - apt: - packages: - - xvfb - - os: linux - language: python - python: 3.6 - env: TESTING=docs - before_script: - - cd Doc - # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. - # (Updating the version is fine as long as no warnings are raised by doing so.) - - python -m pip install sphinx==1.8.2 blurb - script: - - make check suspicious html SPHINXOPTS="-q -W -j4" - - os: linux - language: c - compiler: gcc - env: OPTIONAL=true - addons: - apt: - packages: - - xvfb - before_script: - - ./configure PYTHON_FOR_REGEN=python3 - - make -s -j4 - # Need a venv that can parse covered code. - - ./python -m venv venv - - ./venv/bin/python -m pip install -U coverage - - ./venv/bin/python -m test.pythoninfo - script: - # Skip tests that re-run the entire test suite. - - xvfb-run ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn - after_script: # Probably should be after_success once test suite updated to run under coverage.py. - # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - - source ./venv/bin/activate - - bash <(curl -s https://codecov.io/bash) - - -before_install: - - set -e - - | - # Check short-circuit conditions - if [ "${TESTING}" != "docs" ] - then - if [ "$TRAVIS_PULL_REQUEST" = "false" ] - then - echo "Not a PR, doing full build." - else - # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE - # may include more changes than desired if the history is convoluted. - # Instead, explicitly fetch the base branch and compare against the - # merge-base commit. - git fetch -q origin +refs/heads/$TRAVIS_BRANCH - changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) - echo "Files changed:" - echo "$changes" - if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' - then - echo "Only docs were updated, stopping build process." - exit - fi - fi - fi - -# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. -before_script: - - ./configure --with-pydebug PYTHON_FOR_REGEN=python3 - - make -j4 regen-all - - changes=`git status --porcelain` - - | - # Check for changes in regenerated files - if ! test -z "$changes" - then - echo "Generated files not up to date" - echo "$changes" - exit 1 - fi - - make -j4 - - make pythoninfo - -script: - # Using the built Python as patchcheck.py is built around the idea of using - # a checkout-build of CPython to know things like what base branch the changes - # should be compared against. - # Only run on Linux as the check only needs to be run once. - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi - # Check that all symbols exported by libpython start with "Py" or "_Py" - - make smelly - # `-r -w` implicitly provided through `make buildbottest`. - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then XVFB_RUN=xvfb-run; fi; $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu" - -notifications: - email: false - irc: - channels: - # This is set to a secure variable to prevent forks from notifying the - # IRC channel whenever they fail a build. This can be removed when travis - # implements https://github.com/travis-ci/travis-ci/issues/1094. - # The actual value here is: irc.freenode.net#python-dev - - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU=" - on_success: change - on_failure: always - skip_join: true diff --git a/third_party/python/Doc/README.rst b/third_party/python/Doc/README.rst deleted file mode 100644 index 9fc39834f..000000000 --- a/third_party/python/Doc/README.rst +++ /dev/null @@ -1,138 +0,0 @@ -Python Documentation README -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, `prebuilt versions are -available `_. - -Documentation on authoring Python documentation, including information about -both style and markup, is available in the "`Documenting Python -`_" chapter of the -developers guide. - - -Building the docs -================= - -The documentation is built with several tools which are not included in this -tree but are maintained separately and are available from -`PyPI `_. - -* `Sphinx `_ -* `blurb `_ - -The easiest way to install these tools is to create a virtual environment and -install the tools into there. - - -Using make ----------- - -To get started on UNIX, you can create a virtual environment with the command :: - - make venv - -That will install all the tools necessary to build the documentation. Assuming -the virtual environment was created in the ``venv`` directory (the default; -configurable with the VENVDIR variable), you can run the following command to -build the HTML output files:: - - make html - -By default, if the virtual environment is not created, the Makefile will -look for instances of sphinxbuild and blurb installed on your process PATH -(configurable with the SPHINXBUILD and BLURB variables). - -On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable instead. - -Available make targets are: - -* "clean", which removes all build files. - -* "venv", which creates a virtual environment with all necessary tools - installed. - -* "html", which builds standalone HTML files for offline viewing. - -* "htmlview", which re-uses the "html" builder, but then opens the main page - in your default web browser. - -* "htmlhelp", which builds HTML files and a HTML Help project file usable to - convert them into a single Compiled HTML (.chm) file -- these are popular - under Microsoft Windows, but very handy on every platform. - - To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for - you on Windows. - -* "latex", which builds LaTeX source files as input to "pdflatex" to produce - PDF documents. - -* "text", which builds a plain text file for each source file. - -* "epub", which builds an EPUB document, suitable to be viewed on e-book - readers. - -* "linkcheck", which checks all external references to see whether they are - broken, redirected or malformed, and outputs this information to stdout as - well as a plain-text (.txt) file. - -* "changes", which builds an overview over all versionadded/versionchanged/ - deprecated items in the current version. This is meant as a help for the - writer of the "What's New" document. - -* "coverage", which builds a coverage overview for standard library modules and - C API. - -* "pydoc-topics", which builds a Python module containing a dictionary with - plain text documentation for the labels defined in - `tools/pyspecific.py` -- pydoc needs these to show topic and keyword help. - -* "suspicious", which checks the parsed markup for text that looks like - malformed and thus unconverted reST. - -* "check", which checks for frequent markup errors. - -* "serve", which serves the build/html directory on port 8000. - -* "dist", (Unix only) which creates distributable archives of HTML, text, - PDF, and EPUB builds. - - -Without make ------------- - -First, install the tool dependencies from PyPI. - -Then, from the ``Doc`` directory, run :: - - sphinx-build -b . build/ - -where ```` is one of html, text, latex, or htmlhelp (for explanations -see the make targets above). - -Deprecation header -================== - -You can define the ``outdated`` variable in ``html_context`` to show a -red banner on each page redirecting to the "latest" version. - -The link points to the same page on ``/3/``, sadly for the moment the -language is lost during the process. - - -Contributing -============ - -Bugs in the content should be reported to the -`Python bug tracker `_. - -Bugs in the toolset should be reported to the tools themselves. - -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. diff --git a/third_party/python/Doc/about.rst b/third_party/python/Doc/about.rst deleted file mode 100644 index 3ea311fa6..000000000 --- a/third_party/python/Doc/about.rst +++ /dev/null @@ -1,39 +0,0 @@ -===================== -About these documents -===================== - - -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. - -.. _reStructuredText: http://docutils.sourceforge.net/rst.html -.. _Sphinx: http://sphinx-doc.org/ - -.. In the online version of these documents, you can submit comments and suggest - changes directly on the documentation pages. - -Development of the documentation and its toolchain is an entirely volunteer -effort, just like Python itself. If you want to contribute, please take a -look at the :ref:`reporting-bugs` page for information on how to do so. New -volunteers are always welcome! - -Many thanks go to: - -* Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; -* the `Docutils `_ project for creating - reStructuredText and the Docutils suite; -* Fredrik Lundh for his `Alternative Python Reference - `_ project from which Sphinx got many good - ideas. - - -Contributors to the Python Documentation ----------------------------------------- - -Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. - -It is only with the input and contributions of the Python community -that Python has such wonderful documentation -- Thank You! diff --git a/third_party/python/Doc/bugs.rst b/third_party/python/Doc/bugs.rst deleted file mode 100644 index bc1d10f37..000000000 --- a/third_party/python/Doc/bugs.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. _reporting-bugs: - -***************** -Dealing with Bugs -***************** - -Python is a mature programming language which has established a reputation for -stability. In order to maintain this reputation, the developers would like to -know of any deficiencies you find in Python. - -It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to -:ref:`contribute `. - -Documentation bugs -================== - -If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you -have a suggestion how to fix it, include that as well. - -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. - -.. seealso:: - `Documentation bugs`_ on the Python issue tracker - -.. _using-the-tracker: - -Using the Python issue tracker -============================== - -Bug reports for Python itself should be submitted via the Python Bug Tracker -(https://bugs.python.org/). The bug tracker offers a Web form which allows -pertinent information to be entered and submitted to the developers. - -The first step in filing a report is to determine whether the problem has -already been reported. The advantage in doing so, aside from saving the -developers time, is that you learn what has been done to fix it; it may be that -the problem has already been fixed for the next release, or additional -information is needed (in which case you are welcome to provide it if you can!). -To do this, search the bug database using the search box on the top of the page. - -If the problem you're reporting is not already in the bug tracker, go back to -the Python Bug Tracker and log in. If you don't already have a tracker account, -select the "Register" link or, if you use OpenID, one of the OpenID provider -logos in the sidebar. It is not possible to submit a bug report anonymously. - -Being now logged in, you can submit a bug. Select the "Create New" link in the -sidebar to open the bug reporting form. - -The submission form has a number of fields. For the "Title" field, enter a -*very* short description of the problem; less than ten words is good. In the -"Type" field, select the type of your problem; also select the "Component" and -"Versions" to which the bug relates. - -In the "Comment" field, describe the problem in detail, including what you -expected to happen and what did happen. Be sure to include whether any -extension modules were involved, and what hardware and software platform you -were using (including version information as appropriate). - -Each bug report will be assigned to a developer who will determine what needs to -be done to correct the problem. You will receive an update each time action is -taken on the bug. - - -.. seealso:: - - `How to Report Bugs Effectively `_ - Article which goes into some detail about how to create a useful bug report. - This describes what kind of information is useful and why it is useful. - - `Bug Writing Guidelines `_ - Information about writing a good bug report. Some of this is specific to the - Mozilla project, but describes general good practices. - -.. _contributing-to-python: - -Getting started contributing to Python yourself -=============================================== - -Beyond just reporting bugs that you find, you are also welcome to submit -patches to fix them. You can find more information on how to get started -patching Python in the `Python Developer's Guide`_. If you have questions, -the `core-mentorship mailing list`_ is a friendly place to get answers to -any and all questions pertaining to the process of fixing issues in Python. - -.. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity -.. _Python Developer's Guide: https://devguide.python.org/ -.. _core-mentorship mailing list: https://mail.python.org/mailman/listinfo/core-mentorship/ diff --git a/third_party/python/Doc/c-api/abstract.rst b/third_party/python/Doc/c-api/abstract.rst deleted file mode 100644 index ad5388111..000000000 --- a/third_party/python/Doc/c-api/abstract.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. highlightlang:: c - -.. _abstract: - -********************** -Abstract Objects Layer -********************** - -The functions in this chapter interact with Python objects regardless of their -type, or with wide classes of object types (e.g. all numerical types, or all -sequence types). When used on object types for which they do not apply, they -will raise a Python exception. - -It is not possible to use these functions on objects that are not properly -initialized, such as a list object that has been created by :c:func:`PyList_New`, -but whose items have not been set to some non-\ ``NULL`` value yet. - -.. toctree:: - - object.rst - number.rst - sequence.rst - mapping.rst - iter.rst - buffer.rst - objbuffer.rst diff --git a/third_party/python/Doc/c-api/allocation.rst b/third_party/python/Doc/c-api/allocation.rst deleted file mode 100644 index 25a867f13..000000000 --- a/third_party/python/Doc/c-api/allocation.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. highlightlang:: c - -.. _allocating-objects: - -Allocating Objects on the Heap -============================== - - -.. c:function:: PyObject* _PyObject_New(PyTypeObject *type) - - -.. c:function:: PyVarObject* _PyObject_NewVar(PyTypeObject *type, Py_ssize_t size) - - -.. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) - - Initialize a newly-allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. - - -.. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) - - This does everything :c:func:`PyObject_Init` does, and also initializes the - length information for a variable-size object. - - -.. c:function:: TYPE* PyObject_New(TYPE, PyTypeObject *type) - - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *type*. Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. - - -.. c:function:: TYPE* PyObject_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) - - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *type*. Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *type*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. - - -.. c:function:: void PyObject_Del(PyObject *op) - - Releases memory allocated to an object using :c:func:`PyObject_New` or - :c:func:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of - the object should not be accessed after this call as the memory is no - longer a valid Python object. - - -.. c:var:: PyObject _Py_NoneStruct - - Object which is visible in Python as ``None``. This should only be accessed - using the :c:macro:`Py_None` macro, which evaluates to a pointer to this - object. - - -.. seealso:: - - :c:func:`PyModule_Create` - To allocate and create extension modules. - diff --git a/third_party/python/Doc/c-api/apiabiversion.rst b/third_party/python/Doc/c-api/apiabiversion.rst deleted file mode 100644 index 890a03839..000000000 --- a/third_party/python/Doc/c-api/apiabiversion.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. highlightlang:: c - -.. _apiabiversion: - -*********************** -API and ABI Versioning -*********************** - -``PY_VERSION_HEX`` is the Python version number encoded in a single integer. - -For example if the ``PY_VERSION_HEX`` is set to ``0x030401a2``, the underlying -version information can be found by treating it as a 32 bit number in -the following manner: - - +-------+-------------------------+------------------------------------------------+ - | Bytes | Bits (big endian order) | Meaning | - +=======+=========================+================================================+ - | ``1`` | ``1-8`` | ``PY_MAJOR_VERSION`` (the ``3`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``2`` | ``9-16`` | ``PY_MINOR_VERSION`` (the ``4`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``3`` | ``17-24`` | ``PY_MICRO_VERSION`` (the ``1`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``4`` | ``25-28`` | ``PY_RELEASE_LEVEL`` (``0xA`` for alpha, | - | | | ``0xB`` for beta, ``0xC`` for release | - | | | candidate and ``0xF`` for final), in this | - | | | case it is alpha. | - +-------+-------------------------+------------------------------------------------+ - | | ``29-32`` | ``PY_RELEASE_SERIAL`` (the ``2`` in | - | | | ``3.4.1a2``, zero for final releases) | - +-------+-------------------------+------------------------------------------------+ - -Thus ``3.4.1a2`` is hexversion ``0x030401a2``. - -All the given macros are defined in :source:`Include/patchlevel.h`. - diff --git a/third_party/python/Doc/c-api/arg.rst b/third_party/python/Doc/c-api/arg.rst deleted file mode 100644 index dc58e152a..000000000 --- a/third_party/python/Doc/c-api/arg.rst +++ /dev/null @@ -1,676 +0,0 @@ -.. highlightlang:: c - -.. _arg-parsing: - -Parsing arguments and building values -===================================== - -These functions are useful when creating your own extensions functions and -methods. Additional information and examples are available in -:ref:`extending-index`. - -The first three of these functions described, :c:func:`PyArg_ParseTuple`, -:c:func:`PyArg_ParseTupleAndKeywords`, and :c:func:`PyArg_Parse`, all use *format -strings* which are used to tell the function about the expected arguments. The -format strings use the same syntax for each of these functions. - ------------------ -Parsing arguments ------------------ - -A format string consists of zero or more "format units." A format unit -describes one Python object; it is usually a single character or a parenthesized -sequence of format units. With a few exceptions, a format unit that is not a -parenthesized sequence normally corresponds to a single address argument to -these functions. In the following description, the quoted form is the format -unit; the entry in (round) parentheses is the Python object type that matches -the format unit; and the entry in [square] brackets is the type of the C -variable(s) whose address should be passed. - -Strings and buffers -------------------- - -These formats allow accessing an object as a contiguous chunk of memory. -You don't have to provide raw storage for the returned unicode or bytes -area. - -In general, when a format sets a pointer to a buffer, the buffer is -managed by the corresponding Python object, and the buffer shares -the lifetime of this object. You won't have to release any memory yourself. -The only exceptions are ``es``, ``es#``, ``et`` and ``et#``. - -However, when a :c:type:`Py_buffer` structure gets filled, the underlying -buffer is locked so that the caller can subsequently use the buffer even -inside a :c:type:`Py_BEGIN_ALLOW_THREADS` block without the risk of mutable data -being resized or destroyed. As a result, **you have to call** -:c:func:`PyBuffer_Release` after you have finished processing the data (or -in any early abort case). - -Unless otherwise stated, buffers are not NUL-terminated. - -Some formats require a read-only :term:`bytes-like object`, and set a -pointer instead of a buffer structure. They work by checking that -the object's :c:member:`PyBufferProcs.bf_releasebuffer` field is *NULL*, -which disallows mutable objects such as :class:`bytearray`. - -.. note:: - - For all ``#`` variants of formats (``s#``, ``y#``, etc.), the type of - the length argument (int or :c:type:`Py_ssize_t`) is controlled by - defining the macro :c:macro:`PY_SSIZE_T_CLEAN` before including - :file:`Python.h`. If the macro was defined, length is a - :c:type:`Py_ssize_t` rather than an :c:type:`int`. This behavior will change - in a future Python version to only support :c:type:`Py_ssize_t` and - drop :c:type:`int` support. It is best to always define :c:macro:`PY_SSIZE_T_CLEAN`. - - -``s`` (:class:`str`) [const char \*] - Convert a Unicode object to a C pointer to a character string. - A pointer to an existing string is stored in the character pointer - variable whose address you pass. The C string is NUL-terminated. - The Python string must not contain embedded null code points; if it does, - a :exc:`ValueError` exception is raised. Unicode objects are converted - to C strings using ``'utf-8'`` encoding. If this conversion fails, a - :exc:`UnicodeError` is raised. - - .. note:: - This format does not accept :term:`bytes-like objects - `. If you want to accept - filesystem paths and convert them to C character strings, it is - preferable to use the ``O&`` format with :c:func:`PyUnicode_FSConverter` - as *converter*. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null code points - were encountered in the Python string. - -``s*`` (:class:`str` or :term:`bytes-like object`) [Py_buffer] - This format accepts Unicode objects as well as bytes-like objects. - It fills a :c:type:`Py_buffer` structure provided by the caller. - In this case the resulting C string may contain embedded NUL bytes. - Unicode objects are converted to C strings using ``'utf-8'`` encoding. - -``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable objects. - The result is stored into two C variables, - the first one a pointer to a C string, the second one its length. - The string may contain embedded null bytes. Unicode objects are converted - to C strings using ``'utf-8'`` encoding. - -``z`` (:class:`str` or ``None``) [const char \*] - Like ``s``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. - -``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] - Like ``s*``, but the Python object may also be ``None``, in which case the - ``buf`` member of the :c:type:`Py_buffer` structure is set to *NULL*. - -``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int] - Like ``s#``, but the Python object may also be ``None``, in which case the C - pointer is set to *NULL*. - -``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a character - string; it does not accept Unicode objects. The bytes buffer must not - contain embedded null bytes; if it does, a :exc:`ValueError` - exception is raised. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null bytes were - encountered in the bytes buffer. - -``y*`` (:term:`bytes-like object`) [Py_buffer] - This variant on ``s*`` doesn't accept Unicode objects, only - bytes-like objects. **This is the recommended way to accept - binary data.** - -``y#`` (read-only :term:`bytes-like object`) [const char \*, int] - This variant on ``s#`` doesn't accept Unicode objects, only bytes-like - objects. - -``S`` (:class:`bytes`) [PyBytesObject \*] - Requires that the Python object is a :class:`bytes` object, without - attempting any conversion. Raises :exc:`TypeError` if the object is not - a bytes object. The C variable may also be declared as :c:type:`PyObject\*`. - -``Y`` (:class:`bytearray`) [PyByteArrayObject \*] - Requires that the Python object is a :class:`bytearray` object, without - attempting any conversion. Raises :exc:`TypeError` if the object is not - a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`. - -``u`` (:class:`str`) [Py_UNICODE \*] - Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of - Unicode characters. You must pass the address of a :c:type:`Py_UNICODE` - pointer variable, which will be filled with the pointer to an existing - Unicode buffer. Please note that the width of a :c:type:`Py_UNICODE` - character depends on compilation options (it is either 16 or 32 bits). - The Python string must not contain embedded null code points; if it does, - a :exc:`ValueError` exception is raised. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null code points - were encountered in the Python string. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``u#`` (:class:`str`) [Py_UNICODE \*, int] - This variant on ``u`` stores into two C variables, the first one a pointer to a - Unicode data buffer, the second one its length. This variant allows - null code points. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``Z`` (:class:`str` or ``None``) [Py_UNICODE \*] - Like ``u``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``Z#`` (:class:`str` or ``None``) [Py_UNICODE \*, int] - Like ``u#``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to *NULL*. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``U`` (:class:`str`) [PyObject \*] - Requires that the Python object is a Unicode object, without attempting - any conversion. Raises :exc:`TypeError` if the object is not a Unicode - object. The C variable may also be declared as :c:type:`PyObject\*`. - -``w*`` (read-write :term:`bytes-like object`) [Py_buffer] - This format accepts any object which implements the read-write buffer - interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call - :c:func:`PyBuffer_Release` when it is done with the buffer. - -``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] - This variant on ``s`` is used for encoding Unicode into a character buffer. - It only works for encoded data without embedded NUL bytes. - - This format requires two arguments. The first is only used as input, and - must be a :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. - An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it - references will be set to a buffer with the contents of the argument text. - The text will be encoded in the encoding specified by the first argument. - - :c:func:`PyArg_ParseTuple` will allocate a buffer of the needed size, copy the - encoded data into this buffer and adjust *\*buffer* to reference the newly - allocated storage. The caller is responsible for calling :c:func:`PyMem_Free` to - free the allocated buffer after use. - -``et`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer] - Same as ``es`` except that byte string objects are passed through without - recoding them. Instead, the implementation assumes that the byte string object uses - the encoding passed in as parameter. - -``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int \*buffer_length] - This variant on ``s#`` is used for encoding Unicode into a character buffer. - Unlike the ``es`` format, this variant allows input data which contains NUL - characters. - - It requires three arguments. The first is only used as input, and must be a - :c:type:`const char\*` which points to the name of an encoding as a - NUL-terminated string, or *NULL*, in which case ``'utf-8'`` encoding is used. - An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it - references will be set to a buffer with the contents of the argument text. - The text will be encoded in the encoding specified by the first argument. - The third argument must be a pointer to an integer; the referenced integer - will be set to the number of bytes in the output buffer. - - There are two modes of operation: - - If *\*buffer* points a *NULL* pointer, the function will allocate a buffer of - the needed size, copy the encoded data into this buffer and set *\*buffer* to - reference the newly allocated storage. The caller is responsible for calling - :c:func:`PyMem_Free` to free the allocated buffer after usage. - - If *\*buffer* points to a non-*NULL* pointer (an already allocated buffer), - :c:func:`PyArg_ParseTuple` will use this location as the buffer and interpret the - initial value of *\*buffer_length* as the buffer size. It will then copy the - encoded data into the buffer and NUL-terminate it. If the buffer is not large - enough, a :exc:`ValueError` will be set. - - In both cases, *\*buffer_length* is set to the length of the encoded data - without the trailing NUL byte. - -``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int \*buffer_length] - Same as ``es#`` except that byte string objects are passed through without recoding - them. Instead, the implementation assumes that the byte string object uses the - encoding passed in as parameter. - -Numbers -------- - -``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C - :c:type:`unsigned char`. - -``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C - :c:type:`unsigned char`. - -``h`` (:class:`int`) [short int] - Convert a Python integer to a C :c:type:`short int`. - -``H`` (:class:`int`) [unsigned short int] - Convert a Python integer to a C :c:type:`unsigned short int`, without overflow - checking. - -``i`` (:class:`int`) [int] - Convert a Python integer to a plain C :c:type:`int`. - -``I`` (:class:`int`) [unsigned int] - Convert a Python integer to a C :c:type:`unsigned int`, without overflow - checking. - -``l`` (:class:`int`) [long int] - Convert a Python integer to a C :c:type:`long int`. - -``k`` (:class:`int`) [unsigned long] - Convert a Python integer to a C :c:type:`unsigned long` without - overflow checking. - -``L`` (:class:`int`) [long long] - Convert a Python integer to a C :c:type:`long long`. - -``K`` (:class:`int`) [unsigned long long] - Convert a Python integer to a C :c:type:`unsigned long long` - without overflow checking. - -``n`` (:class:`int`) [Py_ssize_t] - Convert a Python integer to a C :c:type:`Py_ssize_t`. - -``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char] - Convert a Python byte, represented as a :class:`bytes` or - :class:`bytearray` object of length 1, to a C :c:type:`char`. - - .. versionchanged:: 3.3 - Allow :class:`bytearray` objects. - -``C`` (:class:`str` of length 1) [int] - Convert a Python character, represented as a :class:`str` object of - length 1, to a C :c:type:`int`. - -``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:type:`float`. - -``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:type:`double`. - -``D`` (:class:`complex`) [Py_complex] - Convert a Python complex number to a C :c:type:`Py_complex` structure. - -Other objects -------------- - -``O`` (object) [PyObject \*] - Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not *NULL*. - -``O!`` (object) [*typeobject*, PyObject \*] - Store a Python object in a C object pointer. This is similar to ``O``, but - takes two C arguments: the first is the address of a Python type object, the - second is the address of the C variable (of type :c:type:`PyObject\*`) into which - the object pointer is stored. If the Python object does not have the required - type, :exc:`TypeError` is raised. - -.. _o_ampersand: - -``O&`` (object) [*converter*, *anything*] - Convert a Python object to a C variable through a *converter* function. This - takes two arguments: the first is a function, the second is the address of a C - variable (of arbitrary type), converted to :c:type:`void \*`. The *converter* - function in turn is called as follows:: - - status = converter(object, address); - - where *object* is the Python object to be converted and *address* is the - :c:type:`void\*` argument that was passed to the :c:func:`PyArg_Parse\*` function. - The returned *status* should be ``1`` for a successful conversion and ``0`` if - the conversion has failed. When the conversion fails, the *converter* function - should raise an exception and leave the content of *address* unmodified. - - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a - second time if the argument parsing eventually fails, giving the converter a - chance to release any memory that it had already allocated. In this second - call, the *object* parameter will be NULL; *address* will have the same value - as in the original call. - - .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. - -``p`` (:class:`bool`) [int] - Tests the value passed in for truth (a boolean **p**\ redicate) and converts - the result to its equivalent C true/false integer value. - Sets the int to ``1`` if the expression was true and ``0`` if it was false. - This accepts any valid Python value. See :ref:`truth` for more - information about how Python tests values for truth. - - .. versionadded:: 3.3 - -``(items)`` (:class:`tuple`) [*matching-items*] - The object must be a Python sequence whose length is the number of format units - in *items*. The C arguments must correspond to the individual format units in - *items*. Format units for sequences may be nested. - -It is possible to pass "long" integers (integers whose value exceeds the -platform's :const:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). - -A few other characters have a meaning in a format string. These may not occur -inside nested parentheses. They are: - -``|`` - Indicates that the remaining arguments in the Python argument list are optional. - The C variables corresponding to optional arguments should be initialized to - their default value --- when an optional argument is not specified, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding C - variable(s). - -``$`` - :c:func:`PyArg_ParseTupleAndKeywords` only: - Indicates that the remaining arguments in the Python argument list are - keyword-only. Currently, all keyword-only arguments must also be optional - arguments, so ``|`` must always be specified before ``$`` in the format - string. - - .. versionadded:: 3.3 - -``:`` - The list of format units ends here; the string after the colon is used as the - function name in error messages (the "associated value" of the exception that - :c:func:`PyArg_ParseTuple` raises). - -``;`` - The list of format units ends here; the string after the semicolon is used as - the error message *instead* of the default error message. ``:`` and ``;`` - mutually exclude each other. - -Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! - -Additional arguments passed to these functions must be addresses of variables -whose type is determined by the format string; these are used to store values -from the input tuple. There are a few cases, as described in the list of format -units above, where these parameters are used as input values; they should match -what is specified for the corresponding format unit in that case. - -For the conversion to succeed, the *arg* object must match the format -and the format must be exhausted. On success, the -:c:func:`PyArg_Parse\*` functions return true, otherwise they return -false and raise an appropriate exception. When the -:c:func:`PyArg_Parse\*` functions fail due to conversion failure in one -of the format units, the variables at the addresses corresponding to that -and the following format units are left untouched. - -API Functions -------------- - -.. c:function:: int PyArg_ParseTuple(PyObject *args, const char *format, ...) - - Parse the parameters of a function that takes only positional parameters into - local variables. Returns true on success; on failure, it returns false and - raises the appropriate exception. - - -.. c:function:: int PyArg_VaParse(PyObject *args, const char *format, va_list vargs) - - Identical to :c:func:`PyArg_ParseTuple`, except that it accepts a va_list rather - than a variable number of arguments. - - -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) - - Parse the parameters of a function that takes both positional and keyword - parameters into local variables. The *keywords* argument is a - *NULL*-terminated array of keyword parameter names. Empty names denote - :ref:`positional-only parameters `. - Returns true on success; on failure, it returns false and raises the - appropriate exception. - - .. versionchanged:: 3.6 - Added support for :ref:`positional-only parameters - `. - - -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) - - Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a - va_list rather than a variable number of arguments. - - -.. c:function:: int PyArg_ValidateKeywordArguments(PyObject *) - - Ensure that the keys in the keywords argument dictionary are strings. This - is only needed if :c:func:`PyArg_ParseTupleAndKeywords` is not used, since the - latter already does this check. - - .. versionadded:: 3.2 - - -.. XXX deprecated, will be removed -.. c:function:: int PyArg_Parse(PyObject *args, const char *format, ...) - - Function used to deconstruct the argument lists of "old-style" functions --- - these are functions which use the :const:`METH_OLDARGS` parameter parsing - method, which has been removed in Python 3. This is not recommended for use - in parameter parsing in new code, and most code in the standard interpreter - has been modified to no longer use this for that purpose. It does remain a - convenient way to decompose other tuples, however, and may continue to be - used for that purpose. - - -.. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) - - A simpler form of parameter retrieval which does not use a format string to - specify the types of the arguments. Functions which use this method to retrieve - their parameters should be declared as :const:`METH_VARARGS` in function or - method tables. The tuple containing the actual parameters should be passed as - *args*; it must actually be a tuple. The length of the tuple must be at least - *min* and no more than *max*; *min* and *max* may be equal. Additional - arguments must be passed to the function, each of which should be a pointer to a - :c:type:`PyObject\*` variable; these will be filled in with the values from - *args*; they will contain borrowed references. The variables which correspond - to optional parameters not given by *args* will not be filled in; these should - be initialized by the caller. This function returns true on success and false if - *args* is not a tuple or contains the wrong number of elements; an exception - will be set if there was a failure. - - This is an example of the use of this function, taken from the sources for the - :mod:`_weakref` helper module for weak references:: - - static PyObject * - weakref_ref(PyObject *self, PyObject *args) - { - PyObject *object; - PyObject *callback = NULL; - PyObject *result = NULL; - - if (PyArg_UnpackTuple(args, "ref", 1, 2, &object, &callback)) { - result = PyWeakref_NewRef(object, callback); - } - return result; - } - - The call to :c:func:`PyArg_UnpackTuple` in this example is entirely equivalent to - this call to :c:func:`PyArg_ParseTuple`:: - - PyArg_ParseTuple(args, "O|O:ref", &object, &callback) - - ---------------- -Building values ---------------- - -.. c:function:: PyObject* Py_BuildValue(const char *format, ...) - - Create a new value based on a format string similar to those accepted by the - :c:func:`PyArg_Parse\*` family of functions and a sequence of values. Returns - the value or *NULL* in the case of an error; an exception will be raised if - *NULL* is returned. - - :c:func:`Py_BuildValue` does not always build a tuple. It builds a tuple only if - its format string contains two or more format units. If the format string is - empty, it returns ``None``; if it contains exactly one format unit, it returns - whatever object is described by that format unit. To force it to return a tuple - of size 0 or one, parenthesize the format string. - - When memory buffers are passed as parameters to supply data to build objects, as - for the ``s`` and ``s#`` formats, the required data is copied. Buffers provided - by the caller are never referenced by the objects created by - :c:func:`Py_BuildValue`. In other words, if your code invokes :c:func:`malloc` - and passes the allocated memory to :c:func:`Py_BuildValue`, your code is - responsible for calling :c:func:`free` for that memory once - :c:func:`Py_BuildValue` returns. - - In the following description, the quoted form is the format unit; the entry in - (round) parentheses is the Python object type that the format unit will return; - and the entry in [square] brackets is the type of the C value(s) to be passed. - - The characters space, tab, colon and comma are ignored in format strings (but - not within format units such as ``s#``). This can be used to make long format - strings a tad more readable. - - ``s`` (:class:`str` or ``None``) [char \*] - Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, ``None`` is used. - - ``s#`` (:class:`str` or ``None``) [char \*, int] - Convert a C string and its length to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is *NULL*, the length is ignored and - ``None`` is returned. - - ``y`` (:class:`bytes`) [char \*] - This converts a C string to a Python :class:`bytes` object. If the C - string pointer is *NULL*, ``None`` is returned. - - ``y#`` (:class:`bytes`) [char \*, int] - This converts a C string and its lengths to a Python object. If the C - string pointer is *NULL*, ``None`` is returned. - - ``z`` (:class:`str` or ``None``) [char \*] - Same as ``s``. - - ``z#`` (:class:`str` or ``None``) [char \*, int] - Same as ``s#``. - - ``u`` (:class:`str`) [wchar_t \*] - Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) - data to a Python Unicode object. If the Unicode buffer pointer is *NULL*, - ``None`` is returned. - - ``u#`` (:class:`str`) [wchar_t \*, int] - Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python - Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored - and ``None`` is returned. - - ``U`` (:class:`str` or ``None``) [char \*] - Same as ``s``. - - ``U#`` (:class:`str` or ``None``) [char \*, int] - Same as ``s#``. - - ``i`` (:class:`int`) [int] - Convert a plain C :c:type:`int` to a Python integer object. - - ``b`` (:class:`int`) [char] - Convert a plain C :c:type:`char` to a Python integer object. - - ``h`` (:class:`int`) [short int] - Convert a plain C :c:type:`short int` to a Python integer object. - - ``l`` (:class:`int`) [long int] - Convert a C :c:type:`long int` to a Python integer object. - - ``B`` (:class:`int`) [unsigned char] - Convert a C :c:type:`unsigned char` to a Python integer object. - - ``H`` (:class:`int`) [unsigned short int] - Convert a C :c:type:`unsigned short int` to a Python integer object. - - ``I`` (:class:`int`) [unsigned int] - Convert a C :c:type:`unsigned int` to a Python integer object. - - ``k`` (:class:`int`) [unsigned long] - Convert a C :c:type:`unsigned long` to a Python integer object. - - ``L`` (:class:`int`) [long long] - Convert a C :c:type:`long long` to a Python integer object. - - ``K`` (:class:`int`) [unsigned long long] - Convert a C :c:type:`unsigned long long` to a Python integer object. - - ``n`` (:class:`int`) [Py_ssize_t] - Convert a C :c:type:`Py_ssize_t` to a Python integer. - - ``c`` (:class:`bytes` of length 1) [char] - Convert a C :c:type:`int` representing a byte to a Python :class:`bytes` object of - length 1. - - ``C`` (:class:`str` of length 1) [int] - Convert a C :c:type:`int` representing a character to Python :class:`str` - object of length 1. - - ``d`` (:class:`float`) [double] - Convert a C :c:type:`double` to a Python floating point number. - - ``f`` (:class:`float`) [float] - Convert a C :c:type:`float` to a Python floating point number. - - ``D`` (:class:`complex`) [Py_complex \*] - Convert a C :c:type:`Py_complex` structure to a Python complex number. - - ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a *NULL* pointer, it is assumed - that this was caused because the call producing the argument found an error and - set an exception. Therefore, :c:func:`Py_BuildValue` will return *NULL* but won't - raise an exception. If no exception has been raised yet, :exc:`SystemError` is - set. - - ``S`` (object) [PyObject \*] - Same as ``O``. - - ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. - Useful when the object is created by a call to an object constructor in the - argument list. - - ``O&`` (object) [*converter*, *anything*] - Convert *anything* to a Python object through a *converter* function. The - function is called with *anything* (which should be compatible with :c:type:`void - \*`) as its argument and should return a "new" Python object, or *NULL* if an - error occurred. - - ``(items)`` (:class:`tuple`) [*matching-items*] - Convert a sequence of C values to a Python tuple with the same number of items. - - ``[items]`` (:class:`list`) [*matching-items*] - Convert a sequence of C values to a Python list with the same number of items. - - ``{items}`` (:class:`dict`) [*matching-items*] - Convert a sequence of C values to a Python dictionary. Each pair of consecutive - C values adds one item to the dictionary, serving as key and value, - respectively. - - If there is an error in the format string, the :exc:`SystemError` exception is - set and *NULL* returned. - -.. c:function:: PyObject* Py_VaBuildValue(const char *format, va_list vargs) - - Identical to :c:func:`Py_BuildValue`, except that it accepts a va_list - rather than a variable number of arguments. diff --git a/third_party/python/Doc/c-api/bool.rst b/third_party/python/Doc/c-api/bool.rst deleted file mode 100644 index a9fb342f7..000000000 --- a/third_party/python/Doc/c-api/bool.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. highlightlang:: c - -.. _boolobjects: - -Boolean Objects ---------------- - -Booleans in Python are implemented as a subclass of integers. There are only -two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal -creation and deletion functions don't apply to booleans. The following macros -are available, however. - - -.. c:function:: int PyBool_Check(PyObject *o) - - Return true if *o* is of type :c:data:`PyBool_Type`. - - -.. c:var:: PyObject* Py_False - - The Python ``False`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. - - -.. c:var:: PyObject* Py_True - - The Python ``True`` object. This object has no methods. It needs to be treated - just like any other object with respect to reference counts. - - -.. c:macro:: Py_RETURN_FALSE - - Return :const:`Py_False` from a function, properly incrementing its reference - count. - - -.. c:macro:: Py_RETURN_TRUE - - Return :const:`Py_True` from a function, properly incrementing its reference - count. - - -.. c:function:: PyObject* PyBool_FromLong(long v) - - Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the - truth value of *v*. diff --git a/third_party/python/Doc/c-api/buffer.rst b/third_party/python/Doc/c-api/buffer.rst deleted file mode 100644 index c7c1e3cc7..000000000 --- a/third_party/python/Doc/c-api/buffer.rst +++ /dev/null @@ -1,508 +0,0 @@ -.. highlightlang:: c - -.. index:: - single: buffer protocol - single: buffer interface; (see buffer protocol) - single: buffer object; (see buffer protocol) - -.. _bufferobjects: - -Buffer Protocol ---------------- - -.. sectionauthor:: Greg Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - - -Certain objects available in Python wrap access to an underlying memory -array or *buffer*. Such objects include the built-in :class:`bytes` and -:class:`bytearray`, and some extension types like :class:`array.array`. -Third-party libraries may define their own types for special purposes, such -as image processing or numeric analysis. - -While each of these types have their own semantics, they share the common -characteristic of being backed by a possibly large memory buffer. It is -then desirable, in some situations, to access that buffer directly and -without intermediate copying. - -Python provides such a facility at the C level in the form of the :ref:`buffer -protocol `. This protocol has two sides: - -.. index:: single: PyBufferProcs - -- on the producer side, a type can export a "buffer interface" which allows - objects of that type to expose information about their underlying buffer. - This interface is described in the section :ref:`buffer-structs`; - -- on the consumer side, several means are available to obtain a pointer to - the raw underlying data of an object (for example a method parameter). - -Simple objects such as :class:`bytes` and :class:`bytearray` expose their -underlying buffer in byte-oriented form. Other forms are possible; for example, -the elements exposed by an :class:`array.array` can be multi-byte values. - -An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write` -method of file objects: any object that can export a series of bytes through -the buffer interface can be written to a file. While :meth:`write` only -needs read-only access to the internal contents of the object passed to it, -other methods such as :meth:`~io.BufferedIOBase.readinto` need write access -to the contents of their argument. The buffer interface allows objects to -selectively allow or reject exporting of read-write and read-only buffers. - -There are two ways for a consumer of the buffer interface to acquire a buffer -over a target object: - -* call :c:func:`PyObject_GetBuffer` with the right parameters; - -* call :c:func:`PyArg_ParseTuple` (or one of its siblings) with one of the - ``y*``, ``w*`` or ``s*`` :ref:`format codes `. - -In both cases, :c:func:`PyBuffer_Release` must be called when the buffer -isn't needed anymore. Failure to do so could lead to various issues such as -resource leaks. - - -.. _buffer-structure: - -Buffer structure -================ - -Buffer structures (or simply "buffers") are useful as a way to expose the -binary data from another object to the Python programmer. They can also be -used as a zero-copy slicing mechanism. Using their ability to reference a -block of memory, it is possible to expose any data to the Python programmer -quite easily. The memory could be a large, constant array in a C extension, -it could be a raw block of memory for manipulation before passing to an -operating system library, or it could be used to pass around structured data -in its native, in-memory format. - -Contrary to most data types exposed by the Python interpreter, buffers -are not :c:type:`PyObject` pointers but rather simple C structures. This -allows them to be created and copied very simply. When a generic wrapper -around a buffer is needed, a :ref:`memoryview ` object -can be created. - -For short instructions how to write an exporting object, see -:ref:`Buffer Object Structures `. For obtaining -a buffer, see :c:func:`PyObject_GetBuffer`. - -.. c:type:: Py_buffer - - .. c:member:: void \*buf - - A pointer to the start of the logical structure described by the buffer - fields. This can be any location within the underlying physical memory - block of the exporter. For example, with negative :c:member:`~Py_buffer.strides` - the value may point to the end of the memory block. - - For :term:`contiguous` arrays, the value points to the beginning of - the memory block. - - .. c:member:: void \*obj - - A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to *NULL* by - :c:func:`PyBuffer_Release`. The field is the equivalent of the return - value of any standard C-API function. - - As a special case, for *temporary* buffers that are wrapped by - :c:func:`PyMemoryView_FromBuffer` or :c:func:`PyBuffer_FillInfo` - this field is *NULL*. In general, exporting objects MUST NOT - use this scheme. - - .. c:member:: Py_ssize_t len - - ``product(shape) * itemsize``. For contiguous arrays, this is the length - of the underlying memory block. For non-contiguous arrays, it is the length - that the logical structure would have if it were copied to a contiguous - representation. - - Accessing ``((char *)buf)[0] up to ((char *)buf)[len-1]`` is only valid - if the buffer has been obtained by a request that guarantees contiguity. In - most cases such a request will be :c:macro:`PyBUF_SIMPLE` or :c:macro:`PyBUF_WRITABLE`. - - .. c:member:: int readonly - - An indicator of whether the buffer is read-only. This field is controlled - by the :c:macro:`PyBUF_WRITABLE` flag. - - .. c:member:: Py_ssize_t itemsize - - Item size in bytes of a single element. Same as the value of :func:`struct.calcsize` - called on non-NULL :c:member:`~Py_buffer.format` values. - - Important exception: If a consumer requests a buffer without the - :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_buffer.format` will - be set to *NULL*, but :c:member:`~Py_buffer.itemsize` still has - the value for the original format. - - If :c:member:`~Py_buffer.shape` is present, the equality - ``product(shape) * itemsize == len`` still holds and the consumer - can use :c:member:`~Py_buffer.itemsize` to navigate the buffer. - - If :c:member:`~Py_buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE` - or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard - :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - - .. c:member:: const char \*format - - A *NUL* terminated string in :mod:`struct` module style syntax describing - the contents of a single item. If this is *NULL*, ``"B"`` (unsigned bytes) - is assumed. - - This field is controlled by the :c:macro:`PyBUF_FORMAT` flag. - - .. c:member:: int ndim - - The number of dimensions the memory represents as an n-dimensional array. - If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing - a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` - and :c:member:`~Py_buffer.suboffsets` MUST be *NULL*. - - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. - - .. c:member:: Py_ssize_t \*shape - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` - indicating the shape of the memory as an n-dimensional array. Note that - ``shape[0] * ... * shape[ndim-1] * itemsize`` MUST be equal to - :c:member:`~Py_buffer.len`. - - Shape values are restricted to ``shape[n] >= 0``. The case - ``shape[n] == 0`` requires special attention. See `complex arrays`_ - for further information. - - The shape array is read-only for the consumer. - - .. c:member:: Py_ssize_t \*strides - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` - giving the number of bytes to skip to get to a new element in each - dimension. - - Stride values can be any integer. For regular arrays, strides are - usually positive, but a consumer MUST be able to handle the case - ``strides[n] <= 0``. See `complex arrays`_ for further information. - - The strides array is read-only for the consumer. - - .. c:member:: Py_ssize_t \*suboffsets - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`. - If ``suboffsets[n] >= 0``, the values stored along the nth dimension are - pointers and the suboffset value dictates how many bytes to add to each - pointer after de-referencing. A suboffset value that is negative - indicates that no de-referencing should occur (striding in a contiguous - memory block). - - If all suboffsets are negative (i.e. no de-referencing is needed), then - this field must be NULL (the default value). - - This type of array representation is used by the Python Imaging Library - (PIL). See `complex arrays`_ for further information how to access elements - of such an array. - - The suboffsets array is read-only for the consumer. - - .. c:member:: void \*internal - - This is for use internally by the exporting object. For example, this - might be re-cast as an integer by the exporter and used to store flags - about whether or not the shape, strides, and suboffsets arrays must be - freed when the buffer is released. The consumer MUST NOT alter this - value. - -.. _buffer-request-types: - -Buffer request types -==================== - -Buffers are usually obtained by sending a buffer request to an exporting -object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical -structure of the memory can vary drastically, the consumer uses the *flags* -argument to specify the exact buffer type it can handle. - -All :c:data:`Py_buffer` fields are unambiguously defined by the request -type. - -request-independent fields -~~~~~~~~~~~~~~~~~~~~~~~~~~ -The following fields are not influenced by *flags* and must always be filled in -with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`, -:c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`. - - -readonly, format -~~~~~~~~~~~~~~~~ - - .. c:macro:: PyBUF_WRITABLE - - Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter - MUST provide a writable buffer or else report failure. Otherwise, the - exporter MAY provide either a read-only or writable buffer, but the choice - MUST be consistent for all consumers. - - .. c:macro:: PyBUF_FORMAT - - Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST - be filled in correctly. Otherwise, this field MUST be *NULL*. - - -:c:macro:`PyBUF_WRITABLE` can be \|'d to any of the flags in the next section. -Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE` -can be used as a stand-alone flag to request a simple writable buffer. - -:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`. -The latter already implies format ``B`` (unsigned bytes). - - -shape, strides, suboffsets -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The flags that control the logical structure of the memory are listed -in decreasing order of complexity. Note that each flag contains all bits -of the flags below it. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l| - -+-----------------------------+-------+---------+------------+ -| Request | shape | strides | suboffsets | -+=============================+=======+=========+============+ -| .. c:macro:: PyBUF_INDIRECT | yes | yes | if needed | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_STRIDES | yes | yes | NULL | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_SIMPLE | NULL | NULL | NULL | -+-----------------------------+-------+---------+------------+ - - -.. index:: contiguous, C-contiguous, Fortran contiguous - -contiguity requests -~~~~~~~~~~~~~~~~~~~ - -C or Fortran :term:`contiguity ` can be explicitly requested, -with and without stride information. Without stride information, the buffer -must be C-contiguous. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l| - -+-----------------------------------+-------+---------+------------+--------+ -| Request | shape | strides | suboffsets | contig | -+===================================+=======+=========+============+========+ -| .. c:macro:: PyBUF_C_CONTIGUOUS | yes | yes | NULL | C | -+-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_F_CONTIGUOUS | yes | yes | NULL | F | -+-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F | -+-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | C | -+-----------------------------------+-------+---------+------------+--------+ - - -compound requests -~~~~~~~~~~~~~~~~~ - -All possible requests are fully defined by some combination of the flags in -the previous section. For convenience, the buffer protocol provides frequently -used combinations as single flags. - -In the following table *U* stands for undefined contiguity. The consumer would -have to call :c:func:`PyBuffer_IsContiguous` to determine contiguity. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l|l|l| - -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| Request | shape | strides | suboffsets | contig | readonly | format | -+===============================+=======+=========+============+========+==========+========+ -| .. c:macro:: PyBUF_FULL | yes | yes | if needed | U | 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_FULL_RO | yes | yes | if needed | U | 1 or 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_RECORDS | yes | yes | NULL | U | 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_RECORDS_RO | yes | yes | NULL | U | 1 or 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_STRIDED | yes | yes | NULL | U | 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_STRIDED_RO | yes | yes | NULL | U | 1 or 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_CONTIG | yes | NULL | NULL | C | 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_CONTIG_RO | yes | NULL | NULL | C | 1 or 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ - - -Complex arrays -============== - -NumPy-style: shape and strides -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The logical structure of NumPy-style arrays is defined by :c:member:`~Py_buffer.itemsize`, -:c:member:`~Py_buffer.ndim`, :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides`. - -If ``ndim == 0``, the memory location pointed to by :c:member:`~Py_buffer.buf` is -interpreted as a scalar of size :c:member:`~Py_buffer.itemsize`. In that case, -both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are *NULL*. - -If :c:member:`~Py_buffer.strides` is *NULL*, the array is interpreted as -a standard n-dimensional C-array. Otherwise, the consumer must access an -n-dimensional array as follows: - - ``ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]`` - ``item = *((typeof(item) *)ptr);`` - - -As noted above, :c:member:`~Py_buffer.buf` can point to any location within -the actual memory block. An exporter can check the validity of a buffer with -this function: - -.. code-block:: python - - def verify_structure(memlen, itemsize, ndim, shape, strides, offset): - """Verify that the parameters represent a valid array within - the bounds of the allocated memory: - char *mem: start of the physical memory block - memlen: length of the physical memory block - offset: (char *)buf - mem - """ - if offset % itemsize: - return False - if offset < 0 or offset+itemsize > memlen: - return False - if any(v % itemsize for v in strides): - return False - - if ndim <= 0: - return ndim == 0 and not shape and not strides - if 0 in shape: - return True - - imin = sum(strides[j]*(shape[j]-1) for j in range(ndim) - if strides[j] <= 0) - imax = sum(strides[j]*(shape[j]-1) for j in range(ndim) - if strides[j] > 0) - - return 0 <= offset+imin and offset+imax+itemsize <= memlen - - -PIL-style: shape, strides and suboffsets -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In addition to the regular items, PIL-style arrays can contain pointers -that must be followed in order to get to the next element in a dimension. -For example, the regular three-dimensional C-array ``char v[2][2][3]`` can -also be viewed as an array of 2 pointers to 2 two-dimensional arrays: -``char (*v[2])[2][3]``. In suboffsets representation, those two pointers -can be embedded at the start of :c:member:`~Py_buffer.buf`, pointing -to two ``char x[2][3]`` arrays that can be located anywhere in memory. - - -Here is a function that returns a pointer to the element in an N-D array -pointed to by an N-dimensional index when there are both non-NULL strides -and suboffsets:: - - void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, - Py_ssize_t *suboffsets, Py_ssize_t *indices) { - char *pointer = (char*)buf; - int i; - for (i = 0; i < ndim; i++) { - pointer += strides[i] * indices[i]; - if (suboffsets[i] >=0 ) { - pointer = *((char**)pointer) + suboffsets[i]; - } - } - return (void*)pointer; - } - - -Buffer-related functions -======================== - -.. c:function:: int PyObject_CheckBuffer(PyObject *obj) - - Return ``1`` if *obj* supports the buffer interface otherwise ``0``. When ``1`` is - returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will - succeed. This function always succeeds. - - -.. c:function:: int PyObject_GetBuffer(PyObject *exporter, Py_buffer *view, int flags) - - Send a request to *exporter* to fill in *view* as specified by *flags*. - If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set :c:member:`view->obj` to *NULL* and - return ``-1``. - - On success, fill in *view*, set :c:member:`view->obj` to a new reference - to *exporter* and return 0. In the case of chained buffer providers - that redirect requests to a single object, :c:member:`view->obj` MAY - refer to this object instead of *exporter* (See :ref:`Buffer Object Structures `). - - Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls - to :c:func:`PyBuffer_Release`, similar to :c:func:`malloc` and :c:func:`free`. - Thus, after the consumer is done with the buffer, :c:func:`PyBuffer_Release` - must be called exactly once. - - -.. c:function:: void PyBuffer_Release(Py_buffer *view) - - Release the buffer *view* and decrement the reference count for - :c:member:`view->obj`. This function MUST be called when the buffer - is no longer being used, otherwise reference leaks may occur. - - It is an error to call this function on a buffer that was not obtained via - :c:func:`PyObject_GetBuffer`. - - -.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *) - - Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`. - This function is not yet implemented. - - -.. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char order) - - Return ``1`` if the memory defined by the *view* is C-style (*order* is - ``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one - (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. - - -.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) - - Copy *len* bytes from *src* to its contiguous representation in *buf*. - *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). - ``0`` is returned on success, ``-1`` on error. - - This function fails if *len* != *src->len*. - - -.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) - - Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if - *order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the - given shape with the given number of bytes per element. - - -.. c:function:: int PyBuffer_FillInfo(Py_buffer *view, PyObject *exporter, void *buf, Py_ssize_t len, int readonly, int flags) - - Handle buffer requests for an exporter that wants to expose *buf* of size *len* - with writability set according to *readonly*. *buf* is interpreted as a sequence - of unsigned bytes. - - The *flags* argument indicates the request type. This function always fills in - *view* as specified by flags, unless *buf* has been designated as read-only - and :c:macro:`PyBUF_WRITABLE` is set in *flags*. - - On success, set :c:member:`view->obj` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set - :c:member:`view->obj` to *NULL* and return ``-1``; - - If this function is used as part of a :ref:`getbufferproc `, - *exporter* MUST be set to the exporting object and *flags* must be passed - unmodified. Otherwise, *exporter* MUST be NULL. diff --git a/third_party/python/Doc/c-api/bytearray.rst b/third_party/python/Doc/c-api/bytearray.rst deleted file mode 100644 index 41b6e3c71..000000000 --- a/third_party/python/Doc/c-api/bytearray.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. highlightlang:: c - -.. _bytearrayobjects: - -Byte Array Objects ------------------- - -.. index:: object: bytearray - - -.. c:type:: PyByteArrayObject - - This subtype of :c:type:`PyObject` represents a Python bytearray object. - - -.. c:var:: PyTypeObject PyByteArray_Type - - This instance of :c:type:`PyTypeObject` represents the Python bytearray type; - it is the same object as :class:`bytearray` in the Python layer. - - -Type check macros -^^^^^^^^^^^^^^^^^ - -.. c:function:: int PyByteArray_Check(PyObject *o) - - Return true if the object *o* is a bytearray object or an instance of a - subtype of the bytearray type. - - -.. c:function:: int PyByteArray_CheckExact(PyObject *o) - - Return true if the object *o* is a bytearray object, but not an instance of a - subtype of the bytearray type. - - -Direct API functions -^^^^^^^^^^^^^^^^^^^^ - -.. c:function:: PyObject* PyByteArray_FromObject(PyObject *o) - - Return a new bytearray object from any object, *o*, that implements the - :ref:`buffer protocol `. - - .. XXX expand about the buffer protocol, at least somewhere - - -.. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) - - Create a new bytearray object from *string* and its length, *len*. On - failure, *NULL* is returned. - - -.. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) - - Concat bytearrays *a* and *b* and return a new bytearray with the result. - - -.. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) - - Return the size of *bytearray* after checking for a *NULL* pointer. - - -.. c:function:: char* PyByteArray_AsString(PyObject *bytearray) - - Return the contents of *bytearray* as a char array after checking for a - *NULL* pointer. The returned array always has an extra - null byte appended. - - -.. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) - - Resize the internal buffer of *bytearray* to *len*. - -Macros -^^^^^^ - -These macros trade safety for speed and they don't check pointers. - -.. c:function:: char* PyByteArray_AS_STRING(PyObject *bytearray) - - Macro version of :c:func:`PyByteArray_AsString`. - - -.. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) - - Macro version of :c:func:`PyByteArray_Size`. diff --git a/third_party/python/Doc/c-api/bytes.rst b/third_party/python/Doc/c-api/bytes.rst deleted file mode 100644 index ee42f854e..000000000 --- a/third_party/python/Doc/c-api/bytes.rst +++ /dev/null @@ -1,202 +0,0 @@ -.. highlightlang:: c - -.. _bytesobjects: - -Bytes Objects -------------- - -These functions raise :exc:`TypeError` when expecting a bytes parameter and are -called with a non-bytes parameter. - -.. index:: object: bytes - - -.. c:type:: PyBytesObject - - This subtype of :c:type:`PyObject` represents a Python bytes object. - - -.. c:var:: PyTypeObject PyBytes_Type - - This instance of :c:type:`PyTypeObject` represents the Python bytes type; it - is the same object as :class:`bytes` in the Python layer. - - -.. c:function:: int PyBytes_Check(PyObject *o) - - Return true if the object *o* is a bytes object or an instance of a subtype - of the bytes type. - - -.. c:function:: int PyBytes_CheckExact(PyObject *o) - - Return true if the object *o* is a bytes object, but not an instance of a - subtype of the bytes type. - - -.. c:function:: PyObject* PyBytes_FromString(const char *v) - - Return a new bytes object with a copy of the string *v* as value on success, - and *NULL* on failure. The parameter *v* must not be *NULL*; it will not be - checked. - - -.. c:function:: PyObject* PyBytes_FromStringAndSize(const char *v, Py_ssize_t len) - - Return a new bytes object with a copy of the string *v* as value and length - *len* on success, and *NULL* on failure. If *v* is *NULL*, the contents of - the bytes object are uninitialized. - - -.. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) - - Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python bytes object and return - a bytes object with the values formatted into it. The variable arguments - must be C types and must correspond exactly to the format characters in the - *format* string. The following format characters are allowed: - - .. % XXX: This should be exactly the same as the table in PyErr_Format. - .. % One should just refer to the other. - .. % XXX: The descriptions for %zd and %zu are wrong, but the truth is complicated - .. % because not all compilers support the %z width modifier -- we fake it - .. % when necessary via interpolating PY_FORMAT_SIZE_T. - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------+--------------------------------+ - | Format Characters | Type | Comment | - +===================+===============+================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------+--------------------------------+ - | :attr:`%c` | int | A single byte, | - | | | represented as a C int. | - +-------------------+---------------+--------------------------------+ - | :attr:`%d` | int | Exactly equivalent to | - | | | ``printf("%d")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%u` | unsigned int | Exactly equivalent to | - | | | ``printf("%u")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%ld` | long | Exactly equivalent to | - | | | ``printf("%ld")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%lu` | unsigned long | Exactly equivalent to | - | | | ``printf("%lu")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%zd` | Py_ssize_t | Exactly equivalent to | - | | | ``printf("%zd")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%zu` | size_t | Exactly equivalent to | - | | | ``printf("%zu")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%i` | int | Exactly equivalent to | - | | | ``printf("%i")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%x` | int | Exactly equivalent to | - | | | ``printf("%x")``. | - +-------------------+---------------+--------------------------------+ - | :attr:`%s` | char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------+--------------------------------+ - | :attr:`%p` | void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------+--------------------------------+ - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result object, and any extra arguments discarded. - - -.. c:function:: PyObject* PyBytes_FromFormatV(const char *format, va_list vargs) - - Identical to :c:func:`PyBytes_FromFormat` except that it takes exactly two - arguments. - - -.. c:function:: PyObject* PyBytes_FromObject(PyObject *o) - - Return the bytes representation of object *o* that implements the buffer - protocol. - - -.. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) - - Return the length of the bytes in bytes object *o*. - - -.. c:function:: Py_ssize_t PyBytes_GET_SIZE(PyObject *o) - - Macro form of :c:func:`PyBytes_Size` but without error checking. - - -.. c:function:: char* PyBytes_AsString(PyObject *o) - - Return a pointer to the contents of *o*. The pointer - refers to the internal buffer of *o*, which consists of ``len(o) + 1`` - bytes. The last byte in the buffer is always null, regardless of - whether there are any other null bytes. The data must not be - modified in any way, unless the object was just created using - ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns *NULL* - and raises :exc:`TypeError`. - - -.. c:function:: char* PyBytes_AS_STRING(PyObject *string) - - Macro form of :c:func:`PyBytes_AsString` but without error checking. - - -.. c:function:: int PyBytes_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) - - Return the null-terminated contents of the object *obj* - through the output variables *buffer* and *length*. - - If *length* is *NULL*, the bytes object - may not contain embedded null bytes; - if it does, the function returns ``-1`` and a :exc:`ValueError` is raised. - - The buffer refers to an internal buffer of *obj*, which includes an - additional null byte at the end (not counted in *length*). The data - must not be modified in any way, unless the object was just created using - ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *obj* is not a bytes object at all, :c:func:`PyBytes_AsStringAndSize` - returns ``-1`` and raises :exc:`TypeError`. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null bytes were - encountered in the bytes object. - - -.. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) - - Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be - created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to *NULL*; the appropriate exception will be set. - - -.. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) - - Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. - - -.. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an - existing bytes object as an lvalue (it may be written into), and the new size - desired. On success, *\*bytes* holds the resized bytes object and ``0`` is - returned; the address in *\*bytes* may differ from its input value. If the - reallocation fails, the original bytes object at *\*bytes* is deallocated, - *\*bytes* is set to *NULL*, :exc:`MemoryError` is set, and ``-1`` is - returned. diff --git a/third_party/python/Doc/c-api/capsule.rst b/third_party/python/Doc/c-api/capsule.rst deleted file mode 100644 index 8eb6695e2..000000000 --- a/third_party/python/Doc/c-api/capsule.rst +++ /dev/null @@ -1,157 +0,0 @@ -.. highlightlang:: c - -.. _capsules: - -Capsules --------- - -.. index:: object: Capsule - -Refer to :ref:`using-capsules` for more information on using these objects. - -.. versionadded:: 3.1 - - -.. c:type:: PyCapsule - - This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:type:`void\*` - pointer) through Python code to other C code. It is often used to make a C - function pointer defined in one module available to other modules, so the - regular import mechanism can be used to access C APIs defined in dynamically - loaded modules. - - -.. c:type:: PyCapsule_Destructor - - The type of a destructor callback for a capsule. Defined as:: - - typedef void (*PyCapsule_Destructor)(PyObject *); - - See :c:func:`PyCapsule_New` for the semantics of PyCapsule_Destructor - callbacks. - - -.. c:function:: int PyCapsule_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyCapsule`. - - -.. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) - - Create a :c:type:`PyCapsule` encapsulating the *pointer*. The *pointer* - argument may not be *NULL*. - - On failure, set an exception and return *NULL*. - - The *name* string may either be *NULL* or a pointer to a valid C string. If - non-*NULL*, this string must outlive the capsule. (Though it is permitted to - free it inside the *destructor*.) - - If the *destructor* argument is not *NULL*, it will be called with the - capsule as its argument when it is destroyed. - - If this capsule will be stored as an attribute of a module, the *name* should - be specified as ``modulename.attributename``. This will enable other modules - to import the capsule using :c:func:`PyCapsule_Import`. - - -.. c:function:: void* PyCapsule_GetPointer(PyObject *capsule, const char *name) - - Retrieve the *pointer* stored in the capsule. On failure, set an exception - and return *NULL*. - - The *name* parameter must compare exactly to the name stored in the capsule. - If the name stored in the capsule is *NULL*, the *name* passed in must also - be *NULL*. Python uses the C function :c:func:`strcmp` to compare capsule - names. - - -.. c:function:: PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *capsule) - - Return the current destructor stored in the capsule. On failure, set an - exception and return *NULL*. - - It is legal for a capsule to have a *NULL* destructor. This makes a *NULL* - return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyCapsule_GetContext(PyObject *capsule) - - Return the current context stored in the capsule. On failure, set an - exception and return *NULL*. - - It is legal for a capsule to have a *NULL* context. This makes a *NULL* - return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: const char* PyCapsule_GetName(PyObject *capsule) - - Return the current name stored in the capsule. On failure, set an exception - and return *NULL*. - - It is legal for a capsule to have a *NULL* name. This makes a *NULL* return - code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyCapsule_Import(const char *name, int no_block) - - Import a pointer to a C object from a capsule attribute in a module. The - *name* parameter should specify the full name to the attribute, as in - ``module.attribute``. The *name* stored in the capsule must match this - string exactly. If *no_block* is true, import the module without blocking - (using :c:func:`PyImport_ImportModuleNoBlock`). If *no_block* is false, - import the module conventionally (using :c:func:`PyImport_ImportModule`). - - Return the capsule's internal *pointer* on success. On failure, set an - exception and return *NULL*. - - -.. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) - - Determines whether or not *capsule* is a valid capsule. A valid capsule is - non-*NULL*, passes :c:func:`PyCapsule_CheckExact`, has a non-*NULL* pointer - stored in it, and its internal name matches the *name* parameter. (See - :c:func:`PyCapsule_GetPointer` for information on how capsule names are - compared.) - - In other words, if :c:func:`PyCapsule_IsValid` returns a true value, calls to - any of the accessors (any function starting with :c:func:`PyCapsule_Get`) are - guaranteed to succeed. - - Return a nonzero value if the object is valid and matches the name passed in. - Return ``0`` otherwise. This function will not fail. - - -.. c:function:: int PyCapsule_SetContext(PyObject *capsule, void *context) - - Set the context pointer inside *capsule* to *context*. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetDestructor(PyObject *capsule, PyCapsule_Destructor destructor) - - Set the destructor inside *capsule* to *destructor*. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name) - - Set the name inside *capsule* to *name*. If non-*NULL*, the name must - outlive the capsule. If the previous *name* stored in the capsule was not - *NULL*, no attempt is made to free it. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer) - - Set the void pointer inside *capsule* to *pointer*. The pointer may not be - *NULL*. - - Return ``0`` on success. Return nonzero and set an exception on failure. diff --git a/third_party/python/Doc/c-api/cell.rst b/third_party/python/Doc/c-api/cell.rst deleted file mode 100644 index 427259cc2..000000000 --- a/third_party/python/Doc/c-api/cell.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. highlightlang:: c - -.. _cell-objects: - -Cell Objects ------------- - -"Cell" objects are used to implement variables referenced by multiple scopes. -For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to -the cells from outer scopes which also use that variable. When the value is -accessed, the value contained in the cell is used instead of the cell object -itself. This de-referencing of the cell object requires support from the -generated byte-code; these are not automatically de-referenced when accessed. -Cell objects are not likely to be useful elsewhere. - - -.. c:type:: PyCellObject - - The C structure used for cell objects. - - -.. c:var:: PyTypeObject PyCell_Type - - The type object corresponding to cell objects. - - -.. c:function:: int PyCell_Check(ob) - - Return true if *ob* is a cell object; *ob* must not be *NULL*. - - -.. c:function:: PyObject* PyCell_New(PyObject *ob) - - Create and return a new cell object containing the value *ob*. The parameter may - be *NULL*. - - -.. c:function:: PyObject* PyCell_Get(PyObject *cell) - - Return the contents of the cell *cell*. - - -.. c:function:: PyObject* PyCell_GET(PyObject *cell) - - Return the contents of the cell *cell*, but without checking that *cell* is - non-*NULL* and a cell object. - - -.. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) - - Set the contents of the cell object *cell* to *value*. This releases the - reference to any current content of the cell. *value* may be *NULL*. *cell* - must be non-*NULL*; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. - - -.. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) - - Sets the value of the cell object *cell* to *value*. No reference counts are - adjusted, and no checks are made for safety; *cell* must be non-*NULL* and must - be a cell object. diff --git a/third_party/python/Doc/c-api/code.rst b/third_party/python/Doc/c-api/code.rst deleted file mode 100644 index 10d89f297..000000000 --- a/third_party/python/Doc/c-api/code.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. highlightlang:: c - -.. _codeobjects: - -.. index:: object; code, code object - -Code Objects ------------- - -.. sectionauthor:: Jeffrey Yasskin - -Code objects are a low-level detail of the CPython implementation. -Each one represents a chunk of executable code that hasn't yet been -bound into a function. - -.. c:type:: PyCodeObject - - The C structure of the objects used to describe code objects. The - fields of this type are subject to change at any time. - - -.. c:var:: PyTypeObject PyCode_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. - - -.. c:function:: int PyCode_Check(PyObject *co) - - Return true if *co* is a :class:`code` object. - -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) - - Return the number of free variables in *co*. - -.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) - - Return a new code object. If you need a dummy code object to - create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling - :c:func:`PyCode_New` directly can bind you to a precise Python - version since the definition of the bytecode changes often. - - -.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) - - Return a new empty code object with the specified filename, - function name, and first line number. It is illegal to - :func:`exec` or :func:`eval` the resulting code object. diff --git a/third_party/python/Doc/c-api/codec.rst b/third_party/python/Doc/c-api/codec.rst deleted file mode 100644 index c55f19970..000000000 --- a/third_party/python/Doc/c-api/codec.rst +++ /dev/null @@ -1,123 +0,0 @@ -.. _codec-registry: - -Codec registry and support functions -==================================== - -.. c:function:: int PyCodec_Register(PyObject *search_function) - - Register a new codec search function. - - As side effect, this tries to load the :mod:`encodings` package, if not yet - done, to make sure that it is always first in the list of search functions. - -.. c:function:: int PyCodec_KnownEncoding(const char *encoding) - - Return ``1`` or ``0`` depending on whether there is a registered codec for - the given *encoding*. This function always succeeds. - -.. c:function:: PyObject* PyCodec_Encode(PyObject *object, const char *encoding, const char *errors) - - Generic codec based encoding API. - - *object* is passed through the encoder function found for the given - *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. - -.. c:function:: PyObject* PyCodec_Decode(PyObject *object, const char *encoding, const char *errors) - - Generic codec based decoding API. - - *object* is passed through the decoder function found for the given - *encoding* using the error handling method defined by *errors*. *errors* may - be *NULL* to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. - - -Codec lookup API ----------------- - -In the following functions, the *encoding* string is looked up converted to all -lower-case characters, which makes encodings looked up through this mechanism -effectively case-insensitive. If no codec is found, a :exc:`KeyError` is set -and *NULL* returned. - -.. c:function:: PyObject* PyCodec_Encoder(const char *encoding) - - Get an encoder function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_Decoder(const char *encoding) - - Get a decoder function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) - - Get an :class:`~codecs.IncrementalEncoder` object for the given *encoding*. - -.. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) - - Get an :class:`~codecs.IncrementalDecoder` object for the given *encoding*. - -.. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) - - Get a :class:`~codecs.StreamReader` factory function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) - - Get a :class:`~codecs.StreamWriter` factory function for the given *encoding*. - - -Registry API for Unicode encoding error handlers ------------------------------------------------- - -.. c:function:: int PyCodec_RegisterError(const char *name, PyObject *error) - - Register the error handling callback function *error* under the given *name*. - This callback function will be called by a codec when it encounters - unencodable characters/undecodable bytes and *name* is specified as the error - parameter in the call to the encode/decode function. - - The callback gets a single argument, an instance of - :exc:`UnicodeEncodeError`, :exc:`UnicodeDecodeError` or - :exc:`UnicodeTranslateError` that holds information about the problematic - sequence of characters or bytes and their offset in the original string (see - :ref:`unicodeexceptions` for functions to extract this information). The - callback must either raise the given exception, or return a two-item tuple - containing the replacement for the problematic sequence, and an integer - giving the offset in the original string at which encoding/decoding should be - resumed. - - Return ``0`` on success, ``-1`` on error. - -.. c:function:: PyObject* PyCodec_LookupError(const char *name) - - Lookup the error handling callback function registered under *name*. As a - special case *NULL* can be passed, in which case the error handling callback - for "strict" will be returned. - -.. c:function:: PyObject* PyCodec_StrictErrors(PyObject *exc) - - Raise *exc* as an exception. - -.. c:function:: PyObject* PyCodec_IgnoreErrors(PyObject *exc) - - Ignore the unicode error, skipping the faulty input. - -.. c:function:: PyObject* PyCodec_ReplaceErrors(PyObject *exc) - - Replace the unicode encode error with ``?`` or ``U+FFFD``. - -.. c:function:: PyObject* PyCodec_XMLCharRefReplaceErrors(PyObject *exc) - - Replace the unicode encode error with XML character references. - -.. c:function:: PyObject* PyCodec_BackslashReplaceErrors(PyObject *exc) - - Replace the unicode encode error with backslash escapes (``\x``, ``\u`` and - ``\U``). - -.. c:function:: PyObject* PyCodec_NameReplaceErrors(PyObject *exc) - - Replace the unicode encode error with ``\N{...}`` escapes. - - .. versionadded:: 3.5 diff --git a/third_party/python/Doc/c-api/complex.rst b/third_party/python/Doc/c-api/complex.rst deleted file mode 100644 index fc63b57a8..000000000 --- a/third_party/python/Doc/c-api/complex.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. highlightlang:: c - -.. _complexobjects: - -Complex Number Objects ----------------------- - -.. index:: object: complex number - -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex complex) - - Return the negation of the complex number *complex*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:data:`EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:data:`EDOM`. - - -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - -.. c:type:: PyComplexObject - - This subtype of :c:type:`PyObject` represents a Python complex number object. - - -.. c:var:: PyTypeObject PyComplex_Type - - This instance of :c:type:`PyTypeObject` represents the Python complex number - type. It is the same object as :class:`complex` in the Python layer. - - -.. c:function:: int PyComplex_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyComplexObject` or a subtype of - :c:type:`PyComplexObject`. - - -.. c:function:: int PyComplex_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyComplexObject`, but not a subtype of - :c:type:`PyComplexObject`. - - -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - - -.. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) - - Return a new :c:type:`PyComplexObject` object from *real* and *imag*. - - -.. c:function:: double PyComplex_RealAsDouble(PyObject *op) - - Return the real part of *op* as a C :c:type:`double`. - - -.. c:function:: double PyComplex_ImagAsDouble(PyObject *op) - - Return the imaginary part of *op* as a C :c:type:`double`. - - -.. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) - - Return the :c:type:`Py_complex` value of the complex number *op*. - - If *op* is not a Python complex number object but has a :meth:`__complex__` - method, this method will first be called to convert *op* to a Python complex - number object. Upon failure, this method returns ``-1.0`` as a real value. diff --git a/third_party/python/Doc/c-api/concrete.rst b/third_party/python/Doc/c-api/concrete.rst deleted file mode 100644 index 47dab8189..000000000 --- a/third_party/python/Doc/c-api/concrete.rst +++ /dev/null @@ -1,117 +0,0 @@ -.. highlightlang:: c - - -.. _concrete: - -********************** -Concrete Objects Layer -********************** - -The functions in this chapter are specific to certain Python object types. -Passing them an object of the wrong type is not a good idea; if you receive an -object from a Python program and you are not sure that it has the right type, -you must perform a type check first; for example, to check that an object is a -dictionary, use :c:func:`PyDict_Check`. The chapter is structured like the -"family tree" of Python object types. - -.. warning:: - - While the functions described in this chapter carefully check the type of the - objects which are passed in, many of them do not check for *NULL* being passed - instead of a valid object. Allowing *NULL* to be passed in can cause memory - access violations and immediate termination of the interpreter. - - -.. _fundamental: - -Fundamental Objects -=================== - -This section describes Python type objects and the singleton object ``None``. - -.. toctree:: - - type.rst - none.rst - - -.. _numericobjects: - -Numeric Objects -=============== - -.. index:: object: numeric - -.. toctree:: - - long.rst - bool.rst - float.rst - complex.rst - - -.. _sequenceobjects: - -Sequence Objects -================ - -.. index:: object: sequence - -Generic operations on sequence objects were discussed in the previous chapter; -this section deals with the specific kinds of sequence objects that are -intrinsic to the Python language. - -.. XXX sort out unicode, str, bytes and bytearray - -.. toctree:: - - bytes.rst - bytearray.rst - unicode.rst - tuple.rst - list.rst - - -.. _mapobjects: - -Container Objects -================= - -.. index:: object: mapping - -.. toctree:: - - dict.rst - set.rst - - -.. _otherobjects: - -Function Objects -================ - -.. toctree:: - - function.rst - method.rst - cell.rst - code.rst - - -Other Objects -============= - -.. toctree:: - - file.rst - module.rst - iterator.rst - descriptor.rst - slice.rst - memoryview.rst - weakref.rst - capsule.rst - gen.rst - coro.rst - datetime.rst - diff --git a/third_party/python/Doc/c-api/conversion.rst b/third_party/python/Doc/c-api/conversion.rst deleted file mode 100644 index 9566d9d79..000000000 --- a/third_party/python/Doc/c-api/conversion.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. highlightlang:: c - -.. _string-conversion: - -String conversion and formatting -================================ - -Functions for number conversion and formatted string output. - - -.. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) - - Output not more than *size* bytes to *str* according to the format string - *format* and the extra arguments. See the Unix man page :manpage:`snprintf(2)`. - - -.. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) - - Output not more than *size* bytes to *str* according to the format string - *format* and the variable argument list *va*. Unix man page - :manpage:`vsnprintf(2)`. - -:c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library -functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to -guarantee consistent behavior in corner cases, which the Standard C functions do -not. - -The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They -never write more than *size* bytes (including the trailing ``'\0'``) into str. -Both functions require that ``str != NULL``, ``size > 0`` and ``format != -NULL``. - -If the platform doesn't have :c:func:`vsnprintf` and the buffer size needed to -avoid truncation exceeds *size* by more than 512 bytes, Python aborts with a -*Py_FatalError*. - -The return value (*rv*) for these functions should be interpreted as follows: - -* When ``0 <= rv < size``, the output conversion was successful and *rv* - characters were written to *str* (excluding the trailing ``'\0'`` byte at - *str*[*rv*]). - -* When ``rv >= size``, the output conversion was truncated and a buffer with - ``rv + 1`` bytes would have been needed to succeed. *str*[*size*-1] is ``'\0'`` - in this case. - -* When ``rv < 0``, "something bad happened." *str*[*size*-1] is ``'\0'`` in - this case too, but the rest of *str* is undefined. The exact cause of the error - depends on the underlying platform. - -The following functions provide locale-independent string to number conversions. - - -.. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) - - Convert a string ``s`` to a :c:type:`double`, raising a Python - exception on failure. The set of accepted strings corresponds to - the set of strings accepted by Python's :func:`float` constructor, - except that ``s`` must not have leading or trailing whitespace. - The conversion is independent of the current locale. - - If ``endptr`` is ``NULL``, convert the whole string. Raise - ValueError and return ``-1.0`` if the string is not a valid - representation of a floating-point number. - - If endptr is not ``NULL``, convert as much of the string as - possible and set ``*endptr`` to point to the first unconverted - character. If no initial segment of the string is the valid - representation of a floating-point number, set ``*endptr`` to point - to the beginning of the string, raise ValueError, and return - ``-1.0``. - - If ``s`` represents a value that is too large to store in a float - (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with - an appropriate sign) and don't set any exception. Otherwise, - ``overflow_exception`` must point to a Python exception object; - raise that exception and return ``-1.0``. In both cases, set - ``*endptr`` to point to the first character after the converted value. - - If any other error occurs during the conversion (for example an - out-of-memory error), set the appropriate Python exception and - return ``-1.0``. - - .. versionadded:: 3.1 - - -.. c:function:: char* PyOS_double_to_string(double val, char format_code, int precision, int flags, int *ptype) - - Convert a :c:type:`double` *val* to a string using supplied - *format_code*, *precision*, and *flags*. - - *format_code* must be one of ``'e'``, ``'E'``, ``'f'``, ``'F'``, - ``'g'``, ``'G'`` or ``'r'``. For ``'r'``, the supplied *precision* - must be 0 and is ignored. The ``'r'`` format code specifies the - standard :func:`repr` format. - - *flags* can be zero or more of the values *Py_DTSF_SIGN*, - *Py_DTSF_ADD_DOT_0*, or *Py_DTSF_ALT*, or-ed together: - - * *Py_DTSF_SIGN* means to always precede the returned string with a sign - character, even if *val* is non-negative. - - * *Py_DTSF_ADD_DOT_0* means to ensure that the returned string will not look - like an integer. - - * *Py_DTSF_ALT* means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. - - If *ptype* is non-NULL, then the value it points to will be set to one of - *Py_DTST_FINITE*, *Py_DTST_INFINITE*, or *Py_DTST_NAN*, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to *buffer* with the converted string or - *NULL* if the conversion failed. The caller is responsible for freeing the - returned string by calling :c:func:`PyMem_Free`. - - .. versionadded:: 3.1 - - -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) - - Case insensitive comparison of strings. The function works almost - identically to :c:func:`strcmp` except that it ignores the case. - - -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) - - Case insensitive comparison of strings. The function works almost - identically to :c:func:`strncmp` except that it ignores the case. diff --git a/third_party/python/Doc/c-api/coro.rst b/third_party/python/Doc/c-api/coro.rst deleted file mode 100644 index 2fe50b5d8..000000000 --- a/third_party/python/Doc/c-api/coro.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. highlightlang:: c - -.. _coro-objects: - -Coroutine Objects ------------------ - -.. versionadded:: 3.5 - -Coroutine objects are what functions declared with an ``async`` keyword -return. - - -.. c:type:: PyCoroObject - - The C structure used for coroutine objects. - - -.. c:var:: PyTypeObject PyCoro_Type - - The type object corresponding to coroutine objects. - - -.. c:function:: int PyCoro_CheckExact(PyObject *ob) - - Return true if *ob*'s type is *PyCoro_Type*; *ob* must not be *NULL*. - - -.. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new coroutine object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. diff --git a/third_party/python/Doc/c-api/datetime.rst b/third_party/python/Doc/c-api/datetime.rst deleted file mode 100644 index 305e99036..000000000 --- a/third_party/python/Doc/c-api/datetime.rst +++ /dev/null @@ -1,209 +0,0 @@ -.. highlightlang:: c - -.. _datetimeobjects: - -DateTime Objects ----------------- - -Various date and time objects are supplied by the :mod:`datetime` module. -Before using any of these functions, the header file :file:`datetime.h` must be -included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of -the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following -macros. - -Type-check macros: - -.. c:function:: int PyDate_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be *NULL*. - - -.. c:function:: int PyDate_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - *NULL*. - - -.. c:function:: int PyDateTime_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be *NULL*. - - -.. c:function:: int PyDateTime_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be *NULL*. - - -.. c:function:: int PyTime_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be *NULL*. - - -.. c:function:: int PyTime_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - *NULL*. - - -.. c:function:: int PyDelta_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be *NULL*. - - -.. c:function:: int PyDelta_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - *NULL*. - - -.. c:function:: int PyTZInfo_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be *NULL*. - - -.. c:function:: int PyTZInfo_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - *NULL*. - - -Macros to create objects: - -.. c:function:: PyObject* PyDate_FromDate(int year, int month, int day) - - Return a ``datetime.date`` object with the specified year, month and day. - - -.. c:function:: PyObject* PyDateTime_FromDateAndTime(int year, int month, int day, int hour, int minute, int second, int usecond) - - Return a ``datetime.datetime`` object with the specified year, month, day, hour, - minute, second and microsecond. - - -.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) - - Return a ``datetime.time`` object with the specified hour, minute, second and - microsecond. - - -.. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) - - Return a ``datetime.timedelta`` object representing the given number of days, - seconds and microseconds. Normalization is performed so that the resulting - number of microseconds and seconds lie in the ranges documented for - ``datetime.timedelta`` objects. - - -Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be *NULL*, and the type is -not checked: - -.. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) - - Return the year, as a positive int. - - -.. c:function:: int PyDateTime_GET_MONTH(PyDateTime_Date *o) - - Return the month, as an int from 1 through 12. - - -.. c:function:: int PyDateTime_GET_DAY(PyDateTime_Date *o) - - Return the day, as an int from 1 through 31. - - -Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument -must not be *NULL*, and the type is not checked: - -.. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) - - Return the hour, as an int from 0 through 23. - - -.. c:function:: int PyDateTime_DATE_GET_MINUTE(PyDateTime_DateTime *o) - - Return the minute, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_DATE_GET_SECOND(PyDateTime_DateTime *o) - - Return the second, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_DATE_GET_MICROSECOND(PyDateTime_DateTime *o) - - Return the microsecond, as an int from 0 through 999999. - - -Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be *NULL*, -and the type is not checked: - -.. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) - - Return the hour, as an int from 0 through 23. - - -.. c:function:: int PyDateTime_TIME_GET_MINUTE(PyDateTime_Time *o) - - Return the minute, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_TIME_GET_SECOND(PyDateTime_Time *o) - - Return the second, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_TIME_GET_MICROSECOND(PyDateTime_Time *o) - - Return the microsecond, as an int from 0 through 999999. - - -Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must -not be *NULL*, and the type is not checked: - -.. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) - - Return the number of days, as an int from -999999999 to 999999999. - - .. versionadded:: 3.3 - - -.. c:function:: int PyDateTime_DELTA_GET_SECONDS(PyDateTime_Delta *o) - - Return the number of seconds, as an int from 0 through 86399. - - .. versionadded:: 3.3 - - -.. c:function:: int PyDateTime_DELTA_GET_MICROSECONDS(PyDateTime_Delta *o) - - Return the number of microseconds, as an int from 0 through 999999. - - .. versionadded:: 3.3 - - -Macros for the convenience of modules implementing the DB API: - -.. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) - - Create and return a new ``datetime.datetime`` object given an argument tuple - suitable for passing to ``datetime.datetime.fromtimestamp()``. - - -.. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) - - Create and return a new ``datetime.date`` object given an argument tuple - suitable for passing to ``datetime.date.fromtimestamp()``. diff --git a/third_party/python/Doc/c-api/descriptor.rst b/third_party/python/Doc/c-api/descriptor.rst deleted file mode 100644 index c8f6fa5bc..000000000 --- a/third_party/python/Doc/c-api/descriptor.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. highlightlang:: c - -.. _descriptor-objects: - -Descriptor Objects ------------------- - -"Descriptors" are objects that describe some attribute of an object. They are -found in the dictionary of type objects. - -.. XXX document these! - -.. c:var:: PyTypeObject PyProperty_Type - - The type object for the built-in descriptor types. - - -.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) - - -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) - - -.. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) - - -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) - - -.. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) - - -.. c:function:: int PyDescr_IsData(PyObject *descr) - - Return true if the descriptor objects *descr* describes a data attribute, or - false if it describes a method. *descr* must be a descriptor object; there is - no error checking. - - -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) diff --git a/third_party/python/Doc/c-api/dict.rst b/third_party/python/Doc/c-api/dict.rst deleted file mode 100644 index 68ca25ca5..000000000 --- a/third_party/python/Doc/c-api/dict.rst +++ /dev/null @@ -1,240 +0,0 @@ -.. highlightlang:: c - -.. _dictobjects: - -Dictionary Objects ------------------- - -.. index:: object: dictionary - - -.. c:type:: PyDictObject - - This subtype of :c:type:`PyObject` represents a Python dictionary object. - - -.. c:var:: PyTypeObject PyDict_Type - - This instance of :c:type:`PyTypeObject` represents the Python dictionary - type. This is the same object as :class:`dict` in the Python layer. - - -.. c:function:: int PyDict_Check(PyObject *p) - - Return true if *p* is a dict object or an instance of a subtype of the dict - type. - - -.. c:function:: int PyDict_CheckExact(PyObject *p) - - Return true if *p* is a dict object, but not an instance of a subtype of - the dict type. - - -.. c:function:: PyObject* PyDict_New() - - Return a new empty dictionary, or *NULL* on failure. - - -.. c:function:: PyObject* PyDictProxy_New(PyObject *mapping) - - Return a :class:`types.MappingProxyType` object for a mapping which - enforces read-only behavior. This is normally used to create a view to - prevent modification of the dictionary for non-dynamic class types. - - -.. c:function:: void PyDict_Clear(PyObject *p) - - Empty an existing dictionary of all key-value pairs. - - -.. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - - Determine if dictionary *p* contains *key*. If an item in *p* is matches - *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. - This is equivalent to the Python expression ``key in p``. - - -.. c:function:: PyObject* PyDict_Copy(PyObject *p) - - Return a new dictionary that contains the same key-value pairs as *p*. - - -.. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) - - Insert *value* into the dictionary *p* with a key of *key*. *key* must be - :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. - - -.. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) - - .. index:: single: PyUnicode_FromString() - - Insert *value* into the dictionary *p* using *key* as a key. *key* should - be a :c:type:`char\*`. The key object is created using - ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. - - -.. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) - - Remove the entry in dictionary *p* with key *key*. *key* must be hashable; - if it isn't, :exc:`TypeError` is raised. Return ``0`` on success or ``-1`` - on failure. - - -.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - - Remove the entry in dictionary *p* which has a key specified by the string - *key*. Return ``0`` on success or ``-1`` on failure. - - -.. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) - - Return the object from dictionary *p* which has a key *key*. Return *NULL* - if the key *key* is not present, but *without* setting an exception. - - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. - - -.. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) - - Variant of :c:func:`PyDict_GetItem` that does not suppress - exceptions. Return *NULL* **with** an exception set if an exception - occurred. Return *NULL* **without** an exception set if the key - wasn't present. - - -.. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) - - This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:type:`char\*`, rather than a :c:type:`PyObject\*`. - - Note that exceptions which occur while calling :meth:`__hash__` and - :meth:`__eq__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. - - -.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default) - - This is the same as the Python-level :meth:`dict.setdefault`. If present, it - returns the value corresponding to *key* from the dictionary *p*. If the key - is not in the dict, it is inserted with value *defaultobj* and *defaultobj* - is returned. This function evaluates the hash function of *key* only once, - instead of evaluating it independently for the lookup and the insertion. - - .. versionadded:: 3.4 - -.. c:function:: PyObject* PyDict_Items(PyObject *p) - - Return a :c:type:`PyListObject` containing all the items from the dictionary. - - -.. c:function:: PyObject* PyDict_Keys(PyObject *p) - - Return a :c:type:`PyListObject` containing all the keys from the dictionary. - - -.. c:function:: PyObject* PyDict_Values(PyObject *p) - - Return a :c:type:`PyListObject` containing all the values from the dictionary - *p*. - - -.. c:function:: Py_ssize_t PyDict_Size(PyObject *p) - - .. index:: builtin: len - - Return the number of items in the dictionary. This is equivalent to - ``len(p)`` on a dictionary. - - -.. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) - - Iterate over all key-value pairs in the dictionary *p*. The - :c:type:`Py_ssize_t` referred to by *ppos* must be initialized to ``0`` - prior to the first call to this function to start the iteration; the - function returns true for each pair in the dictionary, and false once all - pairs have been reported. The parameters *pkey* and *pvalue* should either - point to :c:type:`PyObject\*` variables that will be filled in with each key - and value, respectively, or may be *NULL*. Any references returned through - them are borrowed. *ppos* should not be altered during iteration. Its - value represents offsets within the internal dictionary structure, and - since the structure is sparse, the offsets are not consecutive. - - For example:: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - /* do something interesting with the values... */ - ... - } - - The dictionary *p* should not be mutated during iteration. It is safe to - modify the values of the keys as you iterate over the dictionary, but only - so long as the set of keys does not change. For example:: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - long i = PyLong_AsLong(value); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - PyObject *o = PyLong_FromLong(i + 1); - if (o == NULL) - return -1; - if (PyDict_SetItem(self->dict, key, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); - } - - -.. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) - - Iterate over mapping object *b* adding key-value pairs to dictionary *a*. - *b* may be a dictionary, or any object supporting :c:func:`PyMapping_Keys` - and :c:func:`PyObject_GetItem`. If *override* is true, existing pairs in *a* - will be replaced if a matching key is found in *b*, otherwise pairs will - only be added if there is not a matching key in *a*. Return ``0`` on - success or ``-1`` if an exception was raised. - - -.. c:function:: int PyDict_Update(PyObject *a, PyObject *b) - - This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to - ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall - back to the iterating over a sequence of key value pairs if the second - argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an - exception was raised. - - -.. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) - - Update or merge into dictionary *a*, from the key-value pairs in *seq2*. - *seq2* must be an iterable object producing iterable objects of length 2, - viewed as key-value pairs. In case of duplicate keys, the last wins if - *override* is true, else the first wins. Return ``0`` on success or ``-1`` - if an exception was raised. Equivalent Python (except for the return - value):: - - def PyDict_MergeFromSeq2(a, seq2, override): - for key, value in seq2: - if override or key not in a: - a[key] = value - - -.. c:function:: int PyDict_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/third_party/python/Doc/c-api/exceptions.rst b/third_party/python/Doc/c-api/exceptions.rst deleted file mode 100644 index 817469af0..000000000 --- a/third_party/python/Doc/c-api/exceptions.rst +++ /dev/null @@ -1,1020 +0,0 @@ -.. highlightlang:: c - - -.. _exceptionhandling: - -****************** -Exception Handling -****************** - -The functions described in this chapter will let you handle and raise Python -exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: -there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually *NULL* if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the :c:func:`PyArg_\*` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be NULL if non-set (although some combinations are -forbidden, for example you can't have a non-NULL traceback if the exception -type is NULL). - -When a function must fail because some function it called failed, it generally -doesn't set the error indicator; the function it called already set it. It is -responsible for either handling the error and clearing the exception or -returning after cleaning up any resources it holds (such as object references or -memory allocations); it should *not* continue normally if it is not prepared to -handle the error. If returning due to an error, it is important to indicate to -the caller that an error has been set. If the error is not handled or carefully -propagated, additional calls into the Python/C API may not behave as intended -and may fail in mysterious ways. - -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). - - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. - - -.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) - - Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``. In that case the no traceback - is printed and Python process will exit with the error code specified by - the ``SystemExit`` instance. - - Call this function **only** when the error indicator is set. Otherwise it - will cause a fatal error! - - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. - - -.. c:function:: void PyErr_Print() - - Alias for ``PyErr_PrintEx(1)``. - - -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) - - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. - - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. If possible, - the repr of *obj* will be printed in the warning message. - - -Raising exceptions -================== - -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -NULL pointer for use in a ``return`` statement. - - -.. c:function:: void PyErr_SetString(PyObject *type, const char *message) - - This is the most common way to set the error indicator. The first argument - specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. - - -.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) - - This function is similar to :c:func:`PyErr_SetString` but lets you specify an - arbitrary Python object for the "value" of the exception. - - -.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - - This function sets the error indicator and returns *NULL*. *exception* - should be a Python exception class. The *format* and subsequent - parameters help format the error message; they have the same meaning and - values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded - string. - - -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - -.. c:function:: void PyErr_SetNone(PyObject *type) - - This is a shorthand for ``PyErr_SetObject(type, Py_None)``. - - -.. c:function:: int PyErr_BadArgument() - - This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where - *message* indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use. - - -.. c:function:: PyObject* PyErr_NoMemory() - - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns *NULL* - so an object allocation function can write ``return PyErr_NoMemory();`` when it - runs out of memory. - - -.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - - .. index:: single: strerror() - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable :c:data:`errno`. It constructs a - tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), - and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, - this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns *NULL*, so a wrapper - function around a system call can write ``return PyErr_SetFromErrno(type);`` - when the system call returns an error. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not *NULL*, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the filesystem encoding - (:func:`os.fsdecode`). - - -.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - - This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` - is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns *NULL*. Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter - specifying the exception type to be raised. Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. - Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - Availability: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional - parameter specifying the exception type to be raised. Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) - - Set file, line, and offset information for the current exception. If the - current exception is not a :exc:`SyntaxError`, then it sets additional - attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. - - .. versionadded:: 3.4 - - -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) - - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is - omitted. - - -.. c:function:: void PyErr_BadInternalCall() - - This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, - where *message* indicates that an internal operation (e.g. a Python/C API - function) was invoked with an illegal argument. It is mostly for internal - use. - - -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) - - Issue a warning message. The *category* argument is a warning category (see - below) or *NULL*; the *message* argument is a UTF-8 encoded string. *stack_level* is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A *stack_level* of 1 - is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, - and so forth. - - Warning categories must be subclasses of :c:data:`PyExc_Warning`; - :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; - the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard - Python warning categories are available as global variables whose names are - enumerated at :ref:`standardwarningcategories`. - - For information about warning control, see the documentation for the - :mod:`warnings` module and the :option:`-W` option in the command line - documentation. There is no C API for warning control. - -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *msg, PyObject *name, PyObject *path) - - Much like :c:func:`PyErr_SetImportError` but this function allows for - specifying a subclass of :exc:`ImportError` to raise. - - .. versionadded:: 3.6 - - -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) - - Issue a warning message with explicit control over all warning attributes. This - is a straightforward wrapper around the Python function - :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to *NULL* to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - filesystem encoding (:func:`os.fsdecode`). - - -.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnEx`, but use - :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is - an ASCII-encoded string. - - .. versionadded:: 3.2 - - -.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. - - .. versionadded:: 3.6 - - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to *NULL*. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be *NULL* even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are *NULL*, the error - indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. Returns new references for the three objects, any of which - may be *NULL*. Does not modify the exception info state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass *NULL* for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - -Signal Handling -=============== - - -.. c:function:: int PyErr_CheckSignals() - - .. index:: - module: signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the :mod:`signal` module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an - exception is raised the error indicator is set and the function returns ``-1``; - otherwise the function returns ``0``. The error indicator may or may not be - cleared if it was previously set. - - -.. c:function:: void PyErr_SetInterrupt() - - .. index:: - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function simulates the effect of a :const:`SIGINT` signal arriving --- the - next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will - be raised. It may be called without holding the interpreter lock. - - .. % XXX This was described as obsolete, but is used in - .. % _thread.interrupt_main() (used from IDLE), so it's still needed. - - -.. c:function:: int PySignal_SetWakeupFd(int fd) - - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. - This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any - error checking. *fd* should be a valid file descriptor. The function should - only be called from the main thread. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) - - This utility function creates and returns a new exception class. The *name* - argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally *NULL*. - This creates a class object derived from :exc:`Exception` (accessible in C as - :c:data:`PyExc_Exception`). - - The :attr:`__module__` attribute of the new class is set to the first part (up - to the last dot) of the *name* argument, and the class name is set to the last - part (after the last dot). The *base* argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The *dict* - argument can be used to specify a dictionary of class variables and methods. - - -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) - - Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-*NULL*, it will be used as the - docstring for the exception class. - - .. versionadded:: 3.2 - - -Exception Objects -================= - -.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) - - Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns *NULL*. - - -.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) - - Set the traceback associated with the exception to *tb*. Use ``Py_None`` to - clear it. - - -.. c:function:: PyObject* PyException_GetContext(PyObject *ex) - - Return the context (another exception instance during whose handling *ex* was - raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns *NULL*. - - -.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - - Set the context associated with the exception to *ctx*. Use *NULL* to clear - it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. - - -.. c:function:: PyObject* PyException_GetCause(PyObject *ex) - - Return the cause (either an exception instance, or :const:`None`, - set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. - - -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - - Set the cause associated with the exception to *cause*. Use *NULL* to clear - it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. - - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. - - -.. _unicodeexceptions: - -Unicode Exception Objects -========================= - -The following functions are used to create and modify Unicode exceptions from C. - -.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeTranslateError` object with the attributes *object*, - *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) - - Return the *encoding* attribute of the given exception object. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) - - Return the *object* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) - - Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be *NULL*. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. - -.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) - - Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be *NULL*. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) - - Set the *end* attribute of the given exception object to *end*. Return ``0`` - on success, ``-1`` on failure. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) - - Return the *reason* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) - - Set the *reason* attribute of the given exception object to *reason*. Return - ``0`` on success, ``-1`` on failure. - - -Recursion Control -================= - -These two functions provide a way to perform safe recursive calls at the C -level, both in the core and in extension modules. They are needed if the -recursive code does not necessarily invoke Python code (which tracks its -recursion depth automatically). - -.. c:function:: int Py_EnterRecursiveCall(const char *where) - - Marks a point where a recursive C-level call is about to be performed. - - If :const:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. - Otherwise, zero is returned. - - *where* should be a string such as ``" in instance check"`` to be - concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. - -.. c:function:: void Py_LeaveRecursiveCall() - - Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each - *successful* invocation of :c:func:`Py_EnterRecursiveCall`. - -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires -special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The -following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. - -.. c:function:: int Py_ReprEnter(PyObject *object) - - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation - should return a string object indicating a cycle. As examples, - :class:`dict` objects return ``{...}`` and :class:`list` objects - return ``[...]``. - - The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should - typically return ``NULL``. - - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` - implementation can continue normally. - -.. c:function:: void Py_ReprLeave(PyObject *object) - - Ends a :c:func:`Py_ReprEnter`. Must be called once for each - invocation of :c:func:`Py_ReprEnter` that returns zero. - - -.. _standardexceptions: - -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.3 - :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, - :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, - :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, - :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, - :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, - :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, - :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` - and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. - -.. versionadded:: 3.5 - :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. - -.. versionadded:: 3.6 - :c:data:`PyExc_ModuleNotFoundError`. - -These are compatibility aliases to :c:data:`PyExc_OSError`: - -.. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | \(3) | -+-------------------------------------+----------+ - -.. versionchanged:: 3.3 - These aliases used to be separate exception types. - -Notes: - -(1) - This is a base class for other standard exceptions. - -(2) - This is the same as :exc:`weakref.ReferenceError`. - -(3) - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. - -.. _standardwarningcategories: - -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.2 - :c:data:`PyExc_ResourceWarning`. - -Notes: - -(1) - This is a base class for other standard warning categories. diff --git a/third_party/python/Doc/c-api/file.rst b/third_party/python/Doc/c-api/file.rst deleted file mode 100644 index 6f2ecee51..000000000 --- a/third_party/python/Doc/c-api/file.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. highlightlang:: c - -.. _fileobjects: - -File Objects ------------- - -.. index:: object: file - -These APIs are a minimal emulation of the Python 2 C API for built-in file -objects, which used to rely on the buffered I/O (:c:type:`FILE\*`) support -from the C standard library. In Python 3, files and streams use the new -:mod:`io` module, which defines several layers over the low-level unbuffered -I/O of the operating system. The functions described below are -convenience C wrappers over these new APIs, and meant mostly for internal -error reporting in the interpreter; third-party code is advised to access -the :mod:`io` APIs instead. - - -.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) - - Create a Python file object from the file descriptor of an already - opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* - can be *NULL* to use the defaults; *buffering* can be *-1* to use the - default. *name* is ignored and kept for backward compatibility. Return - *NULL* on failure. For a more comprehensive description of the arguments, - please refer to the :func:`io.open` function documentation. - - .. warning:: - - Since Python streams have their own buffering layer, mixing them with - OS-level file descriptors can produce various issues (such as unexpected - ordering of data). - - .. versionchanged:: 3.2 - Ignore *name* attribute. - - -.. c:function:: int PyObject_AsFileDescriptor(PyObject *p) - - Return the file descriptor associated with *p* as an :c:type:`int`. If the - object is an integer, its value is returned. If not, the - object's :meth:`~io.IOBase.fileno` method is called if it exists; the - method must return an integer, which is returned as the file descriptor - value. Sets an exception and returns ``-1`` on failure. - - -.. c:function:: PyObject* PyFile_GetLine(PyObject *p, int n) - - .. index:: single: EOFError (built-in exception) - - Equivalent to ``p.readline([n])``, this function reads one line from the - object *p*. *p* may be a file object or any object with a - :meth:`~io.IOBase.readline` - method. If *n* is ``0``, exactly one line is read, regardless of the length of - the line. If *n* is greater than ``0``, no more than *n* bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If *n* is less than - ``0``, however, one line is read regardless of length, but :exc:`EOFError` is - raised if the end of the file is reached immediately. - - -.. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - - .. index:: single: Py_PRINT_RAW - - Write object *obj* to file object *p*. The only supported flag for *flags* is - :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. - - -.. c:function:: int PyFile_WriteString(const char *s, PyObject *p) - - Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on - failure; the appropriate exception will be set. diff --git a/third_party/python/Doc/c-api/float.rst b/third_party/python/Doc/c-api/float.rst deleted file mode 100644 index 27a75e3e0..000000000 --- a/third_party/python/Doc/c-api/float.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. highlightlang:: c - -.. _floatobjects: - -Floating Point Objects ----------------------- - -.. index:: object: floating point - - -.. c:type:: PyFloatObject - - This subtype of :c:type:`PyObject` represents a Python floating point object. - - -.. c:var:: PyTypeObject PyFloat_Type - - This instance of :c:type:`PyTypeObject` represents the Python floating point - type. This is the same object as :class:`float` in the Python layer. - - -.. c:function:: int PyFloat_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyFloatObject` or a subtype of - :c:type:`PyFloatObject`. - - -.. c:function:: int PyFloat_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyFloatObject`, but not a subtype of - :c:type:`PyFloatObject`. - - -.. c:function:: PyObject* PyFloat_FromString(PyObject *str) - - Create a :c:type:`PyFloatObject` object based on the string value in *str*, or - *NULL* on failure. - - -.. c:function:: PyObject* PyFloat_FromDouble(double v) - - Create a :c:type:`PyFloatObject` object from *v*, or *NULL* on failure. - - -.. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) - - Return a C :c:type:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`__float__` - method, this method will first be called to convert *pyfloat* into a float. - This method returns ``-1.0`` upon failure, so one should call - :c:func:`PyErr_Occurred` to check for errors. - - -.. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) - - Return a C :c:type:`double` representation of the contents of *pyfloat*, but - without error checking. - - -.. c:function:: PyObject* PyFloat_GetInfo(void) - - Return a structseq instance which contains information about the - precision, minimum and maximum values of a float. It's a thin wrapper - around the header file :file:`float.h`. - - -.. c:function:: double PyFloat_GetMax() - - Return the maximum representable finite float *DBL_MAX* as C :c:type:`double`. - - -.. c:function:: double PyFloat_GetMin() - - Return the minimum normalized positive float *DBL_MIN* as C :c:type:`double`. - -.. c:function:: int PyFloat_ClearFreeList() - - Clear the float free list. Return the number of items that could not - be freed. diff --git a/third_party/python/Doc/c-api/function.rst b/third_party/python/Doc/c-api/function.rst deleted file mode 100644 index 17279c732..000000000 --- a/third_party/python/Doc/c-api/function.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. highlightlang:: c - -.. _function-objects: - -Function Objects ----------------- - -.. index:: object: function - -There are a few functions specific to Python functions. - - -.. c:type:: PyFunctionObject - - The C structure used for functions. - - -.. c:var:: PyTypeObject PyFunction_Type - - .. index:: single: MethodType (in module types) - - This is an instance of :c:type:`PyTypeObject` and represents the Python function - type. It is exposed to Python programmers as ``types.FunctionType``. - - -.. c:function:: int PyFunction_Check(PyObject *o) - - Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be *NULL*. - - -.. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) - - Return a new function object associated with the code object *code*. *globals* - must be a dictionary with the global variables accessible to the function. - - The function's docstring and name are retrieved from the code object. *__module__* - is retrieved from *globals*. The argument defaults, annotations and closure are - set to *NULL*. *__qualname__* is set to the same value as the function's name. - - -.. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) - - As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or NULL; - if NULL, the ``__qualname__`` attribute is set to the same value as its - ``__name__`` attribute. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyFunction_GetCode(PyObject *op) - - Return the code object associated with the function object *op*. - - -.. c:function:: PyObject* PyFunction_GetGlobals(PyObject *op) - - Return the globals dictionary associated with the function object *op*. - - -.. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - - Return the *__module__* attribute of the function object *op*. This is normally - a string containing the module name, but can be set to any other object by - Python code. - - -.. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) - - Return the argument default values of the function object *op*. This can be a - tuple of arguments or *NULL*. - - -.. c:function:: int PyFunction_SetDefaults(PyObject *op, PyObject *defaults) - - Set the argument default values for the function object *op*. *defaults* must be - *Py_None* or a tuple. - - Raises :exc:`SystemError` and returns ``-1`` on failure. - - -.. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) - - Return the closure associated with the function object *op*. This can be *NULL* - or a tuple of cell objects. - - -.. c:function:: int PyFunction_SetClosure(PyObject *op, PyObject *closure) - - Set the closure associated with the function object *op*. *closure* must be - *Py_None* or a tuple of cell objects. - - Raises :exc:`SystemError` and returns ``-1`` on failure. - - -.. c:function:: PyObject *PyFunction_GetAnnotations(PyObject *op) - - Return the annotations of the function object *op*. This can be a - mutable dictionary or *NULL*. - - -.. c:function:: int PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) - - Set the annotations for the function object *op*. *annotations* - must be a dictionary or *Py_None*. - - Raises :exc:`SystemError` and returns ``-1`` on failure. diff --git a/third_party/python/Doc/c-api/gcsupport.rst b/third_party/python/Doc/c-api/gcsupport.rst deleted file mode 100644 index 472cd93ec..000000000 --- a/third_party/python/Doc/c-api/gcsupport.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. highlightlang:: c - -.. _supporting-cycle-detection: - -Supporting Cyclic Garbage Collection -==================================== - -Python's support for detecting and collecting garbage which involves circular -references requires support from object types which are "containers" for other -objects which may also be containers. Types which do not store references to -other objects, or which only store references to atomic types (such as numbers -or strings), do not need to provide any explicit support for garbage -collection. - -To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must -include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the -:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a -:c:member:`~PyTypeObject.tp_clear` implementation must also be provided. - - -.. data:: Py_TPFLAGS_HAVE_GC - :noindex: - - Objects with a type with this flag set must conform with the rules - documented here. For convenience these objects will be referred to as - container objects. - -Constructors for container types must conform to two rules: - -#. The memory for the object must be allocated using :c:func:`PyObject_GC_New` - or :c:func:`PyObject_GC_NewVar`. - -#. Once all the fields which may contain references to other containers are - initialized, it must call :c:func:`PyObject_GC_Track`. - - -.. c:function:: TYPE* PyObject_GC_New(TYPE, PyTypeObject *type) - - Analogous to :c:func:`PyObject_New` but for container objects with the - :const:`Py_TPFLAGS_HAVE_GC` flag set. - - -.. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) - - Analogous to :c:func:`PyObject_NewVar` but for container objects with the - :const:`Py_TPFLAGS_HAVE_GC` flag set. - - -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) - - Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the - resized object or *NULL* on failure. *op* must not be tracked by the collector yet. - - -.. c:function:: void PyObject_GC_Track(PyObject *op) - - Adds the object *op* to the set of container objects tracked by the - collector. The collector can run at unexpected times so objects must be - valid while being tracked. This should be called once all the fields - followed by the :c:member:`~PyTypeObject.tp_traverse` handler become valid, usually near the - end of the constructor. - - -.. c:function:: void _PyObject_GC_TRACK(PyObject *op) - - A macro version of :c:func:`PyObject_GC_Track`. It should not be used for - extension modules. - - .. deprecated:: 3.6 - This macro is removed from Python 3.8. - -Similarly, the deallocator for the object must conform to a similar pair of -rules: - -#. Before fields which refer to other containers are invalidated, - :c:func:`PyObject_GC_UnTrack` must be called. - -#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`. - - -.. c:function:: void PyObject_GC_Del(void *op) - - Releases memory allocated to an object using :c:func:`PyObject_GC_New` or - :c:func:`PyObject_GC_NewVar`. - - -.. c:function:: void PyObject_GC_UnTrack(void *op) - - Remove the object *op* from the set of container objects tracked by the - collector. Note that :c:func:`PyObject_GC_Track` can be called again on - this object to add it back to the set of tracked objects. The deallocator - (:c:member:`~PyTypeObject.tp_dealloc` handler) should call this for the object before any of - the fields used by the :c:member:`~PyTypeObject.tp_traverse` handler become invalid. - - -.. c:function:: void _PyObject_GC_UNTRACK(PyObject *op) - - A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for - extension modules. - - .. deprecated:: 3.6 - This macro is removed from Python 3.8. - -The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: - - -.. c:type:: int (*visitproc)(PyObject *object, void *arg) - - Type of the visitor function passed to the :c:member:`~PyTypeObject.tp_traverse` handler. - The function should be called with an object to traverse as *object* and - the third parameter to the :c:member:`~PyTypeObject.tp_traverse` handler as *arg*. The - Python core uses several visitor functions to implement cyclic garbage - detection; it's not expected that users will need to write their own - visitor functions. - -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: - - -.. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) - - Traversal function for a container object. Implementations must call the - *visit* function for each object directly contained by *self*, with the - parameters to *visit* being the contained object and the *arg* value passed - to the handler. The *visit* function must not be called with a *NULL* - object argument. If *visit* returns a non-zero value that value should be - returned immediately. - -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation -must name its arguments exactly *visit* and *arg*: - - -.. c:function:: void Py_VISIT(PyObject *o) - - If *o* is not *NULL*, call the *visit* callback, with arguments *o* - and *arg*. If *visit* returns a non-zero value, then return it. - Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers - look like:: - - static int - my_traverse(Noddy *self, visitproc visit, void *arg) - { - Py_VISIT(self->foo); - Py_VISIT(self->bar); - return 0; - } - -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* -if the object is immutable. - - -.. c:type:: int (*inquiry)(PyObject *self) - - Drop references that may have created reference cycles. Immutable objects - do not have to define this method since they can never directly create - reference cycles. Note that the object must still be valid after calling - this method (don't just call :c:func:`Py_DECREF` on a reference). The - collector will call this method if it detects that this object is involved - in a reference cycle. diff --git a/third_party/python/Doc/c-api/gen.rst b/third_party/python/Doc/c-api/gen.rst deleted file mode 100644 index 1efbae4fc..000000000 --- a/third_party/python/Doc/c-api/gen.rst +++ /dev/null @@ -1,44 +0,0 @@ -.. highlightlang:: c - -.. _gen-objects: - -Generator Objects ------------------ - -Generator objects are what Python uses to implement generator iterators. They -are normally created by iterating over a function that yields values, rather -than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. - - -.. c:type:: PyGenObject - - The C structure used for generator objects. - - -.. c:var:: PyTypeObject PyGen_Type - - The type object corresponding to generator objects. - - -.. c:function:: int PyGen_Check(PyObject *ob) - - Return true if *ob* is a generator object; *ob* must not be *NULL*. - - -.. c:function:: int PyGen_CheckExact(PyObject *ob) - - Return true if *ob*'s type is *PyGen_Type*; *ob* must not be *NULL*. - - -.. c:function:: PyObject* PyGen_New(PyFrameObject *frame) - - Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be - *NULL*. - -.. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new generator object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. diff --git a/third_party/python/Doc/c-api/import.rst b/third_party/python/Doc/c-api/import.rst deleted file mode 100644 index 527363366..000000000 --- a/third_party/python/Doc/c-api/import.rst +++ /dev/null @@ -1,315 +0,0 @@ -.. highlightlang:: c - -.. _importing: - -Importing Modules -================= - - -.. c:function:: PyObject* PyImport_ImportModule(const char *name) - - .. index:: - single: package variable; __all__ - single: __all__ (package variable) - single: modules (in module sys) - - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to *NULL* and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or *NULL* with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - - -.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. - - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - - -.. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) - - .. index:: builtin: __import__ - - Import a module. This is best described by referring to the built-in Python - function :func:`__import__`. - - The return value is a new reference to the imported module or top-level - package, or *NULL* with an exception set on failure. Like for - :func:`__import__`, the return value when a submodule of a package was - requested is normally the top-level package, unless a non-empty *fromlist* - was given. - - Failing imports remove incomplete module objects, like with - :c:func:`PyImport_ImportModule`. - - -.. c:function:: PyObject* PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) - - Import a module. This is best described by referring to the built-in Python - function :func:`__import__`, as the standard :func:`__import__` function calls - this function directly. - - The return value is a new reference to the imported module or top-level package, - or *NULL* with an exception set on failure. Like for :func:`__import__`, - the return value when a submodule of a package was requested is normally the - top-level package, unless a non-empty *fromlist* was given. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) - - Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is a - UTF-8 encoded string instead of a Unicode object. - - .. versionchanged:: 3.3 - Negative values for *level* are no longer accepted. - -.. c:function:: PyObject* PyImport_Import(PyObject *name) - - This is a higher-level interface that calls the current "import hook - function" (with an explicit *level* of 0, meaning absolute import). It - invokes the :func:`__import__` function from the ``__builtins__`` of the - current globals. This means that the import is done using whatever import - hooks are installed in the current environment. - - This function always uses absolute imports. - - -.. c:function:: PyObject* PyImport_ReloadModule(PyObject *m) - - Reload a module. Return a new reference to the reloaded module, or *NULL* with - an exception set on failure (the module still exists in this case). - - -.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name) - - Return the module object corresponding to a module name. The *name* argument - may be of the form ``package.module``. First check the modules dictionary if - there's one there, and if not, create a new one and insert it in the modules - dictionary. Return *NULL* with an exception set on failure. - - .. note:: - - This function does not load or import the module; if the module wasn't already - loaded, you will get an empty module object. Use :c:func:`PyImport_ImportModule` - or one of its variants to import a module. Package structures implied by a - dotted name for *name* are not created if not already present. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_AddModule(const char *name) - - Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8 - encoded string instead of a Unicode object. - - -.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) - - .. index:: builtin: compile - - Given a module name (possibly of the form ``package.module``) and a code object - read from a Python bytecode file or obtained from the built-in function - :func:`compile`, load the module. Return a new reference to the module object, - or *NULL* with an exception set if an error occurred. *name* - is removed from :attr:`sys.modules` in error cases, even if *name* was already - in :attr:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving - incompletely initialized modules in :attr:`sys.modules` is dangerous, as imports of - such modules have no way to know that the module object is an unknown (and - probably damaged with respect to the module author's intents) state. - - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`SourceFileLoader` otherwise. - - The module's :attr:`__file__` attribute will be set to the code object's - :c:member:`co_filename`. If applicable, :attr:`__cached__` will also - be set. - - This function will reload the module if it was already imported. See - :c:func:`PyImport_ReloadModule` for the intended way to reload a module. - - If *name* points to a dotted name of the form ``package.module``, any package - structures not already created will still not be created. - - See also :c:func:`PyImport_ExecCodeModuleEx` and - :c:func:`PyImport_ExecCodeModuleWithPathnames`. - - -.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. - - See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. - - -.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) - - Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and - *cpathname* are UTF-8 encoded strings. Attempts are also made to figure out - what the value for *pathname* should be from *cpathname* if the former is - set to ``NULL``. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Uses :func:`imp.source_from_cache()` in calculating the source path if - only the bytecode path is provided. - - -.. c:function:: long PyImport_GetMagicNumber() - - Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` file). - The magic number should be present in the first four bytes of the bytecode - file, in little-endian byte order. Returns ``-1`` on error. - - .. versionchanged:: 3.3 - Return value of ``-1`` upon failure. - - -.. c:function:: const char * PyImport_GetMagicTag() - - Return the magic tag string for :pep:`3147` format Python bytecode file - names. Keep in mind that the value at ``sys.implementation.cache_tag`` is - authoritative and should be used instead of this function. - - .. versionadded:: 3.2 - -.. c:function:: PyObject* PyImport_GetModuleDict() - - Return the dictionary used for the module administration (a.k.a. - ``sys.modules``). Note that this is a per-interpreter variable. - - -.. c:function:: PyObject* PyImport_GetImporter(PyObject *path) - - Return a finder object for a :data:`sys.path`/:attr:`pkg.__path__` item - *path*, possibly by fetching it from the :data:`sys.path_importer_cache` - dict. If it wasn't yet cached, traverse :data:`sys.path_hooks` until a hook - is found that can handle the path item. Return ``None`` if no hook could; - this tells our caller that the :term:`path based finder` could not find a - finder for this path item. Cache the result in :data:`sys.path_importer_cache`. - Return a new reference to the finder object. - - -.. c:function:: void _PyImport_Init() - - Initialize the import mechanism. For internal use only. - - -.. c:function:: void PyImport_Cleanup() - - Empty the module table. For internal use only. - - -.. c:function:: void _PyImport_Fini() - - Finalize the import mechanism. For internal use only. - - -.. c:function:: PyObject* _PyImport_FindExtension(char *, char *) - - For internal use only. - - -.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) - - Load a frozen module named *name*. Return ``1`` for success, ``0`` if the - module is not found, and ``-1`` with an exception set if the initialization - failed. To access the imported module on a successful load, use - :c:func:`PyImport_ImportModule`. (Note the misnomer --- this function would - reload the module if it was already imported.) - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The ``__file__`` attribute is no longer set on the module. - - -.. c:function:: int PyImport_ImportFrozenModule(const char *name) - - Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a - UTF-8 encoded string instead of a Unicode object. - - -.. c:type:: struct _frozen - - .. index:: single: freeze utility - - This is the structure type definition for frozen module descriptors, as - generated by the :program:`freeze` utility (see :file:`Tools/freeze/` in the - Python source distribution). Its definition, found in :file:`Include/import.h`, - is:: - - struct _frozen { - char *name; - unsigned char *code; - int size; - }; - - -.. c:var:: const struct _frozen* PyImport_FrozenModules - - This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen - module is imported, it is searched in this table. Third-party code could play - tricks with this to provide a dynamically created collection of frozen modules. - - -.. c:function:: int PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) - - Add a single module to the existing table of built-in modules. This is a - convenience wrapper around :c:func:`PyImport_ExtendInittab`, returning ``-1`` if - the table could not be extended. The new module can be imported by the name - *name*, and uses the function *initfunc* as the initialization function called - on the first attempted import. This should be called before - :c:func:`Py_Initialize`. - - -.. c:type:: struct _inittab - - Structure describing a single entry in the list of built-in modules. Each of - these structures gives the name and initialization function for a module built - into the interpreter. The name is an ASCII encoded string. Programs which - embed Python may use an array of these structures in conjunction with - :c:func:`PyImport_ExtendInittab` to provide additional built-in modules. - The structure is defined in :file:`Include/import.h` as:: - - struct _inittab { - char *name; /* ASCII encoded string */ - PyObject* (*initfunc)(void); - }; - - -.. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) - - Add a collection of modules to the table of built-in modules. The *newtab* - array must end with a sentinel entry which contains *NULL* for the :attr:`name` - field; failure to provide the sentinel value can result in a memory fault. - Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to - extend the internal table. In the event of failure, no modules are added to the - internal table. This should be called before :c:func:`Py_Initialize`. diff --git a/third_party/python/Doc/c-api/index.rst b/third_party/python/Doc/c-api/index.rst deleted file mode 100644 index 3bfbaf4ee..000000000 --- a/third_party/python/Doc/c-api/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _c-api-index: - -################################## - Python/C API Reference Manual -################################## - -This manual documents the API used by C and C++ programmers who want to write -extension modules or embed Python. It is a companion to :ref:`extending-index`, -which describes the general principles of extension writing but does not -document the API functions in detail. - -.. toctree:: - :maxdepth: 2 - - intro.rst - stable.rst - veryhigh.rst - refcounting.rst - exceptions.rst - utilities.rst - abstract.rst - concrete.rst - init.rst - memory.rst - objimpl.rst - apiabiversion.rst diff --git a/third_party/python/Doc/c-api/init.rst b/third_party/python/Doc/c-api/init.rst deleted file mode 100644 index f60147621..000000000 --- a/third_party/python/Doc/c-api/init.rst +++ /dev/null @@ -1,1227 +0,0 @@ -.. highlightlang:: c - - -.. _initialization: - -***************************************** -Initialization, Finalization, and Threads -***************************************** - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: Py_SetProgramName() - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - module: builtins - module: __main__ - module: sys - triple: module; search; path - single: PySys_SetArgv() - single: PySys_SetArgvEx() - single: Py_FinalizeEx() - - Initialize the Python interpreter. In an application embedding Python, this - should be called before using any other Python/C API functions; with the - exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -Process-wide parameters -======================= - - -.. c:function:: int Py_SetStandardStreamEncoding(const char *encoding, const char *errors) - - .. index:: - single: Py_Initialize() - single: main() - triple: stdin; stdout; sdterr - - This function should be called before :c:func:`Py_Initialize`, if it is - called at all. It specifies which encoding and error handling to use - with standard IO, with the same meanings as in :func:`str.encode`. - - It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code - to control IO encoding when the environment variable does not work. - - ``encoding`` and/or ``errors`` may be NULL to use - :envvar:`PYTHONIOENCODING` and/or default values (depending on other - settings). - - Note that :data:`sys.stderr` always uses the "backslashreplace" error - handler, regardless of this (or any other) setting. - - If :c:func:`Py_FinalizeEx` is called, this function will need to be called - again in order to affect subsequent calls to :c:func:`Py_Initialize`. - - Returns ``0`` if successful, a nonzero value on error (e.g. calling after the - interpreter has already been initialized). - - .. versionadded:: 3.4 - - -.. c:function:: void Py_SetProgramName(wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - -.. c:function:: wchar* Py_GetProgramName() - - .. index:: single: Py_SetProgramName() - - Return the program name set with :c:func:`Py_SetProgramName`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the ``--prefix`` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.prefix``. - It is only useful on Unix. See also the next function. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.exec_prefix``. It is only useful on Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: Py_SetProgramName() - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:func:`Py_SetProgramName` above). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - single: Py_SetPath() - - Return the default module search path; this is computed from the program name - (set by :c:func:`Py_SetProgramName` above) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and Mac OS X, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - .. XXX should give the exact rules - - -.. c:function:: void Py_SetPath(const wchar_t *) - - .. index:: - triple: module; search; path - single: path (in module sys) - single: Py_GetPath() - - Set the default module search path. If this function is called before - :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a - default search path but uses the one provided instead. This is useful if - Python is embedded by an application that has full knowledge of the location - of all modules. The path components should be separated by the platform - dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` - on Windows. - - This also causes :data:`sys.executable` to be set only to the raw program - name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and - :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these - if required after calling :c:func:`Py_Initialize`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - The path argument is copied internally, so the caller may free it after the - call completes. - - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first three characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On Mac OS X, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See `CVE-2008-5983 `_. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - -.. c:function:: void Py_SetPythonHome(wchar_t *home) - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - -.. c:function:: w_char* Py_GetPythonHome() - - Return the default "home", that is, the value set by a previous call to - :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -The Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval() (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState - single: PyThreadState - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`. There's also one -global variable pointing to the current :c:type:`PyThreadState`: it can -be retrieved using :c:func:`PyThreadState_Get`. - -Releasing the GIL from extension code -------------------------------------- - -Most extension code manipulating the :term:`GIL` has the following simple -structure:: - - Save the thread state in a local variable. - Release the global interpreter lock. - ... Do some blocking I/O operation ... - Reacquire the global interpreter lock. - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. These two macros are still available when Python is compiled without -thread support (they simply have an empty expansion). - -When thread support is enabled, the block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ...Do some blocking I/O operation... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() - -Here is how these functions work: the global interpreter lock is used to protect the pointer to the -current thread state. When releasing the lock and saving the thread state, -the current thread state pointer must be retrieved before the lock is released -(since another thread could immediately acquire the lock and store its own thread -state in the global variable). Conversely, when acquiring the lock and restoring -the thread state, the lock must be acquired before storing the thread state -pointer. - -.. note:: - Calling system I/O functions is the most common use case for releasing - the GIL, but it can also be useful before calling long-running computations - which don't need access to Python objects, such as compression or - cryptographic functions operating over memory buffers. For example, the - standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when - compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the GIL, nor is there a thread state structure -for them. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating a thread state data structure, then acquiring the GIL, and finally -storing their thread state pointer, before you can start using the Python/C -API. When you are done, you should reset the thread state pointer, release -the GIL, and finally free the thread state data structure. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the :c:func:`PyGILState_\*` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -:c:func:`PyGILState_\*` API is unsupported. - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. That also -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork` tries to reset the necessary locks, but is not -always able to. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is :c:type:`PyInterpreterState \*`:attr:`interp`, which points to - this thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Initialize and acquire the global interpreter lock. It should be called in the - main thread before creating a second thread or engaging in any other thread - operations such as ``PyEval_ReleaseThread(tstate)``. It is not needed before - calling :c:func:`PyEval_SaveThread` or :c:func:`PyEval_RestoreThread`. - - This is a no-op when called for a second time. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. index:: module: _thread - - .. note:: - - When only the main thread exists, no GIL operations are needed. This is a - common situation (most Python programs do not use threads), and the lock - operations slow the interpreter down a bit. Therefore, the lock is not - created initially. This situation is equivalent to having acquired the lock: - when there is only a single thread, all object accesses are safe. Therefore, - when this function initializes the global interpreter lock, it also acquires - it. Before the Python :mod:`_thread` module creates a new thread, knowing - that either it has the lock or the lock hasn't been created yet, it calls - :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that - the lock has been created and that the calling thread has acquired it. - - It is **not** safe to call this function when it is unknown which thread (if - any) currently has the global interpreter lock. - - This function is not available when thread support is disabled at compile time. - - -.. c:function:: int PyEval_ThreadsInitialized() - - Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This - function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. This function is - not available when thread support is disabled at compile time. - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Release the global interpreter lock (if it has been created and thread - support is enabled) and reset the thread state to *NULL*, returning the - previous thread state (which is not *NULL*). If the lock has been created, - the current thread must have acquired it. (This function is available even - when thread support is disabled at compile time.) - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Acquire the global interpreter lock (if it has been created and thread - support is enabled) and set the thread state to *tstate*, which must not be - *NULL*. If the lock has been created, the current thread must not have - acquired it, otherwise deadlock ensues. (This function is available even - when thread support is disabled at compile time.) - - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the current thread state. The global interpreter lock must be held. - When the current thread state is *NULL*, this issues a fatal error (so that - the caller needn't check for *NULL*). - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Swap the current thread state with the thread state given by the argument - *tstate*, which may be *NULL*. The global interpreter lock must be held - and is not released. - - -.. c:function:: void PyEval_ReInitThreads() - - This function is called from :c:func:`PyOS_AfterFork` to ensure that newly - created child processes don't hold locks referring to threads which - are not running in the child process. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the global interpreter lock. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the thread state when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, the current thread will hold the GIL and be able - to call arbitrary Python code. Failure is a fatal error. - - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the current thread state for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the GIL and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the GIL will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the GIL is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. It is a no-op when thread support is disabled at compile time. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. It is a no-op when thread support is disabled at compile time. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when - thread support is disabled at compile time. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. It is a no-op when thread support is disabled at compile time. - - -Low-level API -------------- - -All of the following functions are only available when thread support is enabled -at compile time, and must be called only when the global interpreter lock has -been created. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. The global interpreter lock need not - be held, but may be held if it is necessary to serialize calls to this - function. - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. The global interpreter - lock must be held. - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. The global interpreter lock need not be - held. The interpreter state must have been reset with a previous call to - :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - The global interpreter lock need not be held, but may be held if it is - necessary to serialize calls to this function. - - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a thread state object. The global interpreter lock - must be held. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a thread state object. The global interpreter lock need not be held. - The thread state must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no current thread state - is available. If this function returns *NULL*, no exception has been raised and - the caller should assume no current thread state is available. - - -.. c:function:: int PyThreadState_SetAsyncExc(long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with the GIL held. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is :const:`NULL`, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - Acquire the global interpreter lock and set the current thread state to - *tstate*, which should not be *NULL*. The lock must have been created earlier. - If this thread already has the lock, deadlock ensues. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Reset the current thread state to *NULL* and release the global interpreter - lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be *NULL*, is only used to check - that it represents the current thread state --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). - - -.. c:function:: void PyEval_AcquireLock() - - Acquire the global interpreter lock. The lock must have been created earlier. - If this thread already has the lock, a deadlock ensues. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread` - instead. - - -.. c:function:: void PyEval_ReleaseLock() - - Release the global interpreter lock. The lock must have been created earlier. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_SaveThread` or :c:func:`PyEval_ReleaseThread` - instead. - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. You can switch between sub-interpreters using the -:c:func:`PyThreadState_Swap` function. You can create and destroy them -using the following functions: - - -.. c:function:: PyThreadState* Py_NewInterpreter() - - .. index:: - module: builtins - module: __main__ - module: sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The return value points to the first thread state created in the new - sub-interpreter. This thread state is made in the current thread state. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, *NULL* is - returned; no exception is set since the exception state is stored in the - current thread state and there may not be a current thread state. (Like all - other Python/C API functions, the global interpreter lock must be held before - calling this function and is still held when it returns; however, unlike most - other Python/C API functions, there needn't be a current thread state on - entry.) - - .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() - - Extension modules are shared between (sub-)interpreters as follows: the first - time a particular extension is imported, it is initialized normally, and a - (shallow) copy of its module's dictionary is squirreled away. When the same - extension is imported by another (sub-)interpreter, a new module is initialized - and filled with the contents of this copy; the extension's ``init`` function is - not called. Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's - ``initmodule`` function *is* called again. - - .. index:: single: close() (in module os) - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx() - - Destroy the (sub-)interpreter represented by the given thread state. The given - thread state must be the current thread state. See the discussion of thread - states below. When the call returns, the current thread state is *NULL*. All - thread states associated with this interpreter are destroyed. (The global - interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when the extension makes use of -(static) global variables, or when the extension manipulates its module's -dictionary after its initialization. It is possible to insert objects created -in one sub-interpreter into a namespace of another sub-interpreter; this should -be done with great care to avoid sharing user-defined functions, methods, -instances or classes between sub-interpreters, since import operations executed -by such objects may affect the wrong (sub-)interpreter's dictionary of loaded -modules. - -Also note that combining this functionality with :c:func:`PyGILState_\*` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - .. index:: single: Py_AddPendingCall() - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding the :term:`global interpreter lock` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the global interpreter lock is released. - - This function doesn't need a current thread state to run, and it doesn't - need the global interpreter lock. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :const:`PyTrace_CALL`, - :const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`, - :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, or - :const:`PyTrace_C_RETURN`, and *arg* depends on the value of *what*: - - +------------------------------+--------------------------------------+ - | Value of *what* | Meaning of *arg* | - +==============================+======================================+ - | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_RETURN` | Value being returned to the caller, | - | | or *NULL* if caused by an exception. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_CALL` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_EXCEPTION` | Function object being called. | - +------------------------------+--------------------------------------+ - | :const:`PyTrace_C_RETURN` | Function object being called. | - +------------------------------+--------------------------------------+ - - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a trace function (but not a - profiling function) when a line-number event is being reported. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or *NULL*. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :const:`PyTrace_LINE` - and :const:`PyTrace_EXCEPTION`. - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and does not receive any event related to C function objects being called. Any - trace function registered using :c:func:`PyEval_SetTrace` will not receive - :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN` - as a value for the *what* parameter. - - -.. c:function:: PyObject* PyEval_GetCallStats(PyObject *self) - - Return a tuple of function call counts. There are constants defined for the - positions within the tuple: - - +-------------------------------+-------+ - | Name | Value | - +===============================+=======+ - | :const:`PCALL_ALL` | 0 | - +-------------------------------+-------+ - | :const:`PCALL_FUNCTION` | 1 | - +-------------------------------+-------+ - | :const:`PCALL_FAST_FUNCTION` | 2 | - +-------------------------------+-------+ - | :const:`PCALL_FASTER_FUNCTION`| 3 | - +-------------------------------+-------+ - | :const:`PCALL_METHOD` | 4 | - +-------------------------------+-------+ - | :const:`PCALL_BOUND_METHOD` | 5 | - +-------------------------------+-------+ - | :const:`PCALL_CFUNCTION` | 6 | - +-------------------------------+-------+ - | :const:`PCALL_TYPE` | 7 | - +-------------------------------+-------+ - | :const:`PCALL_GENERATOR` | 8 | - +-------------------------------+-------+ - | :const:`PCALL_OTHER` | 9 | - +-------------------------------+-------+ - | :const:`PCALL_POP` | 10 | - +-------------------------------+-------+ - - :const:`PCALL_FAST_FUNCTION` means no argument tuple needs to be created. - :const:`PCALL_FASTER_FUNCTION` means that the fast-path frame setup code is used. - - If there is a method call where the call can be optimized by changing - the argument tuple and calling the function directly, it gets recorded - twice. - - This function is only present if Python is compiled with :const:`CALL_PROFILE` - defined. - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - diff --git a/third_party/python/Doc/c-api/intro.rst b/third_party/python/Doc/c-api/intro.rst deleted file mode 100644 index 74681d2c8..000000000 --- a/third_party/python/Doc/c-api/intro.rst +++ /dev/null @@ -1,645 +0,0 @@ -.. highlightlang:: c - - -.. _api-intro: - -************ -Introduction -************ - -The Application Programmer's Interface to Python gives C and C++ programmers -access to the Python interpreter at a variety of levels. The API is equally -usable from C++, but for brevity it is generally referred to as the Python/C -API. There are two fundamentally different reasons for using the Python/C API. -The first reason is to write *extension modules* for specific purposes; these -are C modules that extend the Python interpreter. This is probably the most -common use. The second reason is to use Python as a component in a larger -application; this technique is generally referred to as :dfn:`embedding` Python -in an application. - -Writing an extension module is a relatively well-understood process, where a -"cookbook" approach works well. There are several tools that automate the -process to some extent. While people have embedded Python in other -applications since its early existence, the process of embedding Python is less -straightforward than writing an extension. - -Many API functions are useful independent of whether you're embedding or -extending Python; moreover, most applications that embed Python will need to -provide a custom extension as well, so it's probably a good idea to become -familiar with writing an extension before attempting to embed Python in a real -application. - - -.. _api-includes: - -Include Files -============= - -All function, type and macro definitions needed to use the Python/C API are -included in your code by the following line:: - - #include "Python.h" - -This implies inclusion of the following standard headers: ````, -````, ````, ````, ```` and ```` -(if available). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - -All user visible names defined by Python.h (except those defined by the included -standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning -with ``_Py`` are for internal use by the Python implementation and should not be -used by extension writers. Structure member names do not have a reserved prefix. - -**Important:** user code should never define names that begin with ``Py`` or -``_Py``. This confuses the reader, and jeopardizes the portability of the user -code to future Python versions, which may define additional names beginning with -one of these prefixes. - -The header files are typically installed with Python. On Unix, these are -located in the directories :file:`{prefix}/include/pythonversion/` and -:file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and -:envvar:`exec_prefix` are defined by the corresponding parameters to Python's -:program:`configure` script and *version* is -``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed -in :file:`{prefix}/include`, where :envvar:`prefix` is the installation -directory specified to the installer. - -To include the headers, place both directories (if different) on your compiler's -search path for includes. Do *not* place the parent directories on the search -path and then use ``#include ``; this will break on -multi-platform builds since the platform independent headers under -:envvar:`prefix` include the platform specific headers from -:envvar:`exec_prefix`. - -C++ users should note that though the API is defined entirely using C, the -header files do properly declare the entry points to be ``extern "C"``, so there -is no need to do anything special to use the API from C++. - - -.. _api-objects: - -Objects, Types and Reference Counts -=================================== - -.. index:: object: type - -Most Python/C API functions have one or more arguments as well as a return value -of type :c:type:`PyObject\*`. This type is a pointer to an opaque data type -representing an arbitrary Python object. Since all Python object types are -treated the same way by the Python language in most situations (e.g., -assignments, scope rules, and argument passing), it is only fitting that they -should be represented by a single C type. Almost all Python objects live on the -heap: you never declare an automatic or static variable of type -:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject\*` can be -declared. The sole exception are the type objects; since these must never be -deallocated, they are typically static :c:type:`PyTypeObject` objects. - -All Python objects (even Python integers) have a :dfn:`type` and a -:dfn:`reference count`. An object's type determines what kind of object it is -(e.g., an integer, a list, or a user-defined function; there are many more as -explained in :ref:`types`). For each of the well-known types there is a macro -to check whether an object is of that type; for instance, ``PyList_Check(a)`` is -true if (and only if) the object pointed to by *a* is a Python list. - - -.. _api-refcounts: - -Reference Counts ----------------- - -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") - -.. index:: - single: Py_INCREF() - single: Py_DECREF() - -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro -is considerably more complex than the incref one, since it must check whether -the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound -object type, such as a list, as well as performing any additional finalization -that's needed. There's no chance that the reference count can overflow; at -least as many bits are used to hold the reference count as there are distinct -memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). -Thus, the reference count increment is a simple operation. - -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's -reference count goes up by one when the variable is made to point to it and it -goes down by one when the variable goes out of scope. However, these two -cancel each other out, so at the end the reference count hasn't changed. The -only real reason to use the reference count is to prevent the object from being -deallocated as long as our variable is pointing to it. If we know that there -is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. -An important situation where this arises is in objects that are passed as -arguments to C functions in an extension module that are called from Python; -the call mechanism guarantees to hold a reference to every argument for the -duration of the call. - -However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count -and possible deallocating it. The real danger is that innocent-looking -operations may invoke arbitrary Python code which could do this; there is a code -path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so -almost any operation is potentially dangerous. - -A safe approach is to always use the generic operations (functions whose name -begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. -This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when -they are done with the result; this soon becomes second nature. - - -.. _api-refcountdetails: - -Reference Count Details -^^^^^^^^^^^^^^^^^^^^^^^ - -The reference count behavior of functions in the Python/C API is best explained -in terms of *ownership of references*. Ownership pertains to references, never -to objects (objects are not owned: they are always shared). "Owning a -reference" means being responsible for calling Py_DECREF on it when the -reference is no longer needed. Ownership can also be transferred, meaning that -the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` -when it's no longer needed---or passing on this responsibility (usually to its -caller). When a function passes ownership of a reference on to its caller, the -caller is said to receive a *new* reference. When no ownership is transferred, -the caller is said to *borrow* the reference. Nothing needs to be done for a -borrowed reference. - -Conversely, when a calling function passes in a reference to an object, there -are two possibilities: the function *steals* a reference to the object, or it -does not. *Stealing a reference* means that when you pass a reference to a -function, that function assumes that it now owns that reference, and you are not -responsible for it any longer. - -.. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() - -Few functions steal references; the two notable exceptions are -:c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference -to the item (but not to the tuple or list into which the item is put!). These -functions were designed to steal a reference because of a common idiom for -populating a tuple or list with newly created objects; for example, the code to -create the tuple ``(1, 2, "three")`` could look like this (forgetting about -error handling for the moment; a better way to code this is shown below):: - - PyObject *t; - - t = PyTuple_New(3); - PyTuple_SetItem(t, 0, PyLong_FromLong(1L)); - PyTuple_SetItem(t, 1, PyLong_FromLong(2L)); - PyTuple_SetItem(t, 2, PyUnicode_FromString("three")); - -Here, :c:func:`PyLong_FromLong` returns a new reference which is immediately -stolen by :c:func:`PyTuple_SetItem`. When you want to keep using an object -although the reference to it will be stolen, use :c:func:`Py_INCREF` to grab -another reference before calling the reference-stealing function. - -Incidentally, :c:func:`PyTuple_SetItem` is the *only* way to set tuple items; -:c:func:`PySequence_SetItem` and :c:func:`PyObject_SetItem` refuse to do this -since tuples are an immutable data type. You should only use -:c:func:`PyTuple_SetItem` for tuples that you are creating yourself. - -Equivalent code for populating a list can be written using :c:func:`PyList_New` -and :c:func:`PyList_SetItem`. - -However, in practice, you will rarely use these ways of creating and populating -a tuple or list. There's a generic function, :c:func:`Py_BuildValue`, that can -create most common objects from C values, directed by a :dfn:`format string`. -For example, the above two blocks of code could be replaced by the following -(which also takes care of the error checking):: - - PyObject *tuple, *list; - - tuple = Py_BuildValue("(iis)", 1, 2, "three"); - list = Py_BuildValue("[iis]", 1, 2, "three"); - -It is much more common to use :c:func:`PyObject_SetItem` and friends with items -whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function -sets all items of a list (actually, any mutable sequence) to a given item:: - - int - set_all(PyObject *target, PyObject *item) - { - Py_ssize_t i, n; - - n = PyObject_Length(target); - if (n < 0) - return -1; - for (i = 0; i < n; i++) { - PyObject *index = PyLong_FromSsize_t(i); - if (!index) - return -1; - if (PyObject_SetItem(target, index, item) < 0) { - Py_DECREF(index); - return -1; - } - Py_DECREF(index); - } - return 0; - } - -.. index:: single: set_all() - -The situation is slightly different for function return values. While passing -a reference to most functions does not change your ownership responsibilities -for that reference, many functions that return a reference to an object give -you ownership of the reference. The reason is simple: in many cases, the -returned object is created on the fly, and the reference you get is the only -reference to the object. Therefore, the generic functions that return object -references, like :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetItem`, -always return a new reference (the caller becomes the owner of the reference). - -It is important to realize that whether you own a reference returned by a -function depends on which function you call only --- *the plumage* (the type of -the object passed as an argument to the function) *doesn't enter into it!* -Thus, if you extract an item from a list using :c:func:`PyList_GetItem`, you -don't own the reference --- but if you obtain the same item from the same list -using :c:func:`PySequence_GetItem` (which happens to take exactly the same -arguments), you do own a reference to the returned object. - -.. index:: - single: PyList_GetItem() - single: PySequence_GetItem() - -Here is an example of how you could write a function that computes the sum of -the items in a list of integers; once using :c:func:`PyList_GetItem`, and once -using :c:func:`PySequence_GetItem`. :: - - long - sum_list(PyObject *list) - { - Py_ssize_t i, n; - long total = 0, value; - PyObject *item; - - n = PyList_Size(list); - if (n < 0) - return -1; /* Not a list */ - for (i = 0; i < n; i++) { - item = PyList_GetItem(list, i); /* Can't fail */ - if (!PyLong_Check(item)) continue; /* Skip non-integers */ - value = PyLong_AsLong(item); - if (value == -1 && PyErr_Occurred()) - /* Integer too big to fit in a C long, bail out */ - return -1; - total += value; - } - return total; - } - -.. index:: single: sum_list() - -:: - - long - sum_sequence(PyObject *sequence) - { - Py_ssize_t i, n; - long total = 0, value; - PyObject *item; - n = PySequence_Length(sequence); - if (n < 0) - return -1; /* Has no length */ - for (i = 0; i < n; i++) { - item = PySequence_GetItem(sequence, i); - if (item == NULL) - return -1; /* Not a sequence, or other failure */ - if (PyLong_Check(item)) { - value = PyLong_AsLong(item); - Py_DECREF(item); - if (value == -1 && PyErr_Occurred()) - /* Integer too big to fit in a C long, bail out */ - return -1; - total += value; - } - else { - Py_DECREF(item); /* Discard reference ownership */ - } - } - return total; - } - -.. index:: single: sum_sequence() - - -.. _api-types: - -Types ------ - -There are few other data types that play a significant role in the Python/C -API; most are simple C types such as :c:type:`int`, :c:type:`long`, -:c:type:`double` and :c:type:`char\*`. A few structure types are used to -describe static tables used to list the functions exported by a module or the -data attributes of a new object type, and another is used to describe the value -of a complex number. These will be discussed together with the functions that -use them. - - -.. _api-exceptions: - -Exceptions -========== - -The Python programmer only needs to deal with exceptions if specific error -handling is required; unhandled exceptions are automatically propagated to the -caller, then to the caller's caller, and so on, until they reach the top-level -interpreter, where they are reported to the user accompanied by a stack -traceback. - -.. index:: single: PyErr_Occurred() - -For C programmers, however, error checking always has to be explicit. All -functions in the Python/C API can raise exceptions, unless an explicit claim is -made otherwise in a function's documentation. In general, when a function -encounters an error, it sets an exception, discards any object references that -it owns, and returns an error indicator. If not documented otherwise, this -indicator is either *NULL* or ``-1``, depending on the function's return type. -A few functions return a Boolean true/false result, with false indicating an -error. Very few functions return no explicit error indicator or have an -ambiguous return value, and require explicit testing for errors with -:c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. - -.. index:: - single: PyErr_SetString() - single: PyErr_Clear() - -Exception state is maintained in per-thread storage (this is equivalent to -using global storage in an unthreaded application). A thread can be in one of -two states: an exception has occurred, or not. The function -:c:func:`PyErr_Occurred` can be used to check for this: it returns a borrowed -reference to the exception type object when an exception has occurred, and -*NULL* otherwise. There are a number of functions to set the exception state: -:c:func:`PyErr_SetString` is the most common (though not the most general) -function to set the exception state, and :c:func:`PyErr_Clear` clears the -exception state. - -The full exception state consists of three objects (all of which can be -*NULL*): the exception type, the corresponding exception value, and the -traceback. These have the same meanings as the Python result of -``sys.exc_info()``; however, they are not the same: the Python objects represent -the last exception being handled by a Python :keyword:`try` ... -:keyword:`except` statement, while the C level exception state only exists while -an exception is being passed on between C functions until it reaches the Python -bytecode interpreter's main loop, which takes care of transferring it to -``sys.exc_info()`` and friends. - -.. index:: single: exc_info() (in module sys) - -Note that starting with Python 1.5, the preferred, thread-safe way to access the -exception state from Python code is to call the function :func:`sys.exc_info`, -which returns the per-thread exception state for Python code. Also, the -semantics of both ways to access the exception state have changed so that a -function which catches an exception will save and restore its thread's exception -state so as to preserve the exception state of its caller. This prevents common -bugs in exception handling code caused by an innocent-looking function -overwriting the exception being handled; it also reduces the often unwanted -lifetime extension for objects that are referenced by the stack frames in the -traceback. - -As a general principle, a function that calls another function to perform some -task should check whether the called function raised an exception, and if so, -pass the exception state on to its caller. It should discard any object -references that it owns, and return an error indicator, but it should *not* set -another exception --- that would overwrite the exception that was just raised, -and lose important information about the exact cause of the error. - -.. index:: single: sum_sequence() - -A simple example of detecting exceptions and passing them on is shown in the -:c:func:`sum_sequence` example above. It so happens that this example doesn't -need to clean up any owned references when it detects an error. The following -example function shows some error cleanup. First, to remind you why you like -Python, we show the equivalent Python code:: - - def incr_item(dict, key): - try: - item = dict[key] - except KeyError: - item = 0 - dict[key] = item + 1 - -.. index:: single: incr_item() - -Here is the corresponding C code, in all its glory:: - - int - incr_item(PyObject *dict, PyObject *key) - { - /* Objects all initialized to NULL for Py_XDECREF */ - PyObject *item = NULL, *const_one = NULL, *incremented_item = NULL; - int rv = -1; /* Return value initialized to -1 (failure) */ - - item = PyObject_GetItem(dict, key); - if (item == NULL) { - /* Handle KeyError only: */ - if (!PyErr_ExceptionMatches(PyExc_KeyError)) - goto error; - - /* Clear the error and use zero: */ - PyErr_Clear(); - item = PyLong_FromLong(0L); - if (item == NULL) - goto error; - } - const_one = PyLong_FromLong(1L); - if (const_one == NULL) - goto error; - - incremented_item = PyNumber_Add(item, const_one); - if (incremented_item == NULL) - goto error; - - if (PyObject_SetItem(dict, key, incremented_item) < 0) - goto error; - rv = 0; /* Success */ - /* Continue with cleanup code */ - - error: - /* Cleanup code, shared by success and failure path */ - - /* Use Py_XDECREF() to ignore NULL references */ - Py_XDECREF(item); - Py_XDECREF(const_one); - Py_XDECREF(incremented_item); - - return rv; /* -1 for error, 0 for success */ - } - -.. index:: single: incr_item() - -.. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() - -This example represents an endorsed use of the ``goto`` statement in C! -It illustrates the use of :c:func:`PyErr_ExceptionMatches` and -:c:func:`PyErr_Clear` to handle specific exceptions, and the use of -:c:func:`Py_XDECREF` to dispose of owned references that may be *NULL* (note the -``'X'`` in the name; :c:func:`Py_DECREF` would crash when confronted with a -*NULL* reference). It is important that the variables used to hold owned -references are initialized to *NULL* for this to work; likewise, the proposed -return value is initialized to ``-1`` (failure) and only set to success after -the final call made is successful. - - -.. _api-embedding: - -Embedding Python -================ - -The one important task that only embedders (as opposed to extension writers) of -the Python interpreter have to worry about is the initialization, and possibly -the finalization, of the Python interpreter. Most functionality of the -interpreter can only be used after the interpreter has been initialized. - -.. index:: - single: Py_Initialize() - module: builtins - module: __main__ - module: sys - triple: module; search; path - single: path (in module sys) - -The basic initialization function is :c:func:`Py_Initialize`. This initializes -the table of loaded modules, and creates the fundamental modules -:mod:`builtins`, :mod:`__main__`, and :mod:`sys`. It also -initializes the module search path (``sys.path``). - -.. index:: single: PySys_SetArgvEx() - -:c:func:`Py_Initialize` does not set the "script argument list" (``sys.argv``). -If this variable is needed by Python code that will be executed later, it must -be set explicitly with a call to ``PySys_SetArgvEx(argc, argv, updatepath)`` -after the call to :c:func:`Py_Initialize`. - -On most systems (in particular, on Unix and Windows, although the details are -slightly different), :c:func:`Py_Initialize` calculates the module search path -based upon its best guess for the location of the standard Python interpreter -executable, assuming that the Python library is found in a fixed location -relative to the Python interpreter executable. In particular, it looks for a -directory named :file:`lib/python{X.Y}` relative to the parent directory -where the executable named :file:`python` is found on the shell command search -path (the environment variable :envvar:`PATH`). - -For instance, if the Python executable is found in -:file:`/usr/local/bin/python`, it will assume that the libraries are in -:file:`/usr/local/lib/python{X.Y}`. (In fact, this particular path is also -the "fallback" location, used when no executable file named :file:`python` is -found along :envvar:`PATH`.) The user can override this behavior by setting the -environment variable :envvar:`PYTHONHOME`, or insert additional directories in -front of the standard path by setting :envvar:`PYTHONPATH`. - -.. index:: - single: Py_SetProgramName() - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() - -The embedding application can steer the search by calling -``Py_SetProgramName(file)`` *before* calling :c:func:`Py_Initialize`. Note that -:envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still -inserted in front of the standard path. An application that requires total -control has to provide its own implementation of :c:func:`Py_GetPath`, -:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and -:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). - -.. index:: single: Py_IsInitialized() - -Sometimes, it is desirable to "uninitialize" Python. For instance, the -application may want to start over (make another call to -:c:func:`Py_Initialize`) or the application is simply done with its use of -Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns -true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` -does *not* free all memory allocated by the Python interpreter, e.g. memory -allocated by extension modules currently cannot be released. - - -.. _api-debugging: - -Debugging Builds -================ - -Python can be built with several macros to enable extra checks of the -interpreter and extension modules. These checks tend to add a large amount of -overhead to the runtime so they are not enabled by default. - -A full list of the various types of debugging builds is in the file -:file:`Misc/SpecialBuilds.txt` in the Python source distribution. Builds are -available that support tracing of reference counts, debugging the memory -allocator, or low-level profiling of the main interpreter loop. Only the most -frequently-used builds will be described in the remainder of this section. - -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces -what is generally meant by "a debug build" of Python. :c:macro:`Py_DEBUG` is -enabled in the Unix build by adding ``--with-pydebug`` to the -:file:`./configure` command. It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled -in the Unix build, compiler optimization is disabled. - -In addition to the reference count debugging described below, the following -extra checks are performed: - -* Extra checks are added to the object allocator. - -* Extra checks are added to the parser and compiler. - -* Downcasts from wide types to narrow types are checked for loss of information. - -* A number of assertions are added to the dictionary and set implementations. - In addition, the set object acquires a :meth:`test_c_api` method. - -* Sanity checks of the input arguments are added to frame creation. - -* The storage for ints is initialized with a known invalid pattern to catch - reference to uninitialized digits. - -* Low-level tracing and extra exception checking are added to the runtime - virtual machine. - -* Extra checks are added to the memory arena implementation. - -* Extra debugging is added to the thread module. - -There may be additional checks not mentioned here. - -Defining :c:macro:`Py_TRACE_REFS` enables reference tracing. When defined, a -circular doubly linked list of active objects is maintained by adding two extra -fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon -exit, all existing references are printed. (In interactive mode this happens -after every statement run by the interpreter.) Implied by :c:macro:`Py_DEBUG`. - -Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution -for more detailed information. - diff --git a/third_party/python/Doc/c-api/iter.rst b/third_party/python/Doc/c-api/iter.rst deleted file mode 100644 index 2ba444d1d..000000000 --- a/third_party/python/Doc/c-api/iter.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. highlightlang:: c - -.. _iterator: - -Iterator Protocol -================= - -There are two functions specifically for working with iterators. - -.. c:function:: int PyIter_Check(PyObject *o) - - Return true if the object *o* supports the iterator protocol. - - -.. c:function:: PyObject* PyIter_Next(PyObject *o) - - Return the next value from the iteration *o*. The object must be an iterator - (it is up to the caller to check this). If there are no remaining values, - returns *NULL* with no exception set. If an error occurs while retrieving - the item, returns *NULL* and passes along the exception. - -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while (item = PyIter_Next(iterator)) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } diff --git a/third_party/python/Doc/c-api/iterator.rst b/third_party/python/Doc/c-api/iterator.rst deleted file mode 100644 index 82cb4eba8..000000000 --- a/third_party/python/Doc/c-api/iterator.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. highlightlang:: c - -.. _iterator-objects: - -Iterator Objects ----------------- - -Python provides two general-purpose iterator objects. The first, a sequence -iterator, works with an arbitrary sequence supporting the :meth:`__getitem__` -method. The second works with a callable object and a sentinel value, calling -the callable for each item in the sequence, and ending the iteration when the -sentinel value is returned. - - -.. c:var:: PyTypeObject PySeqIter_Type - - Type object for iterator objects returned by :c:func:`PySeqIter_New` and the - one-argument form of the :func:`iter` built-in function for built-in sequence - types. - - -.. c:function:: int PySeqIter_Check(op) - - Return true if the type of *op* is :c:data:`PySeqIter_Type`. - - -.. c:function:: PyObject* PySeqIter_New(PyObject *seq) - - Return an iterator that works with a general sequence object, *seq*. The - iteration ends when the sequence raises :exc:`IndexError` for the subscripting - operation. - - -.. c:var:: PyTypeObject PyCallIter_Type - - Type object for iterator objects returned by :c:func:`PyCallIter_New` and the - two-argument form of the :func:`iter` built-in function. - - -.. c:function:: int PyCallIter_Check(op) - - Return true if the type of *op* is :c:data:`PyCallIter_Type`. - - -.. c:function:: PyObject* PyCallIter_New(PyObject *callable, PyObject *sentinel) - - Return a new iterator. The first parameter, *callable*, can be any Python - callable object that can be called with no parameters; each call to it should - return the next item in the iteration. When *callable* returns a value equal to - *sentinel*, the iteration will be terminated. diff --git a/third_party/python/Doc/c-api/list.rst b/third_party/python/Doc/c-api/list.rst deleted file mode 100644 index 5b263a7b1..000000000 --- a/third_party/python/Doc/c-api/list.rst +++ /dev/null @@ -1,151 +0,0 @@ -.. highlightlang:: c - -.. _listobjects: - -List Objects ------------- - -.. index:: object: list - - -.. c:type:: PyListObject - - This subtype of :c:type:`PyObject` represents a Python list object. - - -.. c:var:: PyTypeObject PyList_Type - - This instance of :c:type:`PyTypeObject` represents the Python list type. - This is the same object as :class:`list` in the Python layer. - - -.. c:function:: int PyList_Check(PyObject *p) - - Return true if *p* is a list object or an instance of a subtype of the list - type. - - -.. c:function:: int PyList_CheckExact(PyObject *p) - - Return true if *p* is a list object, but not an instance of a subtype of - the list type. - - -.. c:function:: PyObject* PyList_New(Py_ssize_t len) - - Return a new list of length *len* on success, or *NULL* on failure. - - .. note:: - - If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. - - -.. c:function:: Py_ssize_t PyList_Size(PyObject *list) - - .. index:: builtin: len - - Return the length of the list object in *list*; this is equivalent to - ``len(list)`` on a list object. - - -.. c:function:: Py_ssize_t PyList_GET_SIZE(PyObject *list) - - Macro form of :c:func:`PyList_Size` without error checking. - - -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) - - Return the object at position *index* in the list pointed to by *list*. The - position must be positive, indexing from the end of the list is not - supported. If *index* is out of bounds, return *NULL* and set an - :exc:`IndexError` exception. - - -.. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) - - Macro form of :c:func:`PyList_GetItem` without error checking. - - -.. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) - - Set the item at index *index* in list to *item*. Return ``0`` on success - or ``-1`` on failure. - - .. note:: - - This function "steals" a reference to *item* and discards a reference to - an item already in the list at the affected position. - - -.. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) - - Macro form of :c:func:`PyList_SetItem` without error checking. This is - normally only used to fill in new lists where there is no previous content. - - .. note:: - - This macro "steals" a reference to *item*, and, unlike - :c:func:`PyList_SetItem`, does *not* discard a reference to any item that - is being replaced; any reference in *list* at position *i* will be - leaked. - - -.. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) - - Insert the item *item* into list *list* in front of index *index*. Return - ``0`` if successful; return ``-1`` and set an exception if unsuccessful. - Analogous to ``list.insert(index, item)``. - - -.. c:function:: int PyList_Append(PyObject *list, PyObject *item) - - Append the object *item* at the end of list *list*. Return ``0`` if - successful; return ``-1`` and set an exception if unsuccessful. Analogous - to ``list.append(item)``. - - -.. c:function:: PyObject* PyList_GetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high) - - Return a list of the objects in *list* containing the objects *between* *low* - and *high*. Return *NULL* and set an exception if unsuccessful. Analogous - to ``list[low:high]``. Negative indices, as when slicing from Python, are not - supported. - - -.. c:function:: int PyList_SetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high, PyObject *itemlist) - - Set the slice of *list* between *low* and *high* to the contents of - *itemlist*. Analogous to ``list[low:high] = itemlist``. The *itemlist* may - be *NULL*, indicating the assignment of an empty list (slice deletion). - Return ``0`` on success, ``-1`` on failure. Negative indices, as when - slicing from Python, are not supported. - - -.. c:function:: int PyList_Sort(PyObject *list) - - Sort the items of *list* in place. Return ``0`` on success, ``-1`` on - failure. This is equivalent to ``list.sort()``. - - -.. c:function:: int PyList_Reverse(PyObject *list) - - Reverse the items of *list* in place. Return ``0`` on success, ``-1`` on - failure. This is the equivalent of ``list.reverse()``. - - -.. c:function:: PyObject* PyList_AsTuple(PyObject *list) - - .. index:: builtin: tuple - - Return a new tuple object containing the contents of *list*; equivalent to - ``tuple(list)``. - - -.. c:function:: int PyList_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/third_party/python/Doc/c-api/long.rst b/third_party/python/Doc/c-api/long.rst deleted file mode 100644 index 5b1f386fb..000000000 --- a/third_party/python/Doc/c-api/long.rst +++ /dev/null @@ -1,295 +0,0 @@ -.. highlightlang:: c - -.. _longobjects: - -Integer Objects ---------------- - -.. index:: object: long integer - object: integer - -All integers are implemented as "long" integer objects of arbitrary size. - -On error, most ``PyLong_As*`` APIs return ``(return type)-1`` which cannot be -distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. - -.. c:type:: PyLongObject - - This subtype of :c:type:`PyObject` represents a Python integer object. - - -.. c:var:: PyTypeObject PyLong_Type - - This instance of :c:type:`PyTypeObject` represents the Python integer type. - This is the same object as :class:`int` in the Python layer. - - -.. c:function:: int PyLong_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyLongObject` or a subtype of - :c:type:`PyLongObject`. - - -.. c:function:: int PyLong_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyLongObject`, but not a subtype of - :c:type:`PyLongObject`. - - -.. c:function:: PyObject* PyLong_FromLong(long v) - - Return a new :c:type:`PyLongObject` object from *v*, or *NULL* on failure. - - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``, when you create an int in that range you actually - just get back a reference to the existing object. So it should be possible to - change the value of ``1``. I suspect the behaviour of Python in this case is - undefined. :-) - - -.. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long`, or - *NULL* on failure. - - -.. c:function:: PyObject* PyLong_FromSsize_t(Py_ssize_t v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`Py_ssize_t`, or - *NULL* on failure. - - -.. c:function:: PyObject* PyLong_FromSize_t(size_t v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`size_t`, or - *NULL* on failure. - - -.. c:function:: PyObject* PyLong_FromLongLong(long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`long long`, or *NULL* - on failure. - - -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`unsigned long long`, - or *NULL* on failure. - - -.. c:function:: PyObject* PyLong_FromDouble(double v) - - Return a new :c:type:`PyLongObject` object from the integer part of *v*, or - *NULL* on failure. - - -.. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) - - Return a new :c:type:`PyLongObject` based on the string value in *str*, which - is interpreted according to the radix in *base*. If *pend* is non-*NULL*, - *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, *str* is interpreted using - the :ref:`integers` definition; in this case, leading zeros in a - non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, - it must be between ``2`` and ``36``, inclusive. Leading spaces and single - underscores after a base specifier and between digits are ignored. If there - are no digits, :exc:`ValueError` will be raised. - - -.. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) - - Convert a sequence of Unicode digits to a Python integer value. The Unicode - string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` - and then converted using :c:func:`PyLong_FromString`. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyLong_FromUnicodeObject`. - - -.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) - - Convert a sequence of Unicode digits in the string *u* to a Python integer - value. The Unicode string is first encoded to a byte string using - :c:func:`PyUnicode_EncodeDecimal` and then converted using - :c:func:`PyLong_FromString`. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyLong_FromVoidPtr(void *p) - - Create a Python integer from the pointer *p*. The pointer value can be - retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. - - -.. XXX alias PyLong_AS_LONG (for now) -.. c:function:: long PyLong_AsLong(PyObject *obj) - - .. index:: - single: LONG_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) - - Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is greater than :const:`LONG_MAX` or less than - :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and - return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception - occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: long long PyLong_AsLongLong(PyObject *obj) - - .. index:: - single: OverflowError (built-in exception) - - Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) - - Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is greater than :const:`PY_LLONG_MAX` or less than - :const:`PY_LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, - and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other - exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionadded:: 3.2 - - -.. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) - - .. index:: - single: PY_SSIZE_T_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must - be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`Py_ssize_t`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) - - .. index:: - single: ULONG_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`unsigned long` representation of *pylong*. *pylong* - must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`unsigned long`. - - Returns ``(unsigned long)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) - - .. index:: - single: SIZE_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`size_t`. - - Returns ``(size_t)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: unsigned long long PyLong_AsUnsignedLongLong(PyObject *pylong) - - .. index:: - single: OverflowError (built-in exception) - - Return a C :c:type:`unsigned long long` representation of *pylong*. *pylong* - must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for an - :c:type:`unsigned long long`. - - Returns ``(unsigned long long)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionchanged:: 3.1 - A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`. - - -.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) - - Return a C :c:type:`unsigned long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its :meth:`__int__` - method (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is out of range for an :c:type:`unsigned long`, - return the reduction of that value modulo ``ULONG_MAX + 1``. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) - - Return a C :c:type:`unsigned long long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its :meth:`__int__` - method (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is out of range for an :c:type:`unsigned long long`, - return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: double PyLong_AsDouble(PyObject *pylong) - - Return a C :c:type:`double` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`double`. - - Returns ``-1.0`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) - - Convert a Python integer *pylong* to a C :c:type:`void` pointer. - If *pylong* cannot be converted, an :exc:`OverflowError` will be raised. This - is only assured to produce a usable :c:type:`void` pointer for values created - with :c:func:`PyLong_FromVoidPtr`. - - Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/third_party/python/Doc/c-api/mapping.rst b/third_party/python/Doc/c-api/mapping.rst deleted file mode 100644 index cb68ef82c..000000000 --- a/third_party/python/Doc/c-api/mapping.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. highlightlang:: c - -.. _mapping: - -Mapping Protocol -================ - -See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and -:c:func:`PyObject_DelItem`. - - -.. c:function:: int PyMapping_Check(PyObject *o) - - Return ``1`` if the object provides mapping protocol or supports slicing, - and ``0`` otherwise. Note that it returns ``1`` for Python classes with - a :meth:`__getitem__` method since in general case it is impossible to - determine what the type of keys it supports. This function always - succeeds. - - -.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) - Py_ssize_t PyMapping_Length(PyObject *o) - - .. index:: builtin: len - - Returns the number of keys in object *o* on success, and ``-1`` on failure. - This is equivalent to the Python expression ``len(o)``. - - -.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - - Return element of *o* corresponding to the string *key* or *NULL* on failure. - This is the equivalent of the Python expression ``o[key]``. - See also :c:func:`PyObject_GetItem`. - - -.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) - - Map the string *key* to the value *v* in object *o*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. - - -.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) - - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. - This is an alias of :c:func:`PyObject_DelItem`. - - -.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) - - Remove the mapping for the string *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. - - -.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) - - Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. - This is equivalent to the Python expression ``key in o``. - This function always succeeds. - - Note that exceptions which occur while calling the :meth:`__getitem__` - method will get suppressed. - To get error reporting use :c:func:`PyObject_GetItem()` instead. - - -.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) - - Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. - This is equivalent to the Python expression ``key in o``. - This function always succeeds. - - Note that exceptions which occur while calling the :meth:`__getitem__` - method and creating a temporary string object will get suppressed. - To get error reporting use :c:func:`PyMapping_GetItemString()` instead. - - -.. c:function:: PyObject* PyMapping_Keys(PyObject *o) - - On success, return a list or tuple of the keys in object *o*. On failure, - return *NULL*. - - -.. c:function:: PyObject* PyMapping_Values(PyObject *o) - - On success, return a list or tuple of the values in object *o*. On failure, - return *NULL*. - - -.. c:function:: PyObject* PyMapping_Items(PyObject *o) - - On success, return a list or tuple of the items in object *o*, where each item - is a tuple containing a key-value pair. On failure, return *NULL*. diff --git a/third_party/python/Doc/c-api/marshal.rst b/third_party/python/Doc/c-api/marshal.rst deleted file mode 100644 index 17ec62161..000000000 --- a/third_party/python/Doc/c-api/marshal.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. highlightlang:: c - -.. _marshalling-utils: - -Data marshalling support -======================== - -These routines allow C code to work with serialized objects using the same -data format as the :mod:`marshal` module. There are functions to write data -into the serialization format, and additional functions that can be used to -read the data back. Files used to store marshalled data must be opened in -binary mode. - -Numeric values are stored with the least significant byte first. - -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. -*Py_MARSHAL_VERSION* indicates the current file format (currently 2). - - -.. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) - - Marshal a :c:type:`long` integer, *value*, to *file*. This will only write - the least-significant 32 bits of *value*; regardless of the size of the - native :c:type:`long` type. *version* indicates the file format. - - -.. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) - - Marshal a Python object, *value*, to *file*. - *version* indicates the file format. - - -.. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - - Return a bytes object containing the marshalled representation of *value*. - *version* indicates the file format. - - -The following functions allow marshalled values to be read back in. - - -.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - - Return a C :c:type:`long` from the data stream in a :c:type:`FILE\*` opened - for reading. Only a 32-bit value can be read in using this function, - regardless of the native size of :c:type:`long`. - - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. - - -.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - - Return a C :c:type:`short` from the data stream in a :c:type:`FILE\*` opened - for reading. Only a 16-bit value can be read in using this function, - regardless of the native size of :c:type:`short`. - - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. - - -.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) - - Return a Python object from the data stream in a :c:type:`FILE\*` opened for - reading. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. - - -.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) - - Return a Python object from the data stream in a :c:type:`FILE\*` opened for - reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function - assumes that no further objects will be read from the file, allowing it to - aggressively load file data into memory so that the de-serialization can - operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading - anything else from the file. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. - - -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - - Return a Python object from the data stream in a byte buffer - containing *len* bytes pointed to by *data*. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns *NULL*. - diff --git a/third_party/python/Doc/c-api/memory.rst b/third_party/python/Doc/c-api/memory.rst deleted file mode 100644 index b502b25f7..000000000 --- a/third_party/python/Doc/c-api/memory.rst +++ /dev/null @@ -1,546 +0,0 @@ -.. highlightlang:: c - - -.. _memory: - -***************** -Memory Management -***************** - -.. sectionauthor:: Vladimir Marangozov - - - -.. _memoryoverview: - -Overview -======== - -Memory management in Python involves a private heap containing all Python -objects and data structures. The management of this private heap is ensured -internally by the *Python memory manager*. The Python memory manager has -different components which deal with various dynamic storage management aspects, -like sharing, segmentation, preallocation or caching. - -At the lowest level, a raw memory allocator ensures that there is enough room in -the private heap for storing all Python-related data by interacting with the -memory manager of the operating system. On top of the raw memory allocator, -several object-specific allocators operate on the same heap and implement -distinct memory management policies adapted to the peculiarities of every object -type. For example, integer objects are managed differently within the heap than -strings, tuples or dictionaries because integers imply different storage -requirements and speed/space tradeoffs. The Python memory manager thus delegates -some of the work to the object-specific allocators, but ensures that the latter -operate within the bounds of the private heap. - -It is important to understand that the management of the Python heap is -performed by the interpreter itself and that the user has no control over it, -even if they regularly manipulate object pointers to memory blocks inside that -heap. The allocation of heap space for Python objects and other internal -buffers is performed on demand by the Python memory manager through the Python/C -API functions listed in this document. - -.. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() - -To avoid memory corruption, extension writers should never try to operate on -Python objects with the functions exported by the C library: :c:func:`malloc`, -:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed -calls between the C allocator and the Python memory manager with fatal -consequences, because they implement different algorithms and operate on -different heaps. However, one may safely allocate and release memory blocks -with the C library allocator for individual purposes, as shown in the following -example:: - - PyObject *res; - char *buf = (char *) malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); - free(buf); /* malloc'ed */ - return res; - -In this example, the memory request for the I/O buffer is handled by the C -library allocator. The Python memory manager is involved only in the allocation -of the string object returned as a result. - -In most situations, however, it is recommended to allocate memory from the -Python heap specifically because the latter is under control of the Python -memory manager. For example, this is required when the interpreter is extended -with new object types written in C. Another reason for using the Python heap is -the desire to *inform* the Python memory manager about the memory needs of the -extension module. Even when the requested memory is used exclusively for -internal, highly-specific purposes, delegating all memory requests to the Python -memory manager causes the interpreter to have a more accurate image of its -memory footprint as a whole. Consequently, under certain circumstances, the -Python memory manager may or may not trigger appropriate actions, like garbage -collection, memory compaction or other preventive procedures. Note that by using -the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. - -.. seealso:: - - The :envvar:`PYTHONMALLOC` environment variable can be used to configure - the memory allocators used by Python. - - The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - statistics of the :ref:`pymalloc memory allocator ` every time a - new pymalloc object arena is created, and on shutdown. - - -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The default raw memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. - - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as - if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been - called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. - - If *p* is *NULL*, the call is equivalent to ``PyMem_RawMalloc(n)``; else if - *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. - - Unless *p* is *NULL*, it must have been returned by a previous call to - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. - - If the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* - remains a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been - called before, undefined behavior occurs. - - If *p* is *NULL*, no operation is performed. - - -.. _memoryinterface: - -Memory Interface -================ - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -By default, these functions use :ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. versionchanged:: 3.6 - - The default allocator is now pymalloc instead of system :c:func:`malloc`. - -.. c:function:: void* PyMem_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. - - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as - if ``PyMem_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is *NULL*, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. - - Unless *p* is *NULL*, it must have been returned by a previous call to - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - - If the request fails, :c:func:`PyMem_Realloc` returns *NULL* and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or - :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is *NULL*, no operation is performed. - -The following type-oriented macros are provided for convenience. Note that -*TYPE* refers to any C type. - - -.. c:function:: TYPE* PyMem_New(TYPE, size_t n) - - Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE\*`. The memory will not have - been initialized in any way. - - -.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) - - Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE\*`. On return, - *p* will be a pointer to the new memory area, or *NULL* in the event of - failure. - - This is a C preprocessor macro; *p* is always reassigned. Save the original - value of *p* to avoid losing memory when handling errors. - - -.. c:function:: void PyMem_Del(void *p) - - Same as :c:func:`PyMem_Free`. - -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. - -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` - - -Object allocators -================= - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -By default, these functions use :ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. c:function:: void* PyObject_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. - - Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as - if ``PyObject_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-*NULL* pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyObject_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is *NULL*, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-*NULL*. - - Unless *p* is *NULL*, it must have been returned by a previous call to - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - - If the request fails, :c:func:`PyObject_Realloc` returns *NULL* and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyObject_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or - :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is *NULL*, no operation is performed. - - -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - four fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - .. c:var:: PYMEM_DOMAIN_RAW - - Functions: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - - .. c:var:: PYMEM_DOMAIN_MEM - - Functions: - - * :c:func:`PyMem_Malloc`, - * :c:func:`PyMem_Realloc` - * :c:func:`PyMem_Calloc` - * :c:func:`PyMem_Free` - - .. c:var:: PYMEM_DOMAIN_OBJ - - Functions: - - * :c:func:`PyObject_Malloc` - * :c:func:`PyObject_Realloc` - * :c:func:`PyObject_Calloc` - * :c:func:`PyObject_Free` - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-NULL pointer when requesting - zero bytes. - - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup hooks to detect bugs in the Python memory allocator functions. - - Newly allocated memory is filled with the byte ``0xCB``, freed memory is - filled with the byte ``0xDB``. - - Runtime checks: - - - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - Detect write before the start of the buffer (buffer underflow) - - Detect write after the end of the buffer (buffer overflow) - - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called - - On error, the debug hooks use the :mod:`tracemalloc` module to get the - traceback where a memory block was allocated. The traceback is only - displayed if :mod:`tracemalloc` is tracing Python memory allocations and the - memory block was traced. - - These hooks are installed by default if Python is compiled in debug - mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install - debug hooks on a Python compiled in release mode. - - .. versionchanged:: 3.6 - This function now also works on Python compiled in release mode. - On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. The debug hooks now also check - if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and - :c:data:`PYMEM_DOMAIN_MEM` domains are called. - - -.. _pymalloc: - -The pymalloc allocator -====================== - -Python has a *pymalloc* allocator optimized for small objects (smaller or equal -to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and -:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. - -*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: -:c:func:`PyObject_Malloc`) domains. - -The arena allocator uses the following functions: - -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -Customize pymalloc Arena Allocator ----------------------------------- - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - -.. _memoryexamples: - -Examples -======== - -Here is the example from section :ref:`memoryoverview`, rewritten so that the -I/O buffer is allocated from the Python heap by using the first function set:: - - PyObject *res; - char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Free(buf); /* allocated with PyMem_Malloc */ - return res; - -The same code using the type-oriented function set:: - - PyObject *res; - char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ - return res; - -Note that in the two examples above, the buffer is always manipulated via -functions belonging to the same set. Indeed, it is required to use the same -memory API family for a given memory block, so that the risk of mixing different -allocators is reduced to a minimum. The following code sequence contains two -errors, one of which is labeled as *fatal* because it mixes two different -allocators operating on different heaps. :: - - char *buf1 = PyMem_New(char, BUFSIZ); - char *buf2 = (char *) malloc(BUFSIZ); - char *buf3 = (char *) PyMem_Malloc(BUFSIZ); - ... - PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ - free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ - -In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:func:`PyObject_New`, -:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. - -These will be explained in the next chapter on defining and implementing new -object types in C. - diff --git a/third_party/python/Doc/c-api/memoryview.rst b/third_party/python/Doc/c-api/memoryview.rst deleted file mode 100644 index 9f6bfd751..000000000 --- a/third_party/python/Doc/c-api/memoryview.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. highlightlang:: c - -.. _memoryview-objects: - -.. index:: - object: memoryview - -MemoryView objects ------------------- - -A :class:`memoryview` object exposes the C level :ref:`buffer interface -` as a Python object which can then be passed around like -any other object. - - -.. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) - - Create a memoryview object from an object that provides the buffer interface. - If *obj* supports writable buffer exports, the memoryview object will be - read/write, otherwise it may be either read-only or read/write at the - discretion of the exporter. - -.. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) - - Create a memoryview object using *mem* as the underlying buffer. - *flags* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. - - .. versionadded:: 3.3 - -.. c:function:: PyObject *PyMemoryView_FromBuffer(Py_buffer *view) - - Create a memoryview object wrapping the given buffer structure *view*. - For simple byte buffers, :c:func:`PyMemoryView_FromMemory` is the preferred - function. - -.. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) - - Create a memoryview object to a :term:`contiguous` chunk of memory (in either - 'C' or 'F'ortran *order*) from an object that defines the buffer - interface. If memory is contiguous, the memoryview object points to the - original memory. Otherwise, a copy is made and the memoryview points to a - new bytes object. - - -.. c:function:: int PyMemoryView_Check(PyObject *obj) - - Return true if the object *obj* is a memoryview object. It is not - currently allowed to create subclasses of :class:`memoryview`. - - -.. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *mview) - - Return a pointer to the memoryview's private copy of the exporter's buffer. - *mview* **must** be a memoryview instance; this macro doesn't check its type, - you must do it yourself or you will risk crashes. - -.. c:function:: Py_buffer *PyMemoryView_GET_BASE(PyObject *mview) - - Return either a pointer to the exporting object that the memoryview is based - on or *NULL* if the memoryview has been created by one of the functions - :c:func:`PyMemoryView_FromMemory` or :c:func:`PyMemoryView_FromBuffer`. - *mview* **must** be a memoryview instance. - diff --git a/third_party/python/Doc/c-api/method.rst b/third_party/python/Doc/c-api/method.rst deleted file mode 100644 index 7a2a84fe1..000000000 --- a/third_party/python/Doc/c-api/method.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. highlightlang:: c - -.. _instancemethod-objects: - -Instance Method Objects ------------------------ - -.. index:: object: instancemethod - -An instance method is a wrapper for a :c:data:`PyCFunction` and the new way -to bind a :c:data:`PyCFunction` to a class object. It replaces the former call -``PyMethod_New(func, NULL, class)``. - - -.. c:var:: PyTypeObject PyInstanceMethod_Type - - This instance of :c:type:`PyTypeObject` represents the Python instance - method type. It is not exposed to Python programs. - - -.. c:function:: int PyInstanceMethod_Check(PyObject *o) - - Return true if *o* is an instance method object (has type - :c:data:`PyInstanceMethod_Type`). The parameter must not be *NULL*. - - -.. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) - - Return a new instance method object, with *func* being any callable object - *func* is the function that will be called when the instance method is - called. - - -.. c:function:: PyObject* PyInstanceMethod_Function(PyObject *im) - - Return the function object associated with the instance method *im*. - - -.. c:function:: PyObject* PyInstanceMethod_GET_FUNCTION(PyObject *im) - - Macro version of :c:func:`PyInstanceMethod_Function` which avoids error checking. - - -.. _method-objects: - -Method Objects --------------- - -.. index:: object: method - -Methods are bound function objects. Methods are always bound to an instance of -a user-defined class. Unbound methods (methods bound to a class object) are -no longer available. - - -.. c:var:: PyTypeObject PyMethod_Type - - .. index:: single: MethodType (in module types) - - This instance of :c:type:`PyTypeObject` represents the Python method type. This - is exposed to Python programs as ``types.MethodType``. - - -.. c:function:: int PyMethod_Check(PyObject *o) - - Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be *NULL*. - - -.. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) - - Return a new method object, with *func* being any callable object and *self* - the instance the method should be bound. *func* is the function that will - be called when the method is called. *self* must not be *NULL*. - - -.. c:function:: PyObject* PyMethod_Function(PyObject *meth) - - Return the function object associated with the method *meth*. - - -.. c:function:: PyObject* PyMethod_GET_FUNCTION(PyObject *meth) - - Macro version of :c:func:`PyMethod_Function` which avoids error checking. - - -.. c:function:: PyObject* PyMethod_Self(PyObject *meth) - - Return the instance associated with the method *meth*. - - -.. c:function:: PyObject* PyMethod_GET_SELF(PyObject *meth) - - Macro version of :c:func:`PyMethod_Self` which avoids error checking. - - -.. c:function:: int PyMethod_ClearFreeList() - - Clear the free list. Return the total number of freed items. - diff --git a/third_party/python/Doc/c-api/module.rst b/third_party/python/Doc/c-api/module.rst deleted file mode 100644 index 797a67e49..000000000 --- a/third_party/python/Doc/c-api/module.rst +++ /dev/null @@ -1,479 +0,0 @@ -.. highlightlang:: c - -.. _moduleobjects: - -Module Objects --------------- - -.. index:: object: module - - -.. c:var:: PyTypeObject PyModule_Type - - .. index:: single: ModuleType (in module types) - - This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. - - -.. c:function:: int PyModule_Check(PyObject *p) - - Return true if *p* is a module object, or a subtype of a module object. - - -.. c:function:: int PyModule_CheckExact(PyObject *p) - - Return true if *p* is a module object, but not a subtype of - :c:data:`PyModule_Type`. - - -.. c:function:: PyObject* PyModule_NewObject(PyObject *name) - - .. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __package__ (module attribute) - single: __loader__ (module attribute) - - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. - - -.. c:function:: PyObject* PyModule_New(const char *name) - - Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded - string instead of a Unicode object. - - -.. c:function:: PyObject* PyModule_GetDict(PyObject *module) - - .. index:: single: __dict__ (module attribute) - - Return the dictionary object that implements *module*'s namespace; this object - is the same as the :attr:`~object.__dict__` attribute of the module object. - If *module* is not a module object (or a subtype of a module object), - :exc:`SystemError` is raised and *NULL* is returned. - - It is recommended extensions use other :c:func:`PyModule_\*` and - :c:func:`PyObject_\*` functions rather than directly manipulate a module's - :attr:`~object.__dict__`. - - -.. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) - - .. index:: - single: __name__ (module attribute) - single: SystemError (built-in exception) - - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and *NULL* is returned. - - .. versionadded:: 3.3 - - -.. c:function:: char* PyModule_GetName(PyObject *module) - - Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to - ``'utf-8'``. - -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or *NULL*. See - :c:member:`PyModuleDef.m_size`. - - -.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) - - Return a pointer to the :c:type:`PyModuleDef` struct from which the module was - created, or *NULL* if the module wasn't created from a definition. - - -.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) - - .. index:: - single: __file__ (module attribute) - single: SystemError (built-in exception) - - Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return *NULL*; otherwise return - a reference to a Unicode object. - - .. versionadded:: 3.2 - - -.. c:function:: char* PyModule_GetFilename(PyObject *module) - - Similar to :c:func:`PyModule_GetFilenameObject` but return the filename - encoded to 'utf-8'. - - .. deprecated:: 3.2 - :c:func:`PyModule_GetFilename` raises :c:type:`UnicodeEncodeError` on - unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. - - -.. _initializing-modules: - -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ - -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. - -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. - -.. c:type:: PyModuleDef - - The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. - - .. c:member:: PyModuleDef_Base m_base - - Always initialize this member to :const:`PyModuleDef_HEAD_INIT`. - - .. c:member:: char* m_name - - Name for the new module. - - .. c:member:: char* m_doc - - Docstring for the module; usually a docstring variable created with - :c:func:`PyDoc_STRVAR` is used. - - .. c:member:: Py_ssize_t m_size - - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. - - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`m_free` function has been called, if present. - - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. - - See :PEP:`3121` for more details. - - .. c:member:: PyMethodDef* m_methods - - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present. - - .. c:member:: PyModuleDef_Slot* m_slots - - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be *NULL*. - - .. versionchanged:: 3.5 - - Prior to version 3.5, this member was always set to *NULL*, - and was defined as: - - .. c:member:: inquiry m_reload - - .. c:member:: traverseproc m_traverse - - A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. - - .. c:member:: inquiry m_clear - - A clear function to call during GC clearing of the module object, or - *NULL* if not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. - - .. c:member:: freefunc m_free - - A function to call during deallocation of the module object, or *NULL* if - not needed. This function may be called before module state - is allocated (:c:func:`PyModule_GetState()` may return `NULL`), - and before the :c:member:`Py_mod_exec` function is executed. - -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: - -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :const:`PYTHON_API_VERSION`. - - -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. - -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObject`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`__new__` and :py:meth:`__init__` methods -of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`__dict__` or individual classes created with :c:func:`PyType_FromSpec`). - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or *NULL* if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: - -.. c:type:: PyModuleDef_Slot - - .. c:member:: int slot - - A slot ID, chosen from the available values explained below. - - .. c:member:: void* value - - Value of the slot, whose meaning depends on the slot ID. - - .. versionadded:: 3.5 - -The *m_slots* array must be terminated by a slot with id 0. - -The available slot types are: - -.. c:var:: Py_mod_create - - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: - - .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error - and return *NULL*. - - This function should be kept minimal. In particular, it should not - call arbitrary Python code, as trying to import the same module again may - result in an infinite loop. - - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - - If ``Py_mod_create`` is not specified, the import machinery will create - a normal module object using :c:func:`PyModule_New`. The name is taken from - *spec*, not the definition, to allow extension modules to dynamically adjust - to their place in the module hierarchy and be imported under different - names through symlinks, all while sharing a single module definition. - - There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-*NULL* ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. - -.. c:var:: Py_mod_exec - - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. - The signature of the function is: - - .. c:function:: int exec_module(PyObject* module) - - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. - -See :PEP:`489` for more details on multi-phase initialization. - -Low-level module creation functions -................................... - -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. - -.. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - - Create a new module object, given the definition in *module* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :const:`PYTHON_API_VERSION`. - - .. versionadded:: 3.5 - -.. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) - - Create a new module object, given the definition in *module* and the - ModuleSpec *spec*, assuming the API version *module_api_version*. - If that version does not match the version of the running interpreter, - a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` - instead; only use this if you are sure you need it. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) - - Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) - - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) - - Add the functions from the *NULL* terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -Support functions -................. - -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: - -.. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - - Add an object to *module* as *name*. This is a convenience function which can - be used from the module's initialization function. This steals a reference to - *value*. Return ``-1`` on error, ``0`` on success. - -.. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) - - Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. - - -.. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) - - Add a string constant to *module* as *name*. This convenience function can be - used from the module's initialization function. The string *value* must be - *NULL*-terminated. Return ``-1`` on error, ``0`` on success. - - -.. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) - - Add an int constant to *module*. The name and the value are taken from - *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int - constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. - - -.. c:function:: int PyModule_AddStringMacro(PyObject *module, macro) - - Add a string constant to *module*. - - -Module lookup -^^^^^^^^^^^^^ - -Single-phase initialization creates singleton modules that can be looked up -in the context of the current interpreter. This allows the module object to be -retrieved later with only a reference to the module definition. - -These functions will not work on modules created using multi-phase initialization, -since multiple such modules can be created from a single definition. - -.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def) - - Returns the module object that was created from *def* for the current interpreter. - This method requires that the module object has been attached to the interpreter state with - :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not - found or has not been attached to the interpreter state yet, it returns *NULL*. - -.. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def) - - Attaches the module object passed to the function to the interpreter state. This allows - the module object to be accessible via :c:func:`PyState_FindModule`. - - Only effective on modules created using single-phase initialization. - - .. versionadded:: 3.3 - -.. c:function:: int PyState_RemoveModule(PyModuleDef *def) - - Removes the module object created from *def* from the interpreter state. - - .. versionadded:: 3.3 diff --git a/third_party/python/Doc/c-api/none.rst b/third_party/python/Doc/c-api/none.rst deleted file mode 100644 index 45568fe65..000000000 --- a/third_party/python/Doc/c-api/none.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. highlightlang:: c - -.. _noneobject: - -The ``None`` Object -------------------- - -.. index:: object: None - -Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the -Python/C API. Since ``None`` is a singleton, testing for object identity (using -``==`` in C) is sufficient. There is no :c:func:`PyNone_Check` function for the -same reason. - - -.. c:var:: PyObject* Py_None - - The Python ``None`` object, denoting lack of value. This object has no methods. - It needs to be treated just like any other object with respect to reference - counts. - - -.. c:macro:: Py_RETURN_NONE - - Properly handle returning :c:data:`Py_None` from within a C function (that is, - increment the reference count of ``None`` and return it.) diff --git a/third_party/python/Doc/c-api/number.rst b/third_party/python/Doc/c-api/number.rst deleted file mode 100644 index 296b21c13..000000000 --- a/third_party/python/Doc/c-api/number.rst +++ /dev/null @@ -1,283 +0,0 @@ -.. highlightlang:: c - -.. _number: - -Number Protocol -=============== - - -.. c:function:: int PyNumber_Check(PyObject *o) - - Returns ``1`` if the object *o* provides numeric protocols, and false otherwise. - This function always succeeds. - - -.. c:function:: PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) - - Returns the result of adding *o1* and *o2*, or *NULL* on failure. This is the - equivalent of the Python expression ``o1 + o2``. - - -.. c:function:: PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) - - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. This is - the equivalent of the Python expression ``o1 - o2``. - - -.. c:function:: PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) - - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. This is - the equivalent of the Python expression ``o1 * o2``. - - -.. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on - failure. This is the equivalent of the Python expression ``o1 @ o2``. - - .. versionadded:: 3.5 - - -.. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) - - Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is - equivalent to the "classic" division of integers. - - -.. c:function:: PyObject* PyNumber_TrueDivide(PyObject *o1, PyObject *o2) - - Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. - - -.. c:function:: PyObject* PyNumber_Remainder(PyObject *o1, PyObject *o2) - - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. This is - the equivalent of the Python expression ``o1 % o2``. - - -.. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) - - .. index:: builtin: divmod - - See the built-in function :func:`divmod`. Returns *NULL* on failure. This is - the equivalent of the Python expression ``divmod(o1, o2)``. - - -.. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) - - .. index:: builtin: pow - - See the built-in function :func:`pow`. Returns *NULL* on failure. This is the - equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. - If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing *NULL* for - *o3* would cause an illegal memory access). - - -.. c:function:: PyObject* PyNumber_Negative(PyObject *o) - - Returns the negation of *o* on success, or *NULL* on failure. This is the - equivalent of the Python expression ``-o``. - - -.. c:function:: PyObject* PyNumber_Positive(PyObject *o) - - Returns *o* on success, or *NULL* on failure. This is the equivalent of the - Python expression ``+o``. - - -.. c:function:: PyObject* PyNumber_Absolute(PyObject *o) - - .. index:: builtin: abs - - Returns the absolute value of *o*, or *NULL* on failure. This is the equivalent - of the Python expression ``abs(o)``. - - -.. c:function:: PyObject* PyNumber_Invert(PyObject *o) - - Returns the bitwise negation of *o* on success, or *NULL* on failure. This is - the equivalent of the Python expression ``~o``. - - -.. c:function:: PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) - - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on - failure. This is the equivalent of the Python expression ``o1 << o2``. - - -.. c:function:: PyObject* PyNumber_Rshift(PyObject *o1, PyObject *o2) - - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on - failure. This is the equivalent of the Python expression ``o1 >> o2``. - - -.. c:function:: PyObject* PyNumber_And(PyObject *o1, PyObject *o2) - - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. - This is the equivalent of the Python expression ``o1 & o2``. - - -.. c:function:: PyObject* PyNumber_Xor(PyObject *o1, PyObject *o2) - - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on - failure. This is the equivalent of the Python expression ``o1 ^ o2``. - - -.. c:function:: PyObject* PyNumber_Or(PyObject *o1, PyObject *o2) - - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. - This is the equivalent of the Python expression ``o1 | o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2) - - Returns the result of adding *o1* and *o2*, or *NULL* on failure. The operation - is done *in-place* when *o1* supports it. This is the equivalent of the Python - statement ``o1 += o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2) - - Returns the result of subtracting *o2* from *o1*, or *NULL* on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 -= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2) - - Returns the result of multiplying *o1* and *o2*, or *NULL* on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 *= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on - failure. The operation is done *in-place* when *o1* supports it. This is - the equivalent of the Python statement ``o1 @= o2``. - - .. versionadded:: 3.5 - - -.. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) - - Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure. - The operation is done *in-place* when *o1* supports it. This is the equivalent - of the Python statement ``o1 //= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceTrueDivide(PyObject *o1, PyObject *o2) - - Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or *NULL* on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. The operation is done *in-place* when *o1* supports it. - - -.. c:function:: PyObject* PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2) - - Returns the remainder of dividing *o1* by *o2*, or *NULL* on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 %= o2``. - - -.. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) - - .. index:: builtin: pow - - See the built-in function :func:`pow`. Returns *NULL* on failure. The operation - is done *in-place* when *o1* supports it. This is the equivalent of the Python - statement ``o1 **= o2`` when o3 is :c:data:`Py_None`, or an in-place variant of - ``pow(o1, o2, o3)`` otherwise. If *o3* is to be ignored, pass :c:data:`Py_None` - in its place (passing *NULL* for *o3* would cause an illegal memory access). - - -.. c:function:: PyObject* PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2) - - Returns the result of left shifting *o1* by *o2* on success, or *NULL* on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 <<= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2) - - Returns the result of right shifting *o1* by *o2* on success, or *NULL* on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 >>= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2) - - Returns the "bitwise and" of *o1* and *o2* on success and *NULL* on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 &= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceXor(PyObject *o1, PyObject *o2) - - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or *NULL* on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 ^= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceOr(PyObject *o1, PyObject *o2) - - Returns the "bitwise or" of *o1* and *o2* on success, or *NULL* on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 |= o2``. - - -.. c:function:: PyObject* PyNumber_Long(PyObject *o) - - .. index:: builtin: int - - Returns the *o* converted to an integer object on success, or *NULL* on - failure. This is the equivalent of the Python expression ``int(o)``. - - -.. c:function:: PyObject* PyNumber_Float(PyObject *o) - - .. index:: builtin: float - - Returns the *o* converted to a float object on success, or *NULL* on failure. - This is the equivalent of the Python expression ``float(o)``. - - -.. c:function:: PyObject* PyNumber_Index(PyObject *o) - - Returns the *o* converted to a Python int on success or *NULL* with a - :exc:`TypeError` exception raised on failure. - - -.. c:function:: PyObject* PyNumber_ToBase(PyObject *n, int base) - - Returns the integer *n* converted to base *base* as a string. The *base* - argument must be one of 2, 8, 10, or 16. For base 2, 8, or 16, the - returned string is prefixed with a base marker of ``'0b'``, ``'0o'``, or - ``'0x'``, respectively. If *n* is not a Python int, it is converted with - :c:func:`PyNumber_Index` first. - - -.. c:function:: Py_ssize_t PyNumber_AsSsize_t(PyObject *o, PyObject *exc) - - Returns *o* converted to a Py_ssize_t value if *o* can be interpreted as an - integer. If the call fails, an exception is raised and ``-1`` is returned. - - If *o* can be converted to a Python int but the attempt to - convert to a Py_ssize_t value would raise an :exc:`OverflowError`, then the - *exc* argument is the type of exception that will be raised (usually - :exc:`IndexError` or :exc:`OverflowError`). If *exc* is *NULL*, then the - exception is cleared and the value is clipped to *PY_SSIZE_T_MIN* for a negative - integer or *PY_SSIZE_T_MAX* for a positive integer. - - -.. c:function:: int PyIndex_Check(PyObject *o) - - Returns ``1`` if *o* is an index integer (has the nb_index slot of the - tp_as_number structure filled in), and ``0`` otherwise. - This function always succeeds. diff --git a/third_party/python/Doc/c-api/objbuffer.rst b/third_party/python/Doc/c-api/objbuffer.rst deleted file mode 100644 index 9ad7c571c..000000000 --- a/third_party/python/Doc/c-api/objbuffer.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. highlightlang:: c - -Old Buffer Protocol -------------------- - -.. deprecated:: 3.0 - -These functions were part of the "old buffer protocol" API in Python 2. -In Python 3, this protocol doesn't exist anymore but the functions are still -exposed to ease porting 2.x code. They act as a compatibility wrapper -around the :ref:`new buffer protocol `, but they don't give -you control over the lifetime of the resources acquired when a buffer is -exported. - -Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer` -(or the ``y*`` or ``w*`` :ref:`format codes ` with the -:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over -an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. - - -.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location usable as character-based - input. The *obj* argument must support the single-segment character buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location containing arbitrary data. - The *obj* argument must support the single-segment readable buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_CheckReadBuffer(PyObject *o) - - Returns ``1`` if *o* supports the single-segment readable buffer interface. - Otherwise returns ``0``. This function always succeeds. - - Note that this function tries to get and release a buffer, and exceptions - which occur while calling correspoding functions will get suppressed. - To get error reporting use :c:func:`PyObject_GetBuffer()` instead. - - -.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a writable memory location. The *obj* argument must - support the single-segment, character buffer interface. On success, - returns ``0``, sets *buffer* to the memory location and *buffer_len* to the - buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error. - diff --git a/third_party/python/Doc/c-api/object.rst b/third_party/python/Doc/c-api/object.rst deleted file mode 100644 index ea0aa8df3..000000000 --- a/third_party/python/Doc/c-api/object.rst +++ /dev/null @@ -1,425 +0,0 @@ -.. highlightlang:: c - -.. _object: - -Object Protocol -=============== - - -.. c:var:: PyObject* Py_NotImplemented - - The ``NotImplemented`` singleton, used to signal that an operation is - not implemented for the given type combination. - - -.. c:macro:: Py_RETURN_NOTIMPLEMENTED - - Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). - - -.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) - - Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument - is used to enable certain printing options. The only option currently supported - is :const:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. - - -.. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) - - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. - - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttr()` instead. - - -.. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) - - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. - - Note that exceptions which occur while calling :meth:`__getattr__` and - :meth:`__getattribute__` methods and creating a temporary string object - will get suppressed. - To get error reporting use :c:func:`PyObject_GetAttrString()` instead. - - -.. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) - - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python - expression ``o.attr_name``. - - -.. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) - - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or *NULL* on failure. This is the equivalent of the Python - expression ``o.attr_name``. - - -.. c:function:: PyObject* PyObject_GenericGetAttr(PyObject *o, PyObject *name) - - Generic attribute getter function that is meant to be put into a type - object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary - of classes in the object's MRO as well as an attribute in the object's - :attr:`~object.__dict__` (if present). As outlined in :ref:`descriptors`, - data descriptors take preference over instance attributes, while non-data - descriptors don't. Otherwise, an :exc:`AttributeError` is raised. - - -.. c:function:: int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v) - - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. - - If *v* is *NULL*, the attribute is deleted, however this feature is - deprecated in favour of using :c:func:`PyObject_DelAttr`. - - -.. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) - - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. - - If *v* is *NULL*, the attribute is deleted, however this feature is - deprecated in favour of using :c:func:`PyObject_DelAttrString`. - - -.. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) - - Generic attribute setter and deleter function that is meant - to be put into a type object's :c:member:`~PyTypeObject.tp_setattro` - slot. It looks for a data descriptor in the - dictionary of classes in the object's MRO, and if found it takes preference - over setting or deleting the attribute in the instance dictionary. Otherwise, the - attribute is set or deleted in the object's :attr:`~object.__dict__` (if present). - On success, ``0`` is returned, otherwise an :exc:`AttributeError` - is raised and ``-1`` is returned. - - -.. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) - - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. - - -.. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) - - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. - - -.. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) - - A generic implementation for the getter of a ``__dict__`` descriptor. It - creates the dictionary if necessary. - - .. versionadded:: 3.3 - - -.. c:function:: int PyObject_GenericSetDict(PyObject *o, void *context) - - A generic implementation for the setter of a ``__dict__`` descriptor. This - implementation does not allow the dictionary to be deleted. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid) - - Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of - the Python expression ``o1 op o2``, where ``op`` is the operator corresponding - to *opid*. Returns the value of the comparison on success, or *NULL* on failure. - - -.. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) - - Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :const:`Py_LT`, :const:`Py_LE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_GT`, or :const:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. - -.. note:: - If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` - will always return ``1`` for :const:`Py_EQ` and ``0`` for :const:`Py_NE`. - -.. c:function:: PyObject* PyObject_Repr(PyObject *o) - - .. index:: builtin: repr - - Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the - Python expression ``repr(o)``. Called by the :func:`repr` built-in function. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - -.. c:function:: PyObject* PyObject_ASCII(PyObject *o) - - .. index:: builtin: ascii - - As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but - escape the non-ASCII characters in the string returned by - :c:func:`PyObject_Repr` with ``\x``, ``\u`` or ``\U`` escapes. This generates - a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. - Called by the :func:`ascii` built-in function. - - .. index:: string; PyObject_Str (C function) - - -.. c:function:: PyObject* PyObject_Str(PyObject *o) - - Compute a string representation of object *o*. Returns the string - representation on success, *NULL* on failure. This is the equivalent of the - Python expression ``str(o)``. Called by the :func:`str` built-in function - and, therefore, by the :func:`print` function. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - -.. c:function:: PyObject* PyObject_Bytes(PyObject *o) - - .. index:: builtin: bytes - - Compute a bytes representation of object *o*. *NULL* is returned on - failure and a bytes object on success. This is equivalent to the Python - expression ``bytes(o)``, when *o* is not an integer. Unlike ``bytes(o)``, - a TypeError is raised when *o* is an integer instead of a zero-initialized - bytes object. - - -.. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) - - Return ``1`` if the class *derived* is identical to or derived from the class - *cls*, otherwise return ``0``. In case of an error, return ``-1``. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, - *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. - - Normally only class objects, i.e. instances of :class:`type` or a derived - class, are considered classes. However, objects can override this by having - a :attr:`__bases__` attribute (which must be a tuple of base classes). - - -.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) - - Return ``1`` if *inst* is an instance of the class *cls* or a subclass of - *cls*, or ``0`` if not. On error, returns ``-1`` and sets an exception. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, *inst* - is an instance of *cls* if its class is a subclass of *cls*. - - An instance *inst* can override what is considered its class by having a - :attr:`__class__` attribute. - - An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`__bases__` attribute (which must be a tuple - of base classes). - - -.. c:function:: int PyCallable_Check(PyObject *o) - - Determine if the object *o* is callable. Return ``1`` if the object is callable - and ``0`` otherwise. This function always succeeds. - - -.. c:function:: PyObject* PyObject_Call(PyObject *callable_object, PyObject *args, PyObject *kw) - - Call a callable Python object *callable_object*, with arguments given by the - tuple *args*, and named arguments given by the dictionary *kw*. If no named - arguments are needed, *kw* may be *NULL*. *args* must not be *NULL*, use an - empty tuple if no arguments are needed. Returns the result of the call on - success, or *NULL* on failure. This is the equivalent of the Python expression - ``callable_object(*args, **kw)``. - - -.. c:function:: PyObject* PyObject_CallObject(PyObject *callable_object, PyObject *args) - - Call a callable Python object *callable_object*, with arguments given by the - tuple *args*. If no arguments are needed, then *args* may be *NULL*. Returns - the result of the call on success, or *NULL* on failure. This is the equivalent - of the Python expression ``callable_object(*args)``. - - -.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) - - Call a callable Python object *callable*, with a variable number of C arguments. - The C arguments are described using a :c:func:`Py_BuildValue` style format - string. The format may be *NULL*, indicating that no arguments are provided. - Returns the result of the call on success, or *NULL* on failure. This is the - equivalent of the Python expression ``callable(*args)``. Note that if you only - pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a - faster alternative. - - .. versionchanged:: 3.4 - The type of *format* was changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, const char *method, const char *format, ...) - - Call the method named *method* of object *o* with a variable number of C - arguments. The C arguments are described by a :c:func:`Py_BuildValue` format - string that should produce a tuple. The format may be *NULL*, indicating that - no arguments are provided. Returns the result of the call on success, or *NULL* - on failure. This is the equivalent of the Python expression ``o.method(args)``. - Note that if you only pass :c:type:`PyObject \*` args, - :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. - - .. versionchanged:: 3.4 - The types of *method* and *format* were changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) - - Call a callable Python object *callable*, with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or - *NULL* on failure. - - -.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *o, PyObject *name, ..., NULL) - - Calls a method of the object *o*, where the name of the method is given as a - Python string object in *name*. It is called with a variable number of - :c:type:`PyObject\*` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. Returns the result of the call on success, or - *NULL* on failure. - - -.. c:function:: Py_hash_t PyObject_Hash(PyObject *o) - - .. index:: builtin: hash - - Compute and return the hash value of an object *o*. On failure, return ``-1``. - This is the equivalent of the Python expression ``hash(o)``. - - .. versionchanged:: 3.2 - The return type is now Py_hash_t. This is a signed integer the same size - as Py_ssize_t. - - -.. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) - - Set a :exc:`TypeError` indicating that ``type(o)`` is not hashable and return ``-1``. - This function receives special treatment when stored in a ``tp_hash`` slot, - allowing a type to explicitly indicate to the interpreter that it is not - hashable. - - -.. c:function:: int PyObject_IsTrue(PyObject *o) - - Returns ``1`` if the object *o* is considered to be true, and ``0`` otherwise. - This is equivalent to the Python expression ``not not o``. On failure, return - ``-1``. - - -.. c:function:: int PyObject_Not(PyObject *o) - - Returns ``0`` if the object *o* is considered to be true, and ``1`` otherwise. - This is equivalent to the Python expression ``not o``. On failure, return - ``-1``. - - -.. c:function:: PyObject* PyObject_Type(PyObject *o) - - .. index:: builtin: type - - When *o* is non-*NULL*, returns a type object corresponding to the object type - of object *o*. On failure, raises :exc:`SystemError` and returns *NULL*. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this - function instead of the common expression ``o->ob_type``, which returns a - pointer of type :c:type:`PyTypeObject\*`, except when the incremented reference - count is needed. - - -.. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) - - Return true if the object *o* is of type *type* or a subtype of *type*. Both - parameters must be non-*NULL*. - - -.. c:function:: Py_ssize_t PyObject_Size(PyObject *o) - Py_ssize_t PyObject_Length(PyObject *o) - - .. index:: builtin: len - - Return the length of object *o*. If the object *o* provides either the sequence - and mapping protocols, the sequence length is returned. On error, ``-1`` is - returned. This is the equivalent to the Python expression ``len(o)``. - - -.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t default) - - Return an estimated length for the object *o*. First try to return its - actual length, then an estimate using :meth:`~object.__length_hint__`, and - finally return the default value. On error return ``-1``. This is the - equivalent to the Python expression ``operator.length_hint(o, default)``. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) - - Return element of *o* corresponding to the object *key* or *NULL* on failure. - This is the equivalent of the Python expression ``o[key]``. - - -.. c:function:: int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v) - - Map the object *key* to the value *v*. Raise an exception and - return ``-1`` on failure; return ``0`` on success. This is the - equivalent of the Python statement ``o[key] = v``. - - -.. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key) - - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. - - -.. c:function:: PyObject* PyObject_Dir(PyObject *o) - - This is equivalent to the Python expression ``dir(o)``, returning a (possibly - empty) list of strings appropriate for the object argument, or *NULL* if there - was an error. If the argument is *NULL*, this is like the Python ``dir()``, - returning the names of the current locals; in this case, if no execution frame - is active then *NULL* is returned but :c:func:`PyErr_Occurred` will return false. - - -.. c:function:: PyObject* PyObject_GetIter(PyObject *o) - - This is equivalent to the Python expression ``iter(o)``. It returns a new - iterator for the object argument, or the object itself if the object is already - an iterator. Raises :exc:`TypeError` and returns *NULL* if the object cannot be - iterated. diff --git a/third_party/python/Doc/c-api/objimpl.rst b/third_party/python/Doc/c-api/objimpl.rst deleted file mode 100644 index 7023e519e..000000000 --- a/third_party/python/Doc/c-api/objimpl.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. highlightlang:: c - -.. _newtypes: - -***************************** -Object Implementation Support -***************************** - -This chapter describes the functions, types, and macros used when defining new -object types. - -.. toctree:: - - allocation.rst - structures.rst - typeobj.rst - gcsupport.rst diff --git a/third_party/python/Doc/c-api/refcounting.rst b/third_party/python/Doc/c-api/refcounting.rst deleted file mode 100644 index 4f512ecdb..000000000 --- a/third_party/python/Doc/c-api/refcounting.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. highlightlang:: c - - -.. _countingrefs: - -****************** -Reference Counting -****************** - -The macros in this section are used for managing reference counts of Python -objects. - - -.. c:function:: void Py_INCREF(PyObject *o) - - Increment the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XINCREF`. - - -.. c:function:: void Py_XINCREF(PyObject *o) - - Increment the reference count for object *o*. The object may be *NULL*, in - which case the macro has no effect. - - -.. c:function:: void Py_DECREF(PyObject *o) - - Decrement the reference count for object *o*. The object must not be *NULL*; if - you aren't sure that it isn't *NULL*, use :c:func:`Py_XDECREF`. If the reference - count reaches zero, the object's type's deallocation function (which must not be - *NULL*) is invoked. - - .. warning:: - - The deallocation function can cause arbitrary Python code to be invoked (e.g. - when a class instance with a :meth:`__del__` method is deallocated). While - exceptions in such code are not propagated, the executed code has free access to - all Python global variables. This means that any object that is reachable from - a global variable should be in a consistent state before :c:func:`Py_DECREF` is - invoked. For example, code to delete an object from a list should copy a - reference to the deleted object in a temporary variable, update the list data - structure, and then call :c:func:`Py_DECREF` for the temporary variable. - - -.. c:function:: void Py_XDECREF(PyObject *o) - - Decrement the reference count for object *o*. The object may be *NULL*, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. - - -.. c:function:: void Py_CLEAR(PyObject *o) - - Decrement the reference count for object *o*. The object may be *NULL*, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, except that the argument is also set to *NULL*. The warning - for :c:func:`Py_DECREF` does not apply with respect to the object passed because - the macro carefully uses a temporary variable and sets the argument to *NULL* - before decrementing its reference count. - - It is a good idea to use this macro whenever decrementing the value of a - variable that might be traversed during garbage collection. - - -The following functions are for runtime dynamic embedding of Python: -``Py_IncRef(PyObject *o)``, ``Py_DecRef(PyObject *o)``. They are -simply exported function versions of :c:func:`Py_XINCREF` and -:c:func:`Py_XDECREF`, respectively. - -The following functions or macros are only for use within the interpreter core: -:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, -as well as the global variable :c:data:`_Py_RefTotal`. - diff --git a/third_party/python/Doc/c-api/reflection.rst b/third_party/python/Doc/c-api/reflection.rst deleted file mode 100644 index 96893652f..000000000 --- a/third_party/python/Doc/c-api/reflection.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. highlightlang:: c - -.. _reflection: - -Reflection -========== - -.. c:function:: PyObject* PyEval_GetBuiltins() - - Return a dictionary of the builtins in the current execution frame, - or the interpreter of the thread state if no frame is currently executing. - - -.. c:function:: PyObject* PyEval_GetLocals() - - Return a dictionary of the local variables in the current execution frame, - or *NULL* if no frame is currently executing. - - -.. c:function:: PyObject* PyEval_GetGlobals() - - Return a dictionary of the global variables in the current execution frame, - or *NULL* if no frame is currently executing. - - -.. c:function:: PyFrameObject* PyEval_GetFrame() - - Return the current thread state's frame, which is *NULL* if no frame is - currently executing. - - -.. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) - - Return the line number that *frame* is currently executing. - - -.. c:function:: const char* PyEval_GetFuncName(PyObject *func) - - Return the name of *func* if it is a function, class or instance object, else the - name of *func*\s type. - - -.. c:function:: const char* PyEval_GetFuncDesc(PyObject *func) - - Return a description string, depending on the type of *func*. - Return values include "()" for functions and methods, " constructor", - " instance", and " object". Concatenated with the result of - :c:func:`PyEval_GetFuncName`, the result will be a description of - *func*. diff --git a/third_party/python/Doc/c-api/sequence.rst b/third_party/python/Doc/c-api/sequence.rst deleted file mode 100644 index 6d22f35e2..000000000 --- a/third_party/python/Doc/c-api/sequence.rst +++ /dev/null @@ -1,169 +0,0 @@ -.. highlightlang:: c - -.. _sequence: - -Sequence Protocol -================= - - -.. c:function:: int PySequence_Check(PyObject *o) - - Return ``1`` if the object provides sequence protocol, and ``0`` otherwise. - Note that it returns ``1`` for Python classes with a :meth:`__getitem__` - method unless they are :class:`dict` subclasses since in general case it - is impossible to determine what the type of keys it supports. This - function always succeeds. - - -.. c:function:: Py_ssize_t PySequence_Size(PyObject *o) - Py_ssize_t PySequence_Length(PyObject *o) - - .. index:: builtin: len - - Returns the number of objects in sequence *o* on success, and ``-1`` on - failure. This is equivalent to the Python expression ``len(o)``. - - -.. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) - - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. - This is the equivalent of the Python expression ``o1 + o2``. - - -.. c:function:: PyObject* PySequence_Repeat(PyObject *o, Py_ssize_t count) - - Return the result of repeating sequence object *o* *count* times, or *NULL* on - failure. This is the equivalent of the Python expression ``o * count``. - - -.. c:function:: PyObject* PySequence_InPlaceConcat(PyObject *o1, PyObject *o2) - - Return the concatenation of *o1* and *o2* on success, and *NULL* on failure. - The operation is done *in-place* when *o1* supports it. This is the equivalent - of the Python expression ``o1 += o2``. - - -.. c:function:: PyObject* PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) - - Return the result of repeating sequence object *o* *count* times, or *NULL* on - failure. The operation is done *in-place* when *o* supports it. This is the - equivalent of the Python expression ``o *= count``. - - -.. c:function:: PyObject* PySequence_GetItem(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o*, or *NULL* on failure. This is the equivalent of - the Python expression ``o[i]``. - - -.. c:function:: PyObject* PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - - Return the slice of sequence object *o* between *i1* and *i2*, or *NULL* on - failure. This is the equivalent of the Python expression ``o[i1:i2]``. - - -.. c:function:: int PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v) - - Assign object *v* to the *i*\ th element of *o*. Raise an exception - and return ``-1`` on failure; return ``0`` on success. This - is the equivalent of the Python statement ``o[i] = v``. This function *does - not* steal a reference to *v*. - - If *v* is *NULL*, the element is deleted, however this feature is - deprecated in favour of using :c:func:`PySequence_DelItem`. - - -.. c:function:: int PySequence_DelItem(PyObject *o, Py_ssize_t i) - - Delete the *i*\ th element of object *o*. Returns ``-1`` on failure. This is the - equivalent of the Python statement ``del o[i]``. - - -.. c:function:: int PySequence_SetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2, PyObject *v) - - Assign the sequence object *v* to the slice in sequence object *o* from *i1* to - *i2*. This is the equivalent of the Python statement ``o[i1:i2] = v``. - - -.. c:function:: int PySequence_DelSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - - Delete the slice in sequence object *o* from *i1* to *i2*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``del o[i1:i2]``. - - -.. c:function:: Py_ssize_t PySequence_Count(PyObject *o, PyObject *value) - - Return the number of occurrences of *value* in *o*, that is, return the number - of keys for which ``o[key] == value``. On failure, return ``-1``. This is - equivalent to the Python expression ``o.count(value)``. - - -.. c:function:: int PySequence_Contains(PyObject *o, PyObject *value) - - Determine if *o* contains *value*. If an item in *o* is equal to *value*, - return ``1``, otherwise return ``0``. On error, return ``-1``. This is - equivalent to the Python expression ``value in o``. - - -.. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) - - Return the first index *i* for which ``o[i] == value``. On error, return - ``-1``. This is equivalent to the Python expression ``o.index(value)``. - - -.. c:function:: PyObject* PySequence_List(PyObject *o) - - Return a list object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. The returned list is guaranteed to be new. This is - equivalent to the Python expression ``list(o)``. - - -.. c:function:: PyObject* PySequence_Tuple(PyObject *o) - - .. index:: builtin: tuple - - Return a tuple object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. If *o* is a tuple, a new reference will be returned, - otherwise a tuple will be constructed with the appropriate contents. This is - equivalent to the Python expression ``tuple(o)``. - - -.. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - - Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in - which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access - the members of the result. Returns *NULL* on failure. If the object is not - a sequence or iterable, raises :exc:`TypeError` with *m* as the message text. - - -.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o) - - Returns the length of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be - gotten by calling :c:func:`PySequence_Size` on *o*, but - :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list - or tuple. - - -.. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast`, *o* is not *NULL*, and that *i* is within bounds. - - -.. c:function:: PyObject** PySequence_Fast_ITEMS(PyObject *o) - - Return the underlying array of PyObject pointers. Assumes that *o* was returned - by :c:func:`PySequence_Fast` and *o* is not *NULL*. - - Note, if a list gets resized, the reallocation may relocate the items array. - So, only use the underlying array pointer in contexts where the sequence - cannot change. - - -.. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o* or *NULL* on failure. Macro form of - :c:func:`PySequence_GetItem` but without checking that - :c:func:`PySequence_Check` on *o* is true and without adjustment for negative - indices. diff --git a/third_party/python/Doc/c-api/set.rst b/third_party/python/Doc/c-api/set.rst deleted file mode 100644 index 64b6dde3b..000000000 --- a/third_party/python/Doc/c-api/set.rst +++ /dev/null @@ -1,166 +0,0 @@ -.. highlightlang:: c - -.. _setobjects: - -Set Objects ------------ - -.. sectionauthor:: Raymond D. Hettinger - - -.. index:: - object: set - object: frozenset - -This section details the public API for :class:`set` and :class:`frozenset` -objects. Any functionality not listed below is best accessed using the either -the abstract object protocol (including :c:func:`PyObject_CallMethod`, -:c:func:`PyObject_RichCompareBool`, :c:func:`PyObject_Hash`, -:c:func:`PyObject_Repr`, :c:func:`PyObject_IsTrue`, :c:func:`PyObject_Print`, and -:c:func:`PyObject_GetIter`) or the abstract number protocol (including -:c:func:`PyNumber_And`, :c:func:`PyNumber_Subtract`, :c:func:`PyNumber_Or`, -:c:func:`PyNumber_Xor`, :c:func:`PyNumber_InPlaceAnd`, -:c:func:`PyNumber_InPlaceSubtract`, :c:func:`PyNumber_InPlaceOr`, and -:c:func:`PyNumber_InPlaceXor`). - - -.. c:type:: PySetObject - - This subtype of :c:type:`PyObject` is used to hold the internal data for both - :class:`set` and :class:`frozenset` objects. It is like a :c:type:`PyDictObject` - in that it is a fixed size for small sets (much like tuple storage) and will - point to a separate, variable sized block of memory for medium and large sized - sets (much like list storage). None of the fields of this structure should be - considered public and are subject to change. All access should be done through - the documented API rather than by manipulating the values in the structure. - - -.. c:var:: PyTypeObject PySet_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`set` type. - - -.. c:var:: PyTypeObject PyFrozenSet_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`frozenset` type. - -The following type check macros work on pointers to any Python object. Likewise, -the constructor functions work with any iterable Python object. - - -.. c:function:: int PySet_Check(PyObject *p) - - Return true if *p* is a :class:`set` object or an instance of a subtype. - -.. c:function:: int PyFrozenSet_Check(PyObject *p) - - Return true if *p* is a :class:`frozenset` object or an instance of a - subtype. - -.. c:function:: int PyAnySet_Check(PyObject *p) - - Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an - instance of a subtype. - - -.. c:function:: int PyAnySet_CheckExact(PyObject *p) - - Return true if *p* is a :class:`set` object or a :class:`frozenset` object but - not an instance of a subtype. - - -.. c:function:: int PyFrozenSet_CheckExact(PyObject *p) - - Return true if *p* is a :class:`frozenset` object but not an instance of a - subtype. - - -.. c:function:: PyObject* PySet_New(PyObject *iterable) - - Return a new :class:`set` containing objects returned by the *iterable*. The - *iterable* may be *NULL* to create a new empty set. Return the new set on - success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is not - actually iterable. The constructor is also useful for copying a set - (``c=set(s)``). - - -.. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) - - Return a new :class:`frozenset` containing objects returned by the *iterable*. - The *iterable* may be *NULL* to create a new empty frozenset. Return the new - set on success or *NULL* on failure. Raise :exc:`TypeError` if *iterable* is - not actually iterable. - - -The following functions and macros are available for instances of :class:`set` -or :class:`frozenset` or instances of their subtypes. - - -.. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) - - .. index:: builtin: len - - Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to - ``len(anyset)``. Raises a :exc:`PyExc_SystemError` if *anyset* is not a - :class:`set`, :class:`frozenset`, or an instance of a subtype. - - -.. c:function:: Py_ssize_t PySet_GET_SIZE(PyObject *anyset) - - Macro form of :c:func:`PySet_Size` without error checking. - - -.. c:function:: int PySet_Contains(PyObject *anyset, PyObject *key) - - Return ``1`` if found, ``0`` if not found, and ``-1`` if an error is encountered. Unlike - the Python :meth:`__contains__` method, this function does not automatically - convert unhashable sets into temporary frozensets. Raise a :exc:`TypeError` if - the *key* is unhashable. Raise :exc:`PyExc_SystemError` if *anyset* is not a - :class:`set`, :class:`frozenset`, or an instance of a subtype. - - -.. c:function:: int PySet_Add(PyObject *set, PyObject *key) - - Add *key* to a :class:`set` instance. Also works with :class:`frozenset` - instances (like :c:func:`PyTuple_SetItem` it can be used to fill-in the values - of brand new frozensets before they are exposed to other code). Return ``0`` on - success or ``-1`` on failure. Raise a :exc:`TypeError` if the *key* is - unhashable. Raise a :exc:`MemoryError` if there is no room to grow. Raise a - :exc:`SystemError` if *set* is not an instance of :class:`set` or its - subtype. - - -The following functions are available for instances of :class:`set` or its -subtypes but not for instances of :class:`frozenset` or its subtypes. - - -.. c:function:: int PySet_Discard(PyObject *set, PyObject *key) - - Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an - error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` - method, this function does not automatically convert unhashable sets into - temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is not an - instance of :class:`set` or its subtype. - - -.. c:function:: PyObject* PySet_Pop(PyObject *set) - - Return a new reference to an arbitrary object in the *set*, and removes the - object from the *set*. Return *NULL* on failure. Raise :exc:`KeyError` if the - set is empty. Raise a :exc:`SystemError` if *set* is not an instance of - :class:`set` or its subtype. - - -.. c:function:: int PySet_Clear(PyObject *set) - - Empty an existing set of all elements. - - -.. c:function:: int PySet_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - .. versionadded:: 3.3 diff --git a/third_party/python/Doc/c-api/slice.rst b/third_party/python/Doc/c-api/slice.rst deleted file mode 100644 index 8b695e065..000000000 --- a/third_party/python/Doc/c-api/slice.rst +++ /dev/null @@ -1,69 +0,0 @@ -.. highlightlang:: c - -.. _slice-objects: - -Slice Objects -------------- - - -.. c:var:: PyTypeObject PySlice_Type - - The type object for slice objects. This is the same as :class:`slice` in the - Python layer. - - -.. c:function:: int PySlice_Check(PyObject *ob) - - Return true if *ob* is a slice object; *ob* must not be *NULL*. - - -.. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) - - Return a new slice object with the given values. The *start*, *stop*, and - *step* parameters are used as the values of the slice object attributes of - the same names. Any of the values may be *NULL*, in which case the - ``None`` will be used for the corresponding attribute. Return *NULL* if - the new object could not be allocated. - - -.. c:function:: int PySlice_GetIndices(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) - - Retrieve the start, stop and step indices from the slice object *slice*, - assuming a sequence of length *length*. Treats indices greater than - *length* as errors. - - Returns ``0`` on success and ``-1`` on error with no exception set (unless one of - the indices was not :const:`None` and failed to be converted to an integer, - in which case ``-1`` is returned with an exception set). - - You probably do not want to use this function. - - .. versionchanged:: 3.2 - The parameter type for the *slice* parameter was ``PySliceObject*`` - before. - - -.. c:function:: int PySlice_GetIndicesEx(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step, Py_ssize_t *slicelength) - - Usable replacement for :c:func:`PySlice_GetIndices`. Retrieve the start, - stop, and step indices from the slice object *slice* assuming a sequence of - length *length*, and store the length of the slice in *slicelength*. Out - of bounds indices are clipped in a manner consistent with the handling of - normal slices. - - Returns ``0`` on success and ``-1`` on error with exception set. - - .. versionchanged:: 3.2 - The parameter type for the *slice* parameter was ``PySliceObject*`` - before. - - -Ellipsis Object ---------------- - - -.. c:var:: PyObject *Py_Ellipsis - - The Python ``Ellipsis`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. Like - :c:data:`Py_None` it is a singleton object. diff --git a/third_party/python/Doc/c-api/stable.rst b/third_party/python/Doc/c-api/stable.rst deleted file mode 100644 index 5b771dd4a..000000000 --- a/third_party/python/Doc/c-api/stable.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. highlightlang:: c - -.. _stable: - -*********************************** -Stable Application Binary Interface -*********************************** - -Traditionally, the C API of Python will change with every release. Most changes -will be source-compatible, typically by only adding API, rather than changing -existing API or removing API (although some interfaces do get removed after -being deprecated first). - -Unfortunately, the API compatibility does not extend to binary compatibility -(the ABI). The reason is primarily the evolution of struct definitions, where -addition of a new field, or changing the type of a field, might not break the -API, but can break the ABI. As a consequence, extension modules need to be -recompiled for every Python release (although an exception is possible on Unix -when none of the affected interfaces are used). In addition, on Windows, -extension modules link with a specific pythonXY.dll and need to be recompiled to -link with a newer one. - -Since Python 3.2, a subset of the API has been declared to guarantee a stable -ABI. Extension modules wishing to use this API (called "limited API") need to -define ``Py_LIMITED_API``. A number of interpreter details then become hidden -from the extension module; in return, a module is built that works on any 3.x -version (x>=2) without recompilation. - -In some cases, the stable ABI needs to be extended with new functions. -Extension modules wishing to use these new APIs need to set ``Py_LIMITED_API`` -to the ``PY_VERSION_HEX`` value (see :ref:`apiabiversion`) of the minimum Python -version they want to support (e.g. ``0x03030000`` for Python 3.3). Such modules -will work on all subsequent Python releases, but fail to load (because of -missing symbols) on the older releases. - -As of Python 3.2, the set of functions available to the limited API is -documented in :pep:`384`. In the C API documentation, API elements that are not -part of the limited API are marked as "Not part of the limited API." diff --git a/third_party/python/Doc/c-api/structures.rst b/third_party/python/Doc/c-api/structures.rst deleted file mode 100644 index c493f6469..000000000 --- a/third_party/python/Doc/c-api/structures.rst +++ /dev/null @@ -1,337 +0,0 @@ -.. highlightlang:: c - -.. _common-structs: - -Common Object Structures -======================== - -There are a large number of structures which are used in the definition of -object types for Python. This section describes these structures and how they -are used. - -All Python objects ultimately share a small number of fields at the beginning -of the object's representation in memory. These are represented by the -:c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn, -by the expansions of some macros also used, whether directly or indirectly, in -the definition of all other Python objects. - - -.. c:type:: PyObject - - All object types are extensions of this type. This is a type which - contains the information Python needs to treat a pointer to an object as an - object. In a normal "release" build, it contains only the object's - reference count and a pointer to the corresponding type object. - Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:type:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. - - -.. c:type:: PyVarObject - - This is an extension of :c:type:`PyObject` that adds the :attr:`ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. - - -.. c:macro:: PyObject_HEAD - - This is a macro used when declaring new types which represent objects - without a varying length. The PyObject_HEAD macro expands to:: - - PyObject ob_base; - - See documentation of :c:type:`PyObject` above. - - -.. c:macro:: PyObject_VAR_HEAD - - This is a macro used when declaring new types which represent objects - with a length that varies from instance to instance. - The PyObject_VAR_HEAD macro expands to:: - - PyVarObject ob_base; - - See documentation of :c:type:`PyVarObject` above. - - -.. c:macro:: Py_TYPE(o) - - This macro is used to access the :attr:`ob_type` member of a Python object. - It expands to:: - - (((PyObject*)(o))->ob_type) - - -.. c:macro:: Py_REFCNT(o) - - This macro is used to access the :attr:`ob_refcnt` member of a Python - object. - It expands to:: - - (((PyObject*)(o))->ob_refcnt) - - -.. c:macro:: Py_SIZE(o) - - This macro is used to access the :attr:`ob_size` member of a Python object. - It expands to:: - - (((PyVarObject*)(o))->ob_size) - - -.. c:macro:: PyObject_HEAD_INIT(type) - - This is a macro which expands to initialization values for a new - :c:type:`PyObject` type. This macro expands to:: - - _PyObject_EXTRA_INIT - 1, type, - - -.. c:macro:: PyVarObject_HEAD_INIT(type, size) - - This is a macro which expands to initialization values for a new - :c:type:`PyVarObject` type, including the :attr:`ob_size` field. - This macro expands to:: - - _PyObject_EXTRA_INIT - 1, type, size, - - -.. c:type:: PyCFunction - - Type of the functions used to implement most Python callables in C. - Functions of this type take two :c:type:`PyObject\*` parameters and return - one such value. If the return value is *NULL*, an exception shall have - been set. If not *NULL*, the return value is interpreted as the return - value of the function as exposed in Python. The function must return a new - reference. - - -.. c:type:: PyCFunctionWithKeywords - - Type of the functions used to implement Python callables in C that take - keyword arguments: they take three :c:type:`PyObject\*` parameters and return - one such value. See :c:type:`PyCFunction` above for the meaning of the return - value. - - -.. c:type:: PyMethodDef - - Structure used to describe a method of an extension type. This structure has - four fields: - - +------------------+-------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+=============+===============================+ - | :attr:`ml_name` | char \* | name of the method | - +------------------+-------------+-------------------------------+ - | :attr:`ml_meth` | PyCFunction | pointer to the C | - | | | implementation | - +------------------+-------------+-------------------------------+ - | :attr:`ml_flags` | int | flag bits indicating how the | - | | | call should be constructed | - +------------------+-------------+-------------------------------+ - | :attr:`ml_doc` | char \* | points to the contents of the | - | | | docstring | - +------------------+-------------+-------------------------------+ - -The :attr:`ml_meth` is a C function pointer. The functions may be of different -types, but they always return :c:type:`PyObject\*`. If the function is not of -the :c:type:`PyCFunction`, the compiler will require a cast in the method table. -Even though :c:type:`PyCFunction` defines the first parameter as -:c:type:`PyObject\*`, it is common that the method implementation uses the -specific C type of the *self* object. - -The :attr:`ml_flags` field is a bitfield which can include the following flags. -The individual flags indicate either a calling convention or a binding -convention. Of the calling convention flags, only :const:`METH_VARARGS` and -:const:`METH_KEYWORDS` can be combined. Any of the calling convention flags -can be combined with a binding flag. - - -.. data:: METH_VARARGS - - This is the typical calling convention, where the methods have the type - :c:type:`PyCFunction`. The function expects two :c:type:`PyObject\*` values. - The first one is the *self* object for methods; for module functions, it is - the module object. The second parameter (often called *args*) is a tuple - object representing all arguments. This parameter is typically processed - using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. - - -.. data:: METH_KEYWORDS - - Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, and a dictionary of - all the keyword arguments. The flag must be combined with - :const:`METH_VARARGS`, and the parameters are typically processed using - :c:func:`PyArg_ParseTupleAndKeywords`. - - -.. data:: METH_NOARGS - - Methods without parameters don't need to check whether arguments are given if - they are listed with the :const:`METH_NOARGS` flag. They need to be of type - :c:type:`PyCFunction`. The first parameter is typically named *self* and will - hold a reference to the module or object instance. In all cases the second - parameter will be *NULL*. - - -.. data:: METH_O - - Methods with a single object argument can be listed with the :const:`METH_O` - flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. - They have the type :c:type:`PyCFunction`, with the *self* parameter, and a - :c:type:`PyObject\*` parameter representing the single argument. - - -These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions -defined for modules. At most one of these flags may be set for any given -method. - - -.. data:: METH_CLASS - - .. index:: builtin: classmethod - - The method will be passed the type object as the first parameter rather - than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. - - -.. data:: METH_STATIC - - .. index:: builtin: staticmethod - - The method will be passed *NULL* as the first parameter rather than an - instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. - -One other constant controls whether a method is loaded in place of another -definition with the same method name. - - -.. data:: METH_COEXIST - - The method will be loaded in place of existing definitions. Without - *METH_COEXIST*, the default is to skip repeated definitions. Since slot - wrappers are loaded before the method table, the existence of a - *sq_contains* slot, for example, would generate a wrapped method named - :meth:`__contains__` and preclude the loading of a corresponding - PyCFunction with the same name. With the flag defined, the PyCFunction - will be loaded in place of the wrapper object and will co-exist with the - slot. This is helpful because calls to PyCFunctions are optimized more - than wrapper object calls. - - -.. c:type:: PyMemberDef - - Structure which describes an attribute of a type which corresponds to a C - struct member. Its fields are: - - +------------------+-------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+=============+===============================+ - | :attr:`name` | char \* | name of the member | - +------------------+-------------+-------------------------------+ - | :attr:`!type` | int | the type of the member in the | - | | | C struct | - +------------------+-------------+-------------------------------+ - | :attr:`offset` | Py_ssize_t | the offset in bytes that the | - | | | member is located on the | - | | | type's object struct | - +------------------+-------------+-------------------------------+ - | :attr:`flags` | int | flag bits indicating if the | - | | | field should be read-only or | - | | | writable | - +------------------+-------------+-------------------------------+ - | :attr:`doc` | char \* | points to the contents of the | - | | | docstring | - +------------------+-------------+-------------------------------+ - - :attr:`!type` can be one of many ``T_`` macros corresponding to various C - types. When the member is accessed in Python, it will be converted to the - equivalent Python type. - - =============== ================== - Macro name C type - =============== ================== - T_SHORT short - T_INT int - T_LONG long - T_FLOAT float - T_DOUBLE double - T_STRING char \* - T_OBJECT PyObject \* - T_OBJECT_EX PyObject \* - T_CHAR char - T_BYTE char - T_UBYTE unsigned char - T_UINT unsigned int - T_USHORT unsigned short - T_ULONG unsigned long - T_BOOL char - T_LONGLONG long long - T_ULONGLONG unsigned long long - T_PYSSIZET Py_ssize_t - =============== ================== - - :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that - :c:macro:`T_OBJECT` returns ``None`` if the member is *NULL* and - :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use - :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX` - handles use of the :keyword:`del` statement on that attribute more correctly - than :c:macro:`T_OBJECT`. - - :attr:`flags` can be ``0`` for write and read access or :c:macro:`READONLY` for - read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies - :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8. - Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` - members can be deleted. (They are set to *NULL*). - - -.. c:type:: PyGetSetDef - - Structure to define property-like access for a type. See also description of - the :c:member:`PyTypeObject.tp_getset` slot. - - +-------------+------------------+-----------------------------------+ - | Field | C Type | Meaning | - +=============+==================+===================================+ - | name | char \* | attribute name | - +-------------+------------------+-----------------------------------+ - | get | getter | C Function to get the attribute | - +-------------+------------------+-----------------------------------+ - | set | setter | optional C function to set or | - | | | delete the attribute, if omitted | - | | | the attribute is readonly | - +-------------+------------------+-----------------------------------+ - | doc | char \* | optional docstring | - +-------------+------------------+-----------------------------------+ - | closure | void \* | optional function pointer, | - | | | providing additional data for | - | | | getter and setter | - +-------------+------------------+-----------------------------------+ - - The ``get`` function takes one :c:type:`PyObject\*` parameter (the - instance) and a function pointer (the associated ``closure``):: - - typedef PyObject *(*getter)(PyObject *, void *); - - It should return a new reference on success or *NULL* with a set exception - on failure. - - ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: - - typedef int (*setter)(PyObject *, PyObject *, void *); - - In case the attribute should be deleted the second parameter is *NULL*. - Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/third_party/python/Doc/c-api/sys.rst b/third_party/python/Doc/c-api/sys.rst deleted file mode 100644 index 48e2b2bb5..000000000 --- a/third_party/python/Doc/c-api/sys.rst +++ /dev/null @@ -1,267 +0,0 @@ -.. highlightlang:: c - -.. _os: - -Operating System Utilities -========================== - -.. c:function:: PyObject* PyOS_FSPath(PyObject *path) - - Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, - then :meth:`~os.PathLike.__fspath__` is returned as long as it is a - :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised - and ``NULL`` is returned. - - .. versionadded:: 3.6 - - -.. c:function:: int Py_FdIsInteractive(FILE *fp, const char *filename) - - Return true (nonzero) if the standard I/O file *fp* with name *filename* is - deemed interactive. This is the case for files for which ``isatty(fileno(fp))`` - is true. If the global flag :c:data:`Py_InteractiveFlag` is true, this function - also returns true if the *filename* pointer is *NULL* or if the name is equal to - one of the strings ``''`` or ``'???'``. - - -.. c:function:: void PyOS_AfterFork() - - Function to update some internal state after a process fork; this should be - called in the new process if the Python interpreter will continue to be used. - If a new executable is loaded into the new process, this function does not need - to be called. - - -.. c:function:: int PyOS_CheckStack() - - Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :const:`USE_STACKCHECK` is defined (currently - on Windows using the Microsoft Visual C++ compiler). :const:`USE_STACKCHECK` - will be defined automatically; you should never change the definition in your - own code. - - -.. c:function:: PyOS_sighandler_t PyOS_getsig(int i) - - Return the current signal handler for signal *i*. This is a thin wrapper around - either :c:func:`sigaction` or :c:func:`signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:type:`void - (\*)(int)`. - - -.. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) - - Set the signal handler for signal *i* to be *h*; return the old signal handler. - This is a thin wrapper around either :c:func:`sigaction` or :c:func:`signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:type:`void (\*)(int)`. - -.. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) - - Decode a byte string from the locale encoding with the :ref:`surrogateescape - error handler `: undecodable bytes are decoded as - characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a - surrogate character, escape the bytes using the surrogateescape error - handler instead of decoding them. - - Encoding, highest priority to lowest priority: - - * ``UTF-8`` on macOS and Android; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions use the - ``ISO-8859-1`` encoding. - * the current locale encoding (``LC_CTYPE`` locale). - - Return a pointer to a newly allocated wide character string, use - :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write - the number of wide characters excluding the null character into ``*size``. - - Return ``NULL`` on decoding error or memory allocation error. If *size* is - not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to - ``(size_t)-2`` on decoding error. - - Decoding errors should never happen, unless there is a bug in the C - library. - - Use the :c:func:`Py_EncodeLocale` function to encode the character string - back to a byte string. - - .. seealso:: - - The :c:func:`PyUnicode_DecodeFSDefaultAndSize` and - :c:func:`PyUnicode_DecodeLocaleAndSize` functions. - - .. versionadded:: 3.5 - - -.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - - Encode a wide character string to the locale encoding with the - :ref:`surrogateescape error handler `: surrogate characters - in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF. - - Encoding, highest priority to lowest priority: - - * ``UTF-8`` on macOS and Android; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the - ``ISO-8859-1`` encoding. - * the current locale encoding. - - Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` - to free the memory. Return ``NULL`` on encoding error or memory allocation - error - - If error_pos is not ``NULL``, ``*error_pos`` is set to the index of the - invalid character on encoding error, or set to ``(size_t)-1`` otherwise. - - Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back - to a wide character string. - - .. seealso:: - - The :c:func:`PyUnicode_EncodeFSDefault` and - :c:func:`PyUnicode_EncodeLocale` functions. - - .. versionadded:: 3.5 - - -.. _systemfunctions: - -System Functions -================ - -These are utility functions that make functionality from the :mod:`sys` module -accessible to C code. They all work with the current interpreter thread's -:mod:`sys` module's dict, which is contained in the internal thread state structure. - -.. c:function:: PyObject *PySys_GetObject(const char *name) - - Return the object *name* from the :mod:`sys` module or *NULL* if it does - not exist, without setting an exception. - -.. c:function:: int PySys_SetObject(const char *name, PyObject *v) - - Set *name* in the :mod:`sys` module to *v* unless *v* is *NULL*, in which - case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` - on error. - -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. - -.. c:function:: void PySys_AddWarnOption(wchar_t *s) - - Append *s* to :data:`sys.warnoptions`. - -.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) - - Append *unicode* to :data:`sys.warnoptions`. - -.. c:function:: void PySys_SetPath(wchar_t *path) - - Set :data:`sys.path` to a list object of paths found in *path* which should - be a list of paths separated with the platform's search path delimiter - (``:`` on Unix, ``;`` on Windows). - -.. c:function:: void PySys_WriteStdout(const char *format, ...) - - Write the output string described by *format* to :data:`sys.stdout`. No - exceptions are raised, even if truncation occurs (see below). - - *format* should limit the total size of the formatted output string to - 1000 bytes or less -- after 1000 bytes, the output string is truncated. - In particular, this means that no unrestricted "%s" formats should occur; - these should be limited using "%.s" where is a decimal number - calculated so that plus the maximum size of other formatted text does not - exceed 1000 bytes. Also watch out for "%f", which can print hundreds of - digits for very large numbers. - - If a problem occurs, or :data:`sys.stdout` is unset, the formatted message - is written to the real (C level) *stdout*. - -.. c:function:: void PySys_WriteStderr(const char *format, ...) - - As :c:func:`PySys_WriteStdout`, but write to :data:`sys.stderr` or *stderr* - instead. - -.. c:function:: void PySys_FormatStdout(const char *format, ...) - - Function similar to PySys_WriteStdout() but format the message using - :c:func:`PyUnicode_FromFormatV` and don't truncate the message to an - arbitrary length. - - .. versionadded:: 3.2 - -.. c:function:: void PySys_FormatStderr(const char *format, ...) - - As :c:func:`PySys_FormatStdout`, but write to :data:`sys.stderr` or *stderr* - instead. - - .. versionadded:: 3.2 - -.. c:function:: void PySys_AddXOption(const wchar_t *s) - - Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. - - .. versionadded:: 3.2 - -.. c:function:: PyObject *PySys_GetXOptions() - - Return the current dictionary of :option:`-X` options, similarly to - :data:`sys._xoptions`. On error, *NULL* is returned and an exception is - set. - - .. versionadded:: 3.2 - - -.. _processcontrol: - -Process Control -=============== - - -.. c:function:: void Py_FatalError(const char *message) - - .. index:: single: abort() - - Print a fatal error message and kill the process. No cleanup is performed. - This function should only be invoked when a condition is detected that would - make it dangerous to continue using the Python interpreter; e.g., when the - object administration appears to be corrupted. On Unix, the standard C library - function :c:func:`abort` is called which will attempt to produce a :file:`core` - file. - - -.. c:function:: void Py_Exit(int status) - - .. index:: - single: Py_FinalizeEx() - single: exit() - - Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the - standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` - indicates an error, the exit status is set to 120. - - .. versionchanged:: 3.6 - Errors from finalization no longer ignored. - - -.. c:function:: int Py_AtExit(void (*func) ()) - - .. index:: - single: Py_FinalizeEx() - single: cleanup functions - - Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup - function will be called with no arguments and should return no value. At most - 32 cleanup functions can be registered. When the registration is successful, - :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup - function registered last is called first. Each cleanup function will be called - at most once. Since Python's internal finalization will have completed before - the cleanup function, no Python APIs should be called by *func*. diff --git a/third_party/python/Doc/c-api/tuple.rst b/third_party/python/Doc/c-api/tuple.rst deleted file mode 100644 index ab8ee9abf..000000000 --- a/third_party/python/Doc/c-api/tuple.rst +++ /dev/null @@ -1,218 +0,0 @@ -.. highlightlang:: c - -.. _tupleobjects: - -Tuple Objects -------------- - -.. index:: object: tuple - - -.. c:type:: PyTupleObject - - This subtype of :c:type:`PyObject` represents a Python tuple object. - - -.. c:var:: PyTypeObject PyTuple_Type - - This instance of :c:type:`PyTypeObject` represents the Python tuple type; it - is the same object as :class:`tuple` in the Python layer. - - -.. c:function:: int PyTuple_Check(PyObject *p) - - Return true if *p* is a tuple object or an instance of a subtype of the tuple - type. - - -.. c:function:: int PyTuple_CheckExact(PyObject *p) - - Return true if *p* is a tuple object, but not an instance of a subtype of the - tuple type. - - -.. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - - Return a new tuple object of size *len*, or *NULL* on failure. - - -.. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - - Return a new tuple object of size *n*, or *NULL* on failure. The tuple values - are initialized to the subsequent *n* C arguments pointing to Python objects. - ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. - - -.. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) - - Take a pointer to a tuple object, and return the size of that tuple. - - -.. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - - Return the size of the tuple *p*, which must be non-*NULL* and point to a tuple; - no error checking is performed. - - -.. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) - - Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is - out of bounds, return *NULL* and sets an :exc:`IndexError` exception. - - -.. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) - - Like :c:func:`PyTuple_GetItem`, but does no checking of its arguments. - - -.. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) - - Take a slice of the tuple pointed to by *p* from *low* to *high* and return it - as a new tuple. - - -.. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) - - Insert a reference to object *o* at position *pos* of the tuple pointed to by - *p*. Return ``0`` on success. - - .. note:: - - This function "steals" a reference to *o*. - - -.. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) - - Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be - used to fill in brand new tuples. - - .. note:: - - This function "steals" a reference to *o*. - - -.. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) - - Can be used to resize a tuple. *newsize* will be the new length of the tuple. - Because tuples are *supposed* to be immutable, this should only be used if there - is only one reference to the object. Do *not* use this if the tuple may already - be known to some other part of the code. The tuple will always grow or shrink - at the end. Think of this as destroying the old tuple and creating a new one, - only more efficiently. Returns ``0`` on success. Client code should never - assume that the resulting value of ``*p`` will be the same as before calling - this function. If the object referenced by ``*p`` is replaced, the original - ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to *NULL*, and - raises :exc:`MemoryError` or :exc:`SystemError`. - - -.. c:function:: int PyTuple_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - -Struct Sequence Objects ------------------------ - -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. -To create a struct sequence, you first have to create a specific struct sequence -type. - -.. c:function:: PyTypeObject* PyStructSequence_NewType(PyStructSequence_Desc *desc) - - Create a new struct sequence type from the data in *desc*, described below. Instances - of the resulting type can be created with :c:func:`PyStructSequence_New`. - - -.. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) - - Initializes a struct sequence type *type* from *desc* in place. - - -.. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. - - .. versionadded:: 3.4 - - -.. c:type:: PyStructSequence_Desc - - Contains the meta information of a struct sequence type to create. - - +-------------------+------------------------------+------------------------------------+ - | Field | C Type | Meaning | - +===================+==============================+====================================+ - | ``name`` | ``char *`` | name of the struct sequence type | - +-------------------+------------------------------+------------------------------------+ - | ``doc`` | ``char *`` | pointer to docstring for the type | - | | | or NULL to omit | - +-------------------+------------------------------+------------------------------------+ - | ``fields`` | ``PyStructSequence_Field *`` | pointer to *NULL*-terminated array | - | | | with field names of the new type | - +-------------------+------------------------------+------------------------------------+ - | ``n_in_sequence`` | ``int`` | number of fields visible to the | - | | | Python side (if used as tuple) | - +-------------------+------------------------------+------------------------------------+ - - -.. c:type:: PyStructSequence_Field - - Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:type:`PyObject\*`. The index in the - :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which - field of the struct sequence is described. - - +-----------+---------------+--------------------------------------+ - | Field | C Type | Meaning | - +===========+===============+======================================+ - | ``name`` | ``char *`` | name for the field or *NULL* to end | - | | | the list of named fields, set to | - | | | PyStructSequence_UnnamedField to | - | | | leave unnamed | - +-----------+---------------+--------------------------------------+ - | ``doc`` | ``char *`` | field docstring or *NULL* to omit | - +-----------+---------------+--------------------------------------+ - - -.. c:var:: char* PyStructSequence_UnnamedField - - Special value for a field name to leave it unnamed. - - -.. c:function:: PyObject* PyStructSequence_New(PyTypeObject *type) - - Creates an instance of *type*, which must have been created with - :c:func:`PyStructSequence_NewType`. - - -.. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) - - Return the object at position *pos* in the struct sequence pointed to by *p*. - No bounds checking is performed. - - -.. c:function:: PyObject* PyStructSequence_GET_ITEM(PyObject *p, Py_ssize_t pos) - - Macro equivalent of :c:func:`PyStructSequence_GetItem`. - - -.. c:function:: void PyStructSequence_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) - - Sets the field at index *pos* of the struct sequence *p* to value *o*. Like - :c:func:`PyTuple_SET_ITEM`, this should only be used to fill in brand new - instances. - - .. note:: - - This function "steals" a reference to *o*. - - -.. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) - - Macro equivalent of :c:func:`PyStructSequence_SetItem`. - - .. note:: - - This function "steals" a reference to *o*. diff --git a/third_party/python/Doc/c-api/type.rst b/third_party/python/Doc/c-api/type.rst deleted file mode 100644 index 4dfd53fb9..000000000 --- a/third_party/python/Doc/c-api/type.rst +++ /dev/null @@ -1,118 +0,0 @@ -.. highlightlang:: c - -.. _typeobjects: - -Type Objects ------------- - -.. index:: object: type - - -.. c:type:: PyTypeObject - - The C structure of the objects used to describe built-in types. - - -.. c:var:: PyObject* PyType_Type - - This is the type object for type objects; it is the same object as - :class:`type` in the Python layer. - - -.. c:function:: int PyType_Check(PyObject *o) - - Return true if the object *o* is a type object, including instances of types - derived from the standard type object. Return false in all other cases. - - -.. c:function:: int PyType_CheckExact(PyObject *o) - - Return true if the object *o* is a type object, but not a subtype of the - standard type object. Return false in all other cases. - - -.. c:function:: unsigned int PyType_ClearCache() - - Clear the internal lookup cache. Return the current version tag. - -.. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type) - - Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily - meant for use with `Py_LIMITED_API`; the individual flag bits are - guaranteed to be stable across Python releases, but access to - :c:member:`~PyTypeObject.tp_flags` itself is not part of the limited API. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The return type is now ``unsigned long`` rather than ``long``. - - -.. c:function:: void PyType_Modified(PyTypeObject *type) - - Invalidate the internal lookup cache for the type and all of its - subtypes. This function must be called after any manual - modification of the attributes or base classes of the type. - - -.. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) - - Return true if the type object *o* sets the feature *feature*. Type features - are denoted by single bit flags. - - -.. c:function:: int PyType_IS_GC(PyTypeObject *o) - - Return true if the type object includes support for the cycle detector; this - tests the type flag :const:`Py_TPFLAGS_HAVE_GC`. - - -.. c:function:: int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) - - Return true if *a* is a subtype of *b*. - - This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call - :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` - would do. - - -.. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to *NULL*. - -.. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. - -.. c:function:: int PyType_Ready(PyTypeObject *type) - - Finalize a type object. This should be called on all type objects to finish - their initialization. This function is responsible for adding inherited slots - from a type's base class. Return ``0`` on success, or return ``-1`` and sets an - exception on error. - -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - - Creates and returns a heap type object from the *spec* passed to the function. - -.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - - Creates and returns a heap type object from the *spec*. In addition to that, - the created heap type contains all types contained by the *bases* tuple as base - types. This allows the caller to reference other heap types as base types. - - .. versionadded:: 3.3 - -.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) - - Return the function pointer stored in the given slot. If the - result is *NULL*, this indicates that either the slot is *NULL*, - or that the function was called with invalid parameters. - Callers will typically cast the result pointer into the appropriate - function type. - - .. versionadded:: 3.4 diff --git a/third_party/python/Doc/c-api/typeobj.rst b/third_party/python/Doc/c-api/typeobj.rst deleted file mode 100644 index 76515fd2c..000000000 --- a/third_party/python/Doc/c-api/typeobj.rst +++ /dev/null @@ -1,1402 +0,0 @@ -.. highlightlang:: c - -.. _type-structs: - -Type Objects -============ - -Perhaps one of the most important structures of the Python object system is the -structure that defines a new type: the :c:type:`PyTypeObject` structure. Type -objects can be handled using any of the :c:func:`PyObject_\*` or -:c:func:`PyType_\*` functions, but do not offer much that's interesting to most -Python applications. These objects are fundamental to how objects behave, so -they are very important to the interpreter itself and to any extension module -that implements new types. - -Type objects are fairly large compared to most of the standard types. The reason -for the size is that each type object stores a large number of values, mostly C -function pointers, each of which implements a small part of the type's -functionality. The fields of the type object are examined in detail in this -section. The fields will be described in the order in which they occur in the -structure. - -Typedefs: unaryfunc, binaryfunc, ternaryfunc, inquiry, intargfunc, -intintargfunc, intobjargproc, intintobjargproc, objobjargproc, destructor, -freefunc, printfunc, getattrfunc, getattrofunc, setattrfunc, setattrofunc, -reprfunc, hashfunc - -The structure definition for :c:type:`PyTypeObject` can be found in -:file:`Include/object.h`. For convenience of reference, this repeats the -definition found there: - -.. literalinclude:: ../includes/typestruct.h - - -The type object structure extends the :c:type:`PyVarObject` structure. The -:attr:`ob_size` field is used for dynamic types (created by :func:`type_new`, -usually called from a class statement). Note that :c:data:`PyType_Type` (the -metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. -type objects) *must* have the :attr:`ob_size` field. - - -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined. - Their initialization to *NULL* is taken care of by the ``PyObject_HEAD_INIT`` - macro. For statically allocated objects, these fields always remain *NULL*. - For dynamically allocated objects, these two fields are used to link the object - into a doubly-linked list of *all* live objects on the heap. This could be used - for various debugging purposes; currently the only use is to print the objects - that are still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - These fields are not inherited by subtypes. - - -.. c:member:: Py_ssize_t PyObject.ob_refcnt - - This is the type object's reference count, initialized to ``1`` by the - ``PyObject_HEAD_INIT`` macro. Note that for statically allocated type objects, - the type's instances (objects whose :attr:`ob_type` points back to the type) do - *not* count as references. But for dynamically allocated type objects, the - instances *do* count as references. - - This field is not inherited by subtypes. - - -.. c:member:: PyTypeObject* PyObject.ob_type - - This is the type's type, in other words its metatype. It is initialized by the - argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be - ``&PyType_Type``. However, for dynamically loadable extension modules that must - be usable on Windows (at least), the compiler complains that this is not a valid - initializer. Therefore, the convention is to pass *NULL* to the - ``PyObject_HEAD_INIT`` macro and to initialize this field explicitly at the - start of the module's initialization function, before doing anything else. This - is typically done like this:: - - Foo_Type.ob_type = &PyType_Type; - - This should be done before any instances of the type are created. - :c:func:`PyType_Ready` checks if :attr:`ob_type` is *NULL*, and if so, - initializes it to the :attr:`ob_type` field of the base class. - :c:func:`PyType_Ready` will not change this field if it is non-zero. - - This field is inherited by subtypes. - - -.. c:member:: Py_ssize_t PyVarObject.ob_size - - For statically allocated type objects, this should be initialized to zero. For - dynamically allocated type objects, this field has a special internal meaning. - - This field is not inherited by subtypes. - - -.. c:member:: const char* PyTypeObject.tp_name - - Pointer to a NUL-terminated string containing the name of the type. For types - that are accessible as module globals, the string should be the full module - name, followed by a dot, followed by the type name; for built-in types, it - should be just the type name. If the module is a submodule of a package, the - full package name is part of the full module name. For example, a type named - :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` - should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. - - For dynamically allocated type objects, this should just be the type name, and - the module name explicitly stored in the type dict as the value for key - ``'__module__'``. - - For statically allocated type objects, the tp_name field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` - attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. - - If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined - (unless explicitly set in the dictionary, as explained above). This means your - type will be impossible to pickle. Additionally, it will not be listed in - module documentations created with pydoc. - - This field is not inherited by subtypes. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_basicsize - Py_ssize_t PyTypeObject.tp_itemsize - - These fields allow calculating the size in bytes of instances of the type. - - There are two kinds of types: types with fixed-length instances have a zero - :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero - :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. - - For a type with variable-length instances, the instances must have an - :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N - times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of - N is typically stored in the instance's :attr:`ob_size` field. There are - exceptions: for example, ints use a negative :attr:`ob_size` to indicate a - negative number, and N is ``abs(ob_size)`` there. Also, the presence of an - :attr:`ob_size` field in the instance layout doesn't mean that the instance - structure is variable-length (for example, the structure for the list type has - fixed-length instances, yet those instances have a meaningful :attr:`ob_size` - field). - - The basic size includes the fields in the instance declared by the macro - :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to - declare the instance struct) and this in turn includes the :attr:`_ob_prev` and - :attr:`_ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the - ``sizeof`` operator on the struct used to declare the instance layout. - The basic size does not include the GC header size. - - These fields are inherited separately by subtypes. If the base type has a - non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set - :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this - depends on the implementation of the base type). - - A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: - suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is - ``sizeof(double)``. It is the programmer's responsibility that - :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the - alignment requirement for ``double``). - - -.. c:member:: destructor PyTypeObject.tp_dealloc - - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). - - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not - subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Del` if the instance was allocated - using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. - - This field is inherited by subtypes. - - -.. c:member:: printfunc PyTypeObject.tp_print - - Reserved slot, formerly used for print formatting in Python 2.x. - - -.. c:member:: getattrfunc PyTypeObject.tp_getattr - - An optional pointer to the get-attribute-string function. - - This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_getattro` function, but taking a C string - instead of a Python string object to give the attribute name. The signature is :: - - PyObject * tp_getattr(PyObject *o, char *attr_name); - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. - - -.. c:member:: setattrfunc PyTypeObject.tp_setattr - - An optional pointer to the function for setting and deleting attributes. - - This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_setattro` function, but taking a C string - instead of a Python string object to give the attribute name. The signature is :: - - PyObject * tp_setattr(PyObject *o, char *attr_name, PyObject *v); - - The *v* argument is set to *NULL* to delete the attribute. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. - - -.. c:member:: PyAsyncMethods* tp_as_async - - Pointer to an additional structure that contains fields relevant only to - objects which implement :term:`awaitable` and :term:`asynchronous iterator` - protocols at the C-level. See :ref:`async-structs` for details. - - .. versionadded:: 3.5 - Formerly known as ``tp_compare`` and ``tp_reserved``. - - -.. c:member:: reprfunc PyTypeObject.tp_repr - - .. index:: builtin: repr - - An optional pointer to a function that implements the built-in function - :func:`repr`. - - The signature is the same as for :c:func:`PyObject_Repr`; it must return a string - or a Unicode object. Ideally, this function should return a string that, when - passed to :func:`eval`, given a suitable environment, returns an object with the - same value. If this is not feasible, it should return a string starting with - ``'<'`` and ending with ``'>'`` from which both the type and the value of the - object can be deduced. - - When this field is not set, a string of the form ``<%s object at %p>`` is - returned, where ``%s`` is replaced by the type name, and ``%p`` by the object's - memory address. - - This field is inherited by subtypes. - -.. c:member:: PyNumberMethods* tp_as_number - - Pointer to an additional structure that contains fields relevant only to - objects which implement the number protocol. These fields are documented in - :ref:`number-structs`. - - The :c:member:`~PyTypeObject.tp_as_number` field is not inherited, but the contained fields are - inherited individually. - - -.. c:member:: PySequenceMethods* tp_as_sequence - - Pointer to an additional structure that contains fields relevant only to - objects which implement the sequence protocol. These fields are documented - in :ref:`sequence-structs`. - - The :c:member:`~PyTypeObject.tp_as_sequence` field is not inherited, but the contained fields - are inherited individually. - - -.. c:member:: PyMappingMethods* tp_as_mapping - - Pointer to an additional structure that contains fields relevant only to - objects which implement the mapping protocol. These fields are documented in - :ref:`mapping-structs`. - - The :c:member:`~PyTypeObject.tp_as_mapping` field is not inherited, but the contained fields - are inherited individually. - - -.. c:member:: hashfunc PyTypeObject.tp_hash - - .. index:: builtin: hash - - An optional pointer to a function that implements the built-in function - :func:`hash`. - - The signature is the same as for :c:func:`PyObject_Hash`; it must return a - value of the type Py_hash_t. The value ``-1`` should not be returned as a - normal return value; when an error occurs during the computation of the hash - value, the function should set an exception and return ``-1``. - - This field can be set explicitly to :c:func:`PyObject_HashNotImplemented` to - block inheritance of the hash method from a parent type. This is interpreted - as the equivalent of ``__hash__ = None`` at the Python level, causing - ``isinstance(o, collections.Hashable)`` to correctly return ``False``. Note - that the converse is also true - setting ``__hash__ = None`` on a class at - the Python level will result in the ``tp_hash`` slot being set to - :c:func:`PyObject_HashNotImplemented`. - - When this field is not set, an attempt to take the hash of the - object raises :exc:`TypeError`. - - This field is inherited by subtypes together with - :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both *NULL*. - - -.. c:member:: ternaryfunc PyTypeObject.tp_call - - An optional pointer to a function that implements calling the object. This - should be *NULL* if the object is not callable. The signature is the same as - for :c:func:`PyObject_Call`. - - This field is inherited by subtypes. - - -.. c:member:: reprfunc PyTypeObject.tp_str - - An optional pointer to a function that implements the built-in operation - :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the - constructor for that type. This constructor calls :c:func:`PyObject_Str` to do - the actual work, and :c:func:`PyObject_Str` will call this handler.) - - The signature is the same as for :c:func:`PyObject_Str`; it must return a string - or a Unicode object. This function should return a "friendly" string - representation of the object, as this is the representation that will be used, - among other things, by the :func:`print` function. - - When this field is not set, :c:func:`PyObject_Repr` is called to return a string - representation. - - This field is inherited by subtypes. - - -.. c:member:: getattrofunc PyTypeObject.tp_getattro - - An optional pointer to the get-attribute function. - - The signature is the same as for :c:func:`PyObject_GetAttr`. It is usually - convenient to set this field to :c:func:`PyObject_GenericGetAttr`, which - implements the normal way of looking for object attributes. - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. - - -.. c:member:: setattrofunc PyTypeObject.tp_setattro - - An optional pointer to the function for setting and deleting attributes. - - The signature is the same as for :c:func:`PyObject_SetAttr`, but setting - *v* to *NULL* to delete an attribute must be supported. It is usually - convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which - implements the normal way of setting object attributes. - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. - - -.. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer - - Pointer to an additional structure that contains fields relevant only to objects - which implement the buffer interface. These fields are documented in - :ref:`buffer-structs`. - - The :c:member:`~PyTypeObject.tp_as_buffer` field is not inherited, but the contained fields are - inherited individually. - - -.. c:member:: unsigned long PyTypeObject.tp_flags - - This field is a bit mask of various flags. Some flags indicate variant - semantics for certain situations; others are used to indicate that certain - fields in the type object (or in the extension structures referenced via - :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and - :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if - such a flag bit is clear, the type fields it guards must not be accessed and - must be considered to have a zero or *NULL* value instead. - - Inheritance of this field is complicated. Most flag bits are inherited - individually, i.e. if the base type has a flag bit set, the subtype inherits - this flag bit. The flag bits that pertain to extension structures are strictly - inherited if the extension structure is inherited, i.e. the base type's value of - the flag bit is copied into the subtype together with a pointer to the extension - structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with - the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the - :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the - :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have - *NULL* values. - - The following bit masks are currently defined; these can be ORed together using - the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro - :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and - checks whether ``tp->tp_flags & f`` is non-zero. - - - .. data:: Py_TPFLAGS_HEAPTYPE - - This bit is set when the type object itself is allocated on the heap. In this - case, the :attr:`ob_type` field of its instances is considered a reference to - the type, and the type object is INCREF'ed when a new instance is created, and - DECREF'ed when an instance is destroyed (this does not apply to instances of - subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). - - - .. data:: Py_TPFLAGS_BASETYPE - - This bit is set when the type can be used as the base type of another type. If - this bit is clear, the type cannot be subtyped (similar to a "final" class in - Java). - - - .. data:: Py_TPFLAGS_READY - - This bit is set when the type object has been fully initialized by - :c:func:`PyType_Ready`. - - - .. data:: Py_TPFLAGS_READYING - - This bit is set while :c:func:`PyType_Ready` is in the process of initializing - the type object. - - - .. data:: Py_TPFLAGS_HAVE_GC - - This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:func:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. - - - .. data:: Py_TPFLAGS_DEFAULT - - This is a bitmask of all the bits that pertain to the existence of certain - fields in the type object and its extension structures. Currently, it includes - the following bits: :const:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`, - :const:`Py_TPFLAGS_HAVE_VERSION_TAG`. - - - .. data:: Py_TPFLAGS_LONG_SUBCLASS - .. data:: Py_TPFLAGS_LIST_SUBCLASS - .. data:: Py_TPFLAGS_TUPLE_SUBCLASS - .. data:: Py_TPFLAGS_BYTES_SUBCLASS - .. data:: Py_TPFLAGS_UNICODE_SUBCLASS - .. data:: Py_TPFLAGS_DICT_SUBCLASS - .. data:: Py_TPFLAGS_BASE_EXC_SUBCLASS - .. data:: Py_TPFLAGS_TYPE_SUBCLASS - - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass - of a built-in type; such specific checks are faster than a generic - check, like :c:func:`PyObject_IsInstance`. Custom types that inherit - from built-ins should have their :c:member:`~PyTypeObject.tp_flags` - set appropriately, or the code that interacts with such types - will behave differently depending on what kind of check is used. - - - .. data:: Py_TPFLAGS_HAVE_FINALIZE - - This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the - type structure. - - .. versionadded:: 3.4 - - -.. c:member:: const char* PyTypeObject.tp_doc - - An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. - - This field is *not* inherited by subtypes. - - -.. c:member:: traverseproc PyTypeObject.tp_traverse - - An optional pointer to a traversal function for the garbage collector. This is - only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. More information - about Python's garbage collection scheme can be found in section - :ref:`supporting-cycle-detection`. - - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function - simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects. For example, this is function :c:func:`local_traverse` from the - :mod:`_thread` extension module:: - - static int - local_traverse(localobject *self, visitproc visit, void *arg) - { - Py_VISIT(self->args); - Py_VISIT(self->kw); - Py_VISIT(self->dict); - return 0; - } - - Note that :c:func:`Py_VISIT` is called only on those members that can participate - in reference cycles. Although there is also a ``self->key`` member, it can only - be *NULL* or a Python string and therefore cannot be part of a reference cycle. - - On the other hand, even if you know a member can never be part of a cycle, as a - debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. - - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just - anything. - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in - the subtype. - - -.. c:member:: inquiry PyTypeObject.tp_clear - - An optional pointer to a clear function for the garbage collector. This is only - used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. - - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. - - Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to - those of its members that may be Python objects, and set its pointers to those - members to *NULL*, as in the following example:: - - static int - local_clear(localobject *self) - { - Py_CLEAR(self->key); - Py_CLEAR(self->args); - Py_CLEAR(self->kw); - Py_CLEAR(self->dict); - return 0; - } - - The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until - after the pointer to the contained object is set to *NULL*. This is because - decrementing the reference count may cause the contained object to become trash, - triggering a chain of reclamation activity that may include invoking arbitrary - Python code (due to finalizers, or weakref callbacks, associated with the - contained object). If it's possible for such code to reference *self* again, - it's important that the pointer to the contained object be *NULL* at that time, - so that *self* knows the contained object can no longer be used. The - :c:func:`Py_CLEAR` macro performs the operations in a safe order. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - - More information about Python's garbage collection scheme can be found in - section :ref:`supporting-cycle-detection`. - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in - the subtype. - - -.. c:member:: richcmpfunc PyTypeObject.tp_richcompare - - An optional pointer to the rich comparison function, whose signature is - ``PyObject *tp_richcompare(PyObject *a, PyObject *b, int op)``. The first - parameter is guaranteed to be an instance of the type that is defined - by :c:type:`PyTypeObject`. - - The function should return the result of the comparison (usually ``Py_True`` - or ``Py_False``). If the comparison is undefined, it must return - ``Py_NotImplemented``, if another error occurred it must return ``NULL`` and - set an exception condition. - - .. note:: - - If you want to implement a type for which only a limited set of - comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and - friends), directly raise :exc:`TypeError` in the rich comparison function. - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: - a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when - the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both - *NULL*. - - The following constants are defined to be used as the third argument for - :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: - - +----------------+------------+ - | Constant | Comparison | - +================+============+ - | :const:`Py_LT` | ``<`` | - +----------------+------------+ - | :const:`Py_LE` | ``<=`` | - +----------------+------------+ - | :const:`Py_EQ` | ``==`` | - +----------------+------------+ - | :const:`Py_NE` | ``!=`` | - +----------------+------------+ - | :const:`Py_GT` | ``>`` | - +----------------+------------+ - | :const:`Py_GE` | ``>=`` | - +----------------+------------+ - - -.. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset - - If the instances of this type are weakly referenceable, this field is greater - than zero and contains the offset in the instance structure of the weak - reference list head (ignoring the GC header, if present); this offset is used by - :c:func:`PyObject_ClearWeakRefs` and the :c:func:`PyWeakref_\*` functions. The - instance structure needs to include a field of type :c:type:`PyObject\*` which is - initialized to *NULL*. - - Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for - weak references to the type object itself. - - This field is inherited by subtypes, but see the rules listed below. A subtype - may override this offset; this means that the subtype uses a different weak - reference list head than the base type. Since the list head is always found via - :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. - - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, - and none of its base types are weakly referenceable, the type is made weakly - referenceable by adding a weak reference list head slot to the instance layout - and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. - - When a type's :attr:`__slots__` declaration contains a slot named - :attr:`__weakref__`, that slot becomes the weak reference list head for - instances of the type, and the slot's offset is stored in the type's - :c:member:`~PyTypeObject.tp_weaklistoffset`. - - When a type's :attr:`__slots__` declaration does not contain a slot named - :attr:`__weakref__`, the type inherits its :c:member:`~PyTypeObject.tp_weaklistoffset` from its - base type. - -.. c:member:: getiterfunc PyTypeObject.tp_iter - - An optional pointer to a function that returns an iterator for the object. Its - presence normally signals that the instances of this type are iterable (although - sequences may be iterable without this function). - - This function has the same signature as :c:func:`PyObject_GetIter`. - - This field is inherited by subtypes. - - -.. c:member:: iternextfunc PyTypeObject.tp_iternext - - An optional pointer to a function that returns the next item in an iterator. - When the iterator is exhausted, it must return *NULL*; a :exc:`StopIteration` - exception may or may not be set. When another error occurs, it must return - *NULL* too. Its presence signals that the instances of this type are - iterators. - - Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that - function should return the iterator instance itself (not a new iterator - instance). - - This function has the same signature as :c:func:`PyIter_Next`. - - This field is inherited by subtypes. - - -.. c:member:: struct PyMethodDef* PyTypeObject.tp_methods - - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMethodDef` - structures, declaring regular methods of this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a method descriptor. - - This field is not inherited by subtypes (methods are inherited through a - different mechanism). - - -.. c:member:: struct PyMemberDef* PyTypeObject.tp_members - - An optional pointer to a static *NULL*-terminated array of :c:type:`PyMemberDef` - structures, declaring regular data members (fields or slots) of instances of - this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a member descriptor. - - This field is not inherited by subtypes (members are inherited through a - different mechanism). - - -.. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset - - An optional pointer to a static *NULL*-terminated array of :c:type:`PyGetSetDef` - structures, declaring computed attributes of instances of this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a getset descriptor. - - This field is not inherited by subtypes (computed attributes are inherited - through a different mechanism). - - -.. c:member:: PyTypeObject* PyTypeObject.tp_base - - An optional pointer to a base type from which type properties are inherited. At - this level, only single inheritance is supported; multiple inheritance require - dynamically creating a type object by calling the metatype. - - This field is not inherited by subtypes (obviously), but it defaults to - ``&PyBaseObject_Type`` (which to Python programmers is known as the type - :class:`object`). - - -.. c:member:: PyObject* PyTypeObject.tp_dict - - The type's dictionary is stored here by :c:func:`PyType_Ready`. - - This field should normally be initialized to *NULL* before PyType_Ready is - called; it may also be initialized to a dictionary containing initial attributes - for the type. Once :c:func:`PyType_Ready` has initialized the type, extra - attributes for the type may be added to this dictionary only if they don't - correspond to overloaded operations (like :meth:`__add__`). - - This field is not inherited by subtypes (though the attributes defined in here - are inherited through a different mechanism). - - .. warning:: - - It is not safe to use :c:func:`PyDict_SetItem` on or otherwise modify - :c:member:`~PyTypeObject.tp_dict` with the dictionary C-API. - - -.. c:member:: descrgetfunc PyTypeObject.tp_descr_get - - An optional pointer to a "descriptor get" function. - - The function signature is :: - - PyObject * tp_descr_get(PyObject *self, PyObject *obj, PyObject *type); - - .. XXX explain. - - This field is inherited by subtypes. - - -.. c:member:: descrsetfunc PyTypeObject.tp_descr_set - - An optional pointer to a function for setting and deleting - a descriptor's value. - - The function signature is :: - - int tp_descr_set(PyObject *self, PyObject *obj, PyObject *value); - - The *value* argument is set to *NULL* to delete the value. - This field is inherited by subtypes. - - .. XXX explain. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_dictoffset - - If the instances of this type have a dictionary containing instance variables, - this field is non-zero and contains the offset in the instances of the type of - the instance variable dictionary; this offset is used by - :c:func:`PyObject_GenericGetAttr`. - - Do not confuse this field with :c:member:`~PyTypeObject.tp_dict`; that is the dictionary for - attributes of the type object itself. - - If the value of this field is greater than zero, it specifies the offset from - the start of the instance structure. If the value is less than zero, it - specifies the offset from the *end* of the instance structure. A negative - offset is more expensive to use, and should only be used when the instance - structure contains a variable-length part. This is used for example to add an - instance variable dictionary to subtypes of :class:`str` or :class:`tuple`. Note - that the :c:member:`~PyTypeObject.tp_basicsize` field should account for the dictionary added to - the end in that case, even though the dictionary is not included in the basic - object layout. On a system with a pointer size of 4 bytes, - :c:member:`~PyTypeObject.tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is - at the very end of the structure. - - The real dictionary offset in an instance can be computed from a negative - :c:member:`~PyTypeObject.tp_dictoffset` as follows:: - - dictoffset = tp_basicsize + abs(ob_size)*tp_itemsize + tp_dictoffset - if dictoffset is not aligned on sizeof(void*): - round up to sizeof(void*) - - where :c:member:`~PyTypeObject.tp_basicsize`, :c:member:`~PyTypeObject.tp_itemsize` and :c:member:`~PyTypeObject.tp_dictoffset` are - taken from the type object, and :attr:`ob_size` is taken from the instance. The - absolute value is taken because ints use the sign of :attr:`ob_size` to - store the sign of the number. (There's never a need to do this calculation - yourself; it is done for you by :c:func:`_PyObject_GetDictPtr`.) - - This field is inherited by subtypes, but see the rules listed below. A subtype - may override this offset; this means that the subtype instances store the - dictionary at a difference offset than the base type. Since the dictionary is - always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. - - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, - and none of its base types has an instance variable dictionary, a dictionary - slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to - that slot's offset. - - When a type defined by a class statement has a :attr:`__slots__` declaration, - the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. - - (Adding a slot named :attr:`~object.__dict__` to the :attr:`__slots__` declaration does - not have the expected effect, it just causes confusion. Maybe this should be - added as a feature just like :attr:`__weakref__` though.) - - -.. c:member:: initproc PyTypeObject.tp_init - - An optional pointer to an instance initialization function. - - This function corresponds to the :meth:`__init__` method of classes. Like - :meth:`__init__`, it is possible to create an instance without calling - :meth:`__init__`, and it is possible to reinitialize an instance by calling its - :meth:`__init__` method again. - - The function signature is :: - - int tp_init(PyObject *self, PyObject *args, PyObject *kwds) - - The self argument is the instance to be initialized; the *args* and *kwds* - arguments represent positional and keyword arguments of the call to - :meth:`__init__`. - - The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is - created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function - has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an - instance of some other type that is not a subtype of the original type, no - :c:member:`~PyTypeObject.tp_init` function is called; if :c:member:`~PyTypeObject.tp_new` returns an instance of a - subtype of the original type, the subtype's :c:member:`~PyTypeObject.tp_init` is called. - - This field is inherited by subtypes. - - -.. c:member:: allocfunc PyTypeObject.tp_alloc - - An optional pointer to an instance allocation function. - - The function signature is :: - - PyObject *tp_alloc(PyTypeObject *self, Py_ssize_t nitems) - - The purpose of this function is to separate memory allocation from memory - initialization. It should return a pointer to a block of memory of adequate - length for the instance, suitably aligned, and initialized to zeros, but with - :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If - the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field - should be initialized to *nitems* and the length of the allocated memory block - should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of - ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block - should be :c:member:`~PyTypeObject.tp_basicsize`. - - Do not use this function to do any other instance initialization, not even to - allocate additional memory; that should be done by :c:member:`~PyTypeObject.tp_new`. - - This field is inherited by static subtypes, but not by dynamic subtypes - (subtypes created by a class statement); in the latter, this field is always set - to :c:func:`PyType_GenericAlloc`, to force a standard heap allocation strategy. - That is also the recommended value for statically defined types. - - -.. c:member:: newfunc PyTypeObject.tp_new - - An optional pointer to an instance creation function. - - If this function is *NULL* for a particular type, that type cannot be called to - create new instances; presumably there is some other way to create instances, - like a factory function. - - The function signature is :: - - PyObject *tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) - - The subtype argument is the type of the object being created; the *args* and - *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that subtype doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` - function is called; it may be a subtype of that type (but not an unrelated - type). - - The :c:member:`~PyTypeObject.tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` - to allocate space for the object, and then do only as much further - initialization as is absolutely necessary. Initialization that can safely be - ignored or repeated should be placed in the :c:member:`~PyTypeObject.tp_init` handler. A good - rule of thumb is that for immutable types, all initialization should take place - in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be - deferred to :c:member:`~PyTypeObject.tp_init`. - - This field is inherited by subtypes, except it is not inherited by static types - whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. - - -.. c:member:: destructor PyTypeObject.tp_free - - An optional pointer to an instance deallocation function. Its signature is - :c:type:`freefunc`:: - - void tp_free(void *) - - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. - - This field is inherited by static subtypes, but not by dynamic subtypes - (subtypes created by a class statement); in the latter, this field is set to a - deallocator suitable to match :c:func:`PyType_GenericAlloc` and the value of the - :const:`Py_TPFLAGS_HAVE_GC` flag bit. - - -.. c:member:: inquiry PyTypeObject.tp_is_gc - - An optional pointer to a function called by the garbage collector. - - The garbage collector needs to know whether a particular object is collectible - or not. Normally, it is sufficient to look at the object's type's - :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But - some types have a mixture of statically and dynamically allocated instances, and - the statically allocated instances are not collectible. Such types should - define this function; it should return ``1`` for a collectible instance, and - ``0`` for a non-collectible instance. The signature is :: - - int tp_is_gc(PyObject *self) - - (The only example of this are types themselves. The metatype, - :c:data:`PyType_Type`, defines this function to distinguish between statically - and dynamically allocated types.) - - This field is inherited by subtypes. - - -.. c:member:: PyObject* PyTypeObject.tp_bases - - Tuple of base types. - - This is set for types created by a class statement. It should be *NULL* for - statically defined types. - - This field is not inherited. - - -.. c:member:: PyObject* PyTypeObject.tp_mro - - Tuple containing the expanded set of base types, starting with the type itself - and ending with :class:`object`, in Method Resolution Order. - - This field is not inherited; it is calculated fresh by :c:func:`PyType_Ready`. - - -.. c:member:: destructor PyTypeObject.tp_finalize - - An optional pointer to an instance finalization function. Its signature is - :c:type:`destructor`:: - - void tp_finalize(PyObject *) - - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. - - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: - - static void - local_finalize(PyObject *self) - { - PyObject *error_type, *error_value, *error_traceback; - - /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - /* ... */ - - /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); - } - - For this field to be taken into account (even through inheritance), - you must also set the :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit. - - This field is inherited by subtypes. - - .. versionadded:: 3.4 - - .. seealso:: "Safe object finalization" (:pep:`442`) - - -.. c:member:: PyObject* PyTypeObject.tp_cache - - Unused. Not inherited. Internal use only. - - -.. c:member:: PyObject* PyTypeObject.tp_subclasses - - List of weak references to subclasses. Not inherited. Internal use only. - - -.. c:member:: PyObject* PyTypeObject.tp_weaklist - - Weak reference list head, for weak references to this type object. Not - inherited. Internal use only. - -The remaining fields are only defined if the feature test macro -:const:`COUNT_ALLOCS` is defined, and are for internal use only. They are -documented here for completeness. None of these fields are inherited by -subtypes. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_allocs - - Number of allocations. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_frees - - Number of frees. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_maxalloc - - Maximum simultaneously allocated objects. - - -.. c:member:: PyTypeObject* PyTypeObject.tp_next - - Pointer to the next type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. - -Also, note that, in a garbage collected Python, tp_dealloc may be called from -any Python thread, not just the thread which created the object (if the object -becomes part of a refcount cycle, that cycle might be collected by a garbage -collection on any thread). This is not a problem for Python API calls, since -the thread on which tp_dealloc is called will own the Global Interpreter Lock -(GIL). However, if the object being destroyed in turn destroys objects from some -other C or C++ library, care should be taken to ensure that destroying those -objects on the thread which called tp_dealloc will not violate any assumptions -of the library. - - -.. _number-structs: - -Number Object Structures -======================== - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PyNumberMethods - - This structure holds pointers to the functions which an object uses to - implement the number protocol. Each function is used by the function of - similar name documented in the :ref:`number` section. - - Here is the structure definition:: - - typedef struct { - binaryfunc nb_add; - binaryfunc nb_subtract; - binaryfunc nb_multiply; - binaryfunc nb_remainder; - binaryfunc nb_divmod; - ternaryfunc nb_power; - unaryfunc nb_negative; - unaryfunc nb_positive; - unaryfunc nb_absolute; - inquiry nb_bool; - unaryfunc nb_invert; - binaryfunc nb_lshift; - binaryfunc nb_rshift; - binaryfunc nb_and; - binaryfunc nb_xor; - binaryfunc nb_or; - unaryfunc nb_int; - void *nb_reserved; - unaryfunc nb_float; - - binaryfunc nb_inplace_add; - binaryfunc nb_inplace_subtract; - binaryfunc nb_inplace_multiply; - binaryfunc nb_inplace_remainder; - ternaryfunc nb_inplace_power; - binaryfunc nb_inplace_lshift; - binaryfunc nb_inplace_rshift; - binaryfunc nb_inplace_and; - binaryfunc nb_inplace_xor; - binaryfunc nb_inplace_or; - - binaryfunc nb_floor_divide; - binaryfunc nb_true_divide; - binaryfunc nb_inplace_floor_divide; - binaryfunc nb_inplace_true_divide; - - unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; - } PyNumberMethods; - - .. note:: - - Binary and ternary functions must check the type of all their operands, - and implement the necessary conversions (at least one of the operands is - an instance of the defined type). If the operation is not defined for the - given operands, binary and ternary functions must return - ``Py_NotImplemented``, if another error occurred they must return ``NULL`` - and set an exception. - - .. note:: - - The :c:data:`nb_reserved` field should always be ``NULL``. It - was previously called :c:data:`nb_long`, and was renamed in - Python 3.0.1. - - -.. _mapping-structs: - -Mapping Object Structures -========================= - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PyMappingMethods - - This structure holds pointers to the functions which an object uses to - implement the mapping protocol. It has three members: - -.. c:member:: lenfunc PyMappingMethods.mp_length - - This function is used by :c:func:`PyMapping_Size` and - :c:func:`PyObject_Size`, and has the same signature. This slot may be set to - *NULL* if the object has no defined length. - -.. c:member:: binaryfunc PyMappingMethods.mp_subscript - - This function is used by :c:func:`PyObject_GetItem` and - :c:func:`PySequence_GetSlice`, and has the same signature as - :c:func:`!PyObject_GetItem`. This slot must be filled for the - :c:func:`PyMapping_Check` function to return ``1``, it can be *NULL* - otherwise. - -.. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript - - This function is used by :c:func:`PyObject_SetItem`, - :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and - :c:func:`PyObject_DelSlice`. It has the same signature as - :c:func:`!PyObject_SetItem`, but *v* can also be set to *NULL* to delete - an item. If this slot is *NULL*, the object does not support item - assignment and deletion. - - -.. _sequence-structs: - -Sequence Object Structures -========================== - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PySequenceMethods - - This structure holds pointers to the functions which an object uses to - implement the sequence protocol. - -.. c:member:: lenfunc PySequenceMethods.sq_length - - This function is used by :c:func:`PySequence_Size` and - :c:func:`PyObject_Size`, and has the same signature. It is also used for - handling negative indices via the :c:member:`~PySequenceMethods.sq_item` - and the :c:member:`~PySequenceMethods.sq_ass_item` slots. - -.. c:member:: binaryfunc PySequenceMethods.sq_concat - - This function is used by :c:func:`PySequence_Concat` and has the same - signature. It is also used by the ``+`` operator, after trying the numeric - addition via the :c:member:`~PyNumberMethods.nb_add` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_repeat - - This function is used by :c:func:`PySequence_Repeat` and has the same - signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_item - - This function is used by :c:func:`PySequence_GetItem` and has the same - signature. It is also used by :c:func:`PyObject_GetItem`, after trying - the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. - This slot must be filled for the :c:func:`PySequence_Check` - function to return ``1``, it can be *NULL* otherwise. - - Negative indexes are handled as follows: if the :attr:`sq_length` slot is - filled, it is called and the sequence length is used to compute a positive - index which is passed to :attr:`sq_item`. If :attr:`sq_length` is *NULL*, - the index is passed as is to the function. - -.. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item - - This function is used by :c:func:`PySequence_SetItem` and has the same - signature. It is also used by :c:func:`PyObject_SetItem` and - :c:func:`PyObject_DelItem`, after trying the item assignment and deletion - via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot. - This slot may be left to *NULL* if the object does not support - item assignment and deletion. - -.. c:member:: objobjproc PySequenceMethods.sq_contains - - This function may be used by :c:func:`PySequence_Contains` and has the same - signature. This slot may be left to *NULL*, in this case - :c:func:`!PySequence_Contains` simply traverses the sequence until it - finds a match. - -.. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat - - This function is used by :c:func:`PySequence_InPlaceConcat` and has the same - signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat` - will fall back to :c:func:`PySequence_Concat`. It is also used by the - augmented assignment ``+=``, after trying numeric inplace addition - via the :c:member:`~PyNumberMethods.nb_inplace_add` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat - - This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same - signature. It should modify its first operand, and return it. This slot - may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat` - will fall back to :c:func:`PySequence_Repeat`. It is also used by the - augmented assignment ``*=``, after trying numeric inplace multiplication - via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot. - - -.. _buffer-structs: - -Buffer Object Structures -======================== - -.. sectionauthor:: Greg J. Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - -.. c:type:: PyBufferProcs - - This structure holds pointers to the functions required by the - :ref:`Buffer protocol `. The protocol defines how - an exporter object can expose its internal data to consumer objects. - -.. c:member:: getbufferproc PyBufferProcs.bf_getbuffer - - The signature of this function is:: - - int (PyObject *exporter, Py_buffer *view, int flags); - - Handle a request to *exporter* to fill in *view* as specified by *flags*. - Except for point (3), an implementation of this function MUST take these - steps: - - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, - set :c:data:`view->obj` to *NULL* and return ``-1``. - - (2) Fill in the requested fields. - - (3) Increment an internal counter for the number of exports. - - (4) Set :c:data:`view->obj` to *exporter* and increment :c:data:`view->obj`. - - (5) Return ``0``. - - If *exporter* is part of a chain or tree of buffer providers, two main - schemes can be used: - - * Re-export: Each member of the tree acts as the exporting object and - sets :c:data:`view->obj` to a new reference to itself. - - * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:data:`view->obj` will be a new reference to the root - object. - - The individual fields of *view* are described in section - :ref:`Buffer structure `, the rules how an exporter - must react to specific requests are in section - :ref:`Buffer request types `. - - All memory pointed to in the :c:type:`Py_buffer` structure belongs to - the exporter and must remain valid until there are no consumers left. - :c:member:`~Py_buffer.format`, :c:member:`~Py_buffer.shape`, - :c:member:`~Py_buffer.strides`, :c:member:`~Py_buffer.suboffsets` - and :c:member:`~Py_buffer.internal` - are read-only for the consumer. - - :c:func:`PyBuffer_FillInfo` provides an easy way of exposing a simple - bytes buffer while dealing correctly with all request types. - - :c:func:`PyObject_GetBuffer` is the interface for the consumer that - wraps this function. - -.. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer - - The signature of this function is:: - - void (PyObject *exporter, Py_buffer *view); - - Handle a request to release the resources of the buffer. If no resources - need to be released, :c:member:`PyBufferProcs.bf_releasebuffer` may be - *NULL*. Otherwise, a standard implementation of this function will take - these optional steps: - - (1) Decrement an internal counter for the number of exports. - - (2) If the counter is ``0``, free all memory associated with *view*. - - The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep - track of buffer-specific resources. This field is guaranteed to remain - constant, while a consumer MAY pass a copy of the original buffer as the - *view* argument. - - - This function MUST NOT decrement :c:data:`view->obj`, since that is - done automatically in :c:func:`PyBuffer_Release` (this scheme is - useful for breaking reference cycles). - - - :c:func:`PyBuffer_Release` is the interface for the consumer that - wraps this function. - - -.. _async-structs: - - -Async Object Structures -======================= - -.. sectionauthor:: Yury Selivanov - -.. versionadded:: 3.5 - -.. c:type:: PyAsyncMethods - - This structure holds pointers to the functions required to implement - :term:`awaitable` and :term:`asynchronous iterator` objects. - - Here is the structure definition:: - - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } PyAsyncMethods; - -.. c:member:: unaryfunc PyAsyncMethods.am_await - - The signature of this function is:: - - PyObject *am_await(PyObject *self) - - The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must - return ``1`` for it. - - This slot may be set to *NULL* if an object is not an :term:`awaitable`. - -.. c:member:: unaryfunc PyAsyncMethods.am_aiter - - The signature of this function is:: - - PyObject *am_aiter(PyObject *self) - - Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - - This slot may be set to *NULL* if an object does not implement - asynchronous iteration protocol. - -.. c:member:: unaryfunc PyAsyncMethods.am_anext - - The signature of this function is:: - - PyObject *am_anext(PyObject *self) - - Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL*. diff --git a/third_party/python/Doc/c-api/unicode.rst b/third_party/python/Doc/c-api/unicode.rst deleted file mode 100644 index 55d5b070f..000000000 --- a/third_party/python/Doc/c-api/unicode.rst +++ /dev/null @@ -1,1701 +0,0 @@ -.. highlightlang:: c - -.. _unicodeobjects: - -Unicode Objects and Codecs --------------------------- - -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Georg Brandl - -Unicode Objects -^^^^^^^^^^^^^^^ - -Since the implementation of :pep:`393` in Python 3.3, Unicode objects internally -use a variety of representations, in order to allow handling the complete range -of Unicode characters while staying memory efficient. There are special cases -for strings where all code points are below 128, 256, or 65536; otherwise, code -points must be below 1114112 (which is the full Unicode range). - -:c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached -in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated -and inefficient; it should be avoided in performance- or memory-sensitive -situations. - -Due to the transition between the old APIs and the new APIs, unicode objects -can internally be in two states depending on how they were created: - -* "canonical" unicode objects are all objects created by a non-deprecated - unicode API. They use the most efficient representation allowed by the - implementation. - -* "legacy" unicode objects have been created through one of the deprecated - APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the - :c:type:`Py_UNICODE*` representation; you will have to call - :c:func:`PyUnicode_READY` on them before calling any other API. - - -Unicode Type -"""""""""""" - -These are the basic Unicode object types used for the Unicode implementation in -Python: - -.. c:type:: Py_UCS4 - Py_UCS2 - Py_UCS1 - - These types are typedefs for unsigned integer types wide enough to contain - characters of 32 bits, 16 bits and 8 bits, respectively. When dealing with - single Unicode characters, use :c:type:`Py_UCS4`. - - .. versionadded:: 3.3 - - -.. c:type:: Py_UNICODE - - This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type - depending on the platform. - - .. versionchanged:: 3.3 - In previous versions, this was a 16-bit type or a 32-bit type depending on - whether you selected a "narrow" or "wide" Unicode version of Python at - build time. - - -.. c:type:: PyASCIIObject - PyCompactUnicodeObject - PyUnicodeObject - - These subtypes of :c:type:`PyObject` represent a Python Unicode object. In - almost all cases, they shouldn't be used directly, since all API functions - that deal with Unicode objects take and return :c:type:`PyObject` pointers. - - .. versionadded:: 3.3 - - -.. c:var:: PyTypeObject PyUnicode_Type - - This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It - is exposed to Python code as ``str``. - - -The following APIs are really C macros and can be used to do fast checks and to -access internal read-only data of Unicode objects: - -.. c:function:: int PyUnicode_Check(PyObject *o) - - Return true if the object *o* is a Unicode object or an instance of a Unicode - subtype. - - -.. c:function:: int PyUnicode_CheckExact(PyObject *o) - - Return true if the object *o* is a Unicode object, but not an instance of a - subtype. - - -.. c:function:: int PyUnicode_READY(PyObject *o) - - Ensure the string object *o* is in the "canonical" representation. This is - required before using any of the access macros described below. - - .. XXX expand on when it is not required - - Returns ``0`` on success and ``-1`` with an exception set on failure, which in - particular happens if memory allocation fails. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) - - Return the length of the Unicode string, in code points. *o* has to be a - Unicode object in the "canonical" representation (not checked). - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) - - Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 - integer types for direct character access. No checks are performed if the - canonical representation has the correct character size; use - :c:func:`PyUnicode_KIND` to select the right macro. Make sure - :c:func:`PyUnicode_READY` has been called before accessing this. - - .. versionadded:: 3.3 - - -.. c:macro:: PyUnicode_WCHAR_KIND - PyUnicode_1BYTE_KIND - PyUnicode_2BYTE_KIND - PyUnicode_4BYTE_KIND - - Return values of the :c:func:`PyUnicode_KIND` macro. - - .. versionadded:: 3.3 - - -.. c:function:: int PyUnicode_KIND(PyObject *o) - - Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to - be a Unicode object in the "canonical" representation (not checked). - - .. XXX document "0" return value? - - .. versionadded:: 3.3 - - -.. c:function:: void* PyUnicode_DATA(PyObject *o) - - Return a void pointer to the raw unicode buffer. *o* has to be a Unicode - object in the "canonical" representation (not checked). - - .. versionadded:: 3.3 - - -.. c:function:: void PyUnicode_WRITE(int kind, void *data, Py_ssize_t index, \ - Py_UCS4 value) - - Write into a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). This macro does not do any sanity checks and is - intended for usage in loops. The caller should cache the *kind* value and - *data* pointer as obtained from other macro calls. *index* is the index in - the string (starts at 0) and *value* is the new code point value which should - be written to that location. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index) - - Read a code point from a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). No checks or ready calls are performed. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) - - Read a character from a Unicode object *o*, which must be in the "canonical" - representation. This is less efficient than :c:func:`PyUnicode_READ` if you - do multiple consecutive reads. - - .. versionadded:: 3.3 - - -.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o) - - Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is - always an approximation but more efficient than iterating over the string. - - .. versionadded:: 3.3 - - -.. c:function:: int PyUnicode_ClearFreeList() - - Clear the free list. Return the total number of freed items. - - -.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation, in - code units (this includes surrogate pairs as 2 units). *o* has to be a - Unicode object (not checked). - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_GET_LENGTH`. - - -.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation in - bytes. *o* has to be a Unicode object (not checked). - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_GET_LENGTH`. - - -.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o) - const char* PyUnicode_AS_DATA(PyObject *o) - - Return a pointer to a :c:type:`Py_UNICODE` representation of the object. The - returned buffer is always terminated with an extra null code point. It - may also contain embedded null code points, which would cause the string - to be truncated when used in most C functions. The ``AS_DATA`` form - casts the pointer to :c:type:`const char *`. The *o* argument has to be - a Unicode object (not checked). - - .. versionchanged:: 3.3 - This macro is now inefficient -- because in many cases the - :c:type:`Py_UNICODE` representation does not exist and needs to be created - -- and can fail (return *NULL* with an exception set). Try to port the - code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use - :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style Unicode API, please migrate to using the - :c:func:`PyUnicode_nBYTE_DATA` family of macros. - - -Unicode Character Properties -"""""""""""""""""""""""""""" - -Unicode provides many different character properties. The most often needed ones -are available through these macros which are mapped to C functions depending on -the Python configuration. - - -.. c:function:: int Py_UNICODE_ISSPACE(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a whitespace character. - - -.. c:function:: int Py_UNICODE_ISLOWER(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a lowercase character. - - -.. c:function:: int Py_UNICODE_ISUPPER(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is an uppercase character. - - -.. c:function:: int Py_UNICODE_ISTITLE(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a titlecase character. - - -.. c:function:: int Py_UNICODE_ISLINEBREAK(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a linebreak character. - - -.. c:function:: int Py_UNICODE_ISDECIMAL(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a decimal character. - - -.. c:function:: int Py_UNICODE_ISDIGIT(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a digit character. - - -.. c:function:: int Py_UNICODE_ISNUMERIC(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a numeric character. - - -.. c:function:: int Py_UNICODE_ISALPHA(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is an alphabetic character. - - -.. c:function:: int Py_UNICODE_ISALNUM(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is an alphanumeric character. - - -.. c:function:: int Py_UNICODE_ISPRINTABLE(Py_UNICODE ch) - - Return ``1`` or ``0`` depending on whether *ch* is a printable character. - Nonprintable characters are those characters defined in the Unicode character - database as "Other" or "Separator", excepting the ASCII space (0x20) which is - considered printable. (Note that printable characters in this context are - those which should not be escaped when :func:`repr` is invoked on a string. - It has no bearing on the handling of strings written to :data:`sys.stdout` or - :data:`sys.stderr`.) - - -These APIs can be used for fast direct character conversions: - - -.. c:function:: Py_UNICODE Py_UNICODE_TOLOWER(Py_UNICODE ch) - - Return the character *ch* converted to lower case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: Py_UNICODE Py_UNICODE_TOUPPER(Py_UNICODE ch) - - Return the character *ch* converted to upper case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: Py_UNICODE Py_UNICODE_TOTITLE(Py_UNICODE ch) - - Return the character *ch* converted to title case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: int Py_UNICODE_TODECIMAL(Py_UNICODE ch) - - Return the character *ch* converted to a decimal positive integer. Return - ``-1`` if this is not possible. This macro does not raise exceptions. - - -.. c:function:: int Py_UNICODE_TODIGIT(Py_UNICODE ch) - - Return the character *ch* converted to a single digit integer. Return ``-1`` if - this is not possible. This macro does not raise exceptions. - - -.. c:function:: double Py_UNICODE_TONUMERIC(Py_UNICODE ch) - - Return the character *ch* converted to a double. Return ``-1.0`` if this is not - possible. This macro does not raise exceptions. - - -These APIs can be used to work with surrogates: - -.. c:macro:: Py_UNICODE_IS_SURROGATE(ch) - - Check if *ch* is a surrogate (``0xD800 <= ch <= 0xDFFF``). - -.. c:macro:: Py_UNICODE_IS_HIGH_SURROGATE(ch) - - Check if *ch* is a high surrogate (``0xD800 <= ch <= 0xDBFF``). - -.. c:macro:: Py_UNICODE_IS_LOW_SURROGATE(ch) - - Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). - -.. c:macro:: Py_UNICODE_JOIN_SURROGATES(high, low) - - Join two surrogate characters and return a single Py_UCS4 value. - *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. - - -Creating and accessing Unicode strings -"""""""""""""""""""""""""""""""""""""" - -To create Unicode objects and access their basic sequence properties, use these -APIs: - -.. c:function:: PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) - - Create a new Unicode object. *maxchar* should be the true maximum code point - to be placed in the string. As an approximation, it can be rounded up to the - nearest value in the sequence 127, 255, 65535, 1114111. - - This is the recommended way to allocate a new Unicode object. Objects - created using this function are not resizable. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \ - Py_ssize_t size) - - Create a new Unicode object with the given *kind* (possible values are - :c:macro:`PyUnicode_1BYTE_KIND` etc., as returned by - :c:func:`PyUnicode_KIND`). The *buffer* must point to an array of *size* - units of 1, 2 or 4 bytes per character, as given by the kind. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) - - Create a Unicode object from the char buffer *u*. The bytes will be - interpreted as being UTF-8 encoded. The buffer is copied into the new - object. If the buffer is not *NULL*, the return value might be a shared - object, i.e. modification of the data is not allowed. - - If *u* is *NULL*, this function behaves like :c:func:`PyUnicode_FromUnicode` - with the buffer set to *NULL*. This usage is deprecated in favor of - :c:func:`PyUnicode_New`. - - -.. c:function:: PyObject *PyUnicode_FromString(const char *u) - - Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. - - -.. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) - - Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python unicode string and return - a string with the values formatted into it. The variable arguments must be C - types and must correspond exactly to the format characters in the *format* - ASCII-encoded string. The following format characters are allowed: - - .. % This should be exactly the same as the table in PyErr_Format. - .. % The descriptions for %zd and %zu are wrong, but the truth is complicated - .. % because not all compilers support the %z width modifier -- we fake it - .. % when necessary via interpolating PY_FORMAT_SIZE_T. - .. % Similar comments apply to the %ll width modifier and - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------------+--------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%d` | int | Exactly equivalent to | - | | | ``printf("%d")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%u` | unsigned int | Exactly equivalent to | - | | | ``printf("%u")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%ld` | long | Exactly equivalent to | - | | | ``printf("%ld")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%li` | long | Exactly equivalent to | - | | | ``printf("%li")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lu` | unsigned long | Exactly equivalent to | - | | | ``printf("%lu")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lld` | long long | Exactly equivalent to | - | | | ``printf("%lld")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%lli` | long long | Exactly equivalent to | - | | | ``printf("%lli")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%llu` | unsigned long long | Exactly equivalent to | - | | | ``printf("%llu")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zd` | Py_ssize_t | Exactly equivalent to | - | | | ``printf("%zd")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zi` | Py_ssize_t | Exactly equivalent to | - | | | ``printf("%zi")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%zu` | size_t | Exactly equivalent to | - | | | ``printf("%zu")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%i` | int | Exactly equivalent to | - | | | ``printf("%i")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%x` | int | Exactly equivalent to | - | | | ``printf("%x")``. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%s` | char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%p` | void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%U` | PyObject\* | A unicode object. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%V` | PyObject\*, char \* | A unicode object (which may be | - | | | *NULL*) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | *NULL*). | - +-------------------+---------------------+--------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+--------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+--------------------------------+ - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result string, and any extra arguments discarded. - - .. note:: - The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and - ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of - characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` - (if the ``PyObject*`` argument is not NULL). - - .. versionchanged:: 3.2 - Support for ``"%lld"`` and ``"%llu"`` added. - - .. versionchanged:: 3.3 - Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added. - - .. versionchanged:: 3.4 - Support width and precision formatter for ``"%s"``, ``"%A"``, ``"%U"``, - ``"%V"``, ``"%S"``, ``"%R"`` added. - - -.. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) - - Identical to :c:func:`PyUnicode_FromFormat` except that it takes exactly two - arguments. - - -.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \ - const char *encoding, const char *errors) - - Decode an encoded object *obj* to a Unicode object. - - :class:`bytes`, :class:`bytearray` and other - :term:`bytes-like objects ` - are decoded according to the given *encoding* and using the error handling - defined by *errors*. Both can be *NULL* to have the interface use the default - values (see :ref:`builtincodecs` for details). - - All other objects, including Unicode objects, cause a :exc:`TypeError` to be - set. - - The API returns *NULL* if there was an error. The caller is responsible for - decref'ing the returned objects. - - -.. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode) - - Return the length of the Unicode object, in code points. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_CopyCharacters(PyObject *to, \ - Py_ssize_t to_start, \ - PyObject *from, \ - Py_ssize_t from_start, \ - Py_ssize_t how_many) - - Copy characters from one Unicode object into another. This function performs - character conversion when necessary and falls back to :c:func:`memcpy` if - possible. Returns ``-1`` and sets an exception on error, otherwise returns - the number of copied characters. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \ - Py_ssize_t length, Py_UCS4 fill_char) - - Fill a string with a character: write *fill_char* into - ``unicode[start:start+length]``. - - Fail if *fill_char* is bigger than the string maximum character, or if the - string has more than 1 reference. - - Return the number of written character, or return ``-1`` and raise an - exception on error. - - .. versionadded:: 3.3 - - -.. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \ - Py_UCS4 character) - - Write a character to a string. The string must have been created through - :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable, - the string must not be shared, or have been hashed yet. - - This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object can be modified safely (i.e. that it - its reference count is one). - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) - - Read a character from a string. This function checks that *unicode* is a - Unicode object and the index is not out of bounds, in contrast to the macro - version :c:func:`PyUnicode_READ_CHAR`. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ - Py_ssize_t end) - - Return a substring of *str*, from character index *start* (included) to - character index *end* (excluded). Negative indices are not supported. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ - Py_ssize_t buflen, int copy_null) - - Copy the string *u* into a UCS4 buffer, including a null character, if - *copy_null* is set. Returns *NULL* and sets an exception on error (in - particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) - - Copy the string *u* into a new UCS4 buffer that is allocated using - :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a - :exc:`MemoryError` set. The returned buffer always has an extra - null code point appended. - - .. versionadded:: 3.3 - - -Deprecated Py_UNICODE APIs -"""""""""""""""""""""""""" - -.. deprecated-removed:: 3.3 4.0 - -These API functions are deprecated with the implementation of :pep:`393`. -Extension modules can continue using them, as they will not be removed in Python -3.x, but need to be aware that their use can now cause performance and memory hits. - - -.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) - - Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u* - may be *NULL* which causes the contents to be undefined. It is the user's - responsibility to fill in the needed data. The buffer is copied into the new - object. - - If the buffer is not *NULL*, the return value might be a shared object. - Therefore, modification of the resulting Unicode object is only allowed when - *u* is *NULL*. - - If the buffer is *NULL*, :c:func:`PyUnicode_READY` must be called once the - string content has been filled before using any of the access macros such as - :c:func:`PyUnicode_KIND`. - - Please migrate to using :c:func:`PyUnicode_FromKindAndData`, - :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_New`. - - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) - - Return a read-only pointer to the Unicode object's internal - :c:type:`Py_UNICODE` buffer, or *NULL* on error. This will create the - :c:type:`Py_UNICODE*` representation of the object if it is not yet - available. The buffer is always terminated with an extra null code point. - Note that the resulting :c:type:`Py_UNICODE` string may also contain - embedded null code points, which would cause the string to be truncated when - used in most C functions. - - Please migrate to using :c:func:`PyUnicode_AsUCS4`, - :c:func:`PyUnicode_AsWideChar`, :c:func:`PyUnicode_ReadChar` or similar new - APIs. - - -.. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size) - - Create a Unicode object by replacing all decimal digits in - :c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9 - according to their decimal value. Return *NULL* if an exception occurs. - - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) - - Like :c:func:`PyUnicode_AsUnicode`, but also saves the :c:func:`Py_UNICODE` - array length (excluding the extra null terminator) in *size*. - Note that the resulting :c:type:`Py_UNICODE*` string - may contain embedded null code points, which would cause the string to be - truncated when used in most C functions. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode) - - Create a copy of a Unicode string ending with a null code point. Return *NULL* - and raise a :exc:`MemoryError` exception on memory allocation failure, - otherwise return a new allocated buffer (use :c:func:`PyMem_Free` to free - the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may - contain embedded null code points, which would cause the string to be - truncated when used in most C functions. - - .. versionadded:: 3.2 - - Please migrate to using :c:func:`PyUnicode_AsUCS4Copy` or similar new APIs. - - -.. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation, in - code units (this includes surrogate pairs as 2 units). - - Please migrate to using :c:func:`PyUnicode_GetLength`. - - -.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj) - - Copy an instance of a Unicode subtype to a new true Unicode object if - necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. - - Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. - - -Locale Encoding -""""""""""""""" - -The current locale encoding can be used to decode text from the operating -system. - -.. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ - const char *errors) - - Decode a string from the current locale encoding. The supported - error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The decoder uses ``"strict"`` error handler if - *errors* is ``NULL``. *str* must end with a null character but - cannot contain embedded null characters. - - Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). - - .. seealso:: - - The :c:func:`Py_DecodeLocale` function. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6.5 - The function now also uses the current locale encoding for the - ``surrogateescape`` error handler. Previously, :c:func:`Py_DecodeLocale` - was used for the ``surrogateescape``, and the current locale encoding was - used for ``strict``. - - -.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors) - - Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string - length using :c:func:`strlen`. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) - - Encode a Unicode object to the current locale encoding. The - supported error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The encoder uses ``"strict"`` error handler if - *errors* is ``NULL``. Return a :class:`bytes` object. *unicode* cannot - contain embedded null characters. - - Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). - - .. seealso:: - - The :c:func:`Py_EncodeLocale` function. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6.5 - The function now also uses the current locale encoding for the - ``surrogateescape`` error handler. Previously, :c:func:`Py_EncodeLocale` - was used for the ``surrogateescape``, and the current locale encoding was - used for ``strict``. - - -File System Encoding -"""""""""""""""""""" - -To encode and decode file names and other environment strings, -:c:data:`Py_FileSystemDefaultEncoding` should be used as the encoding, and -:c:data:`Py_FileSystemDefaultEncodeErrors` should be used as the error handler -(:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during -argument parsing, the ``"O&"`` converter should be used, passing -:c:func:`PyUnicode_FSConverter` as the conversion function: - -.. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - - ParseTuple converter: encode :class:`str` objects -- obtained directly or - through the :class:`os.PathLike` interface -- to :class:`bytes` using - :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:type:`PyBytesObject*` which must be released when it is - no longer used. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the -conversion function: - -.. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - - ParseTuple converter: decode :class:`bytes` objects -- obtained either - directly or indirectly through the :class:`os.PathLike` interface -- to - :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:type:`PyUnicodeObject*` which - must be released when it is no longer used. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) - - Decode a string using :c:data:`Py_FileSystemDefaultEncoding` and the - :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to decode a string - from the current locale encoding, use - :c:func:`PyUnicode_DecodeLocaleAndSize`. - - .. seealso:: - - The :c:func:`Py_DecodeLocale` function. - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) - - Decode a null-terminated string using :c:data:`Py_FileSystemDefaultEncoding` - and the :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` if you know the string length. - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - -.. c:function:: PyObject* PyUnicode_EncodeFSDefault(PyObject *unicode) - - Encode a Unicode object to :c:data:`Py_FileSystemDefaultEncoding` with the - :c:data:`Py_FileSystemDefaultEncodeErrors` error handler, and return - :class:`bytes`. Note that the resulting :class:`bytes` object may contain - null bytes. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to encode a string - to the current locale encoding, use :c:func:`PyUnicode_EncodeLocale`. - - .. seealso:: - - The :c:func:`Py_EncodeLocale` function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - -wchar_t Support -""""""""""""""" - -:c:type:`wchar_t` support for platforms which support it: - -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) - - Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. - Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. - Return *NULL* on failure. - - -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) - - Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most - *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing - null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:type:`wchar_t*` - string may or may not be null-terminated. It is the responsibility of the caller - to make sure that the :c:type:`wchar_t*` string is null-terminated in case this is - required by the application. Also, note that the :c:type:`wchar_t*` string - might contain null characters, which would cause the string to be truncated - when used with most C functions. - - -.. c:function:: wchar_t* PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size) - - Convert the Unicode object to a wide character string. The output string - always ends with a null character. If *size* is not *NULL*, write the number - of wide characters (excluding the trailing null termination character) into - *\*size*. - - Returns a buffer allocated by :c:func:`PyMem_Alloc` (use - :c:func:`PyMem_Free` to free it) on success. On error, returns *NULL*, - *\*size* is undefined and raises a :exc:`MemoryError`. Note that the - resulting :c:type:`wchar_t` string might contain null characters, which - would cause the string to be truncated when used with most C functions. - - .. versionadded:: 3.2 - - -.. _builtincodecs: - -Built-in Codecs -^^^^^^^^^^^^^^^ - -Python provides a set of built-in codecs which are written in C for speed. All of -these codecs are directly usable via the following functions. - -Many of the following APIs take two arguments encoding and errors, and they -have the same semantics as the ones of the built-in :func:`str` string object -constructor. - -Setting encoding to *NULL* causes the default encoding to be used -which is ASCII. The file system calls should use -:c:func:`PyUnicode_FSConverter` for encoding file names. This uses the -variable :c:data:`Py_FileSystemDefaultEncoding` internally. This -variable should be treated as read-only: on some systems, it will be a -pointer to a static string, on others, it will change at run-time -(such as when the application invokes setlocale). - -Error handling is set by errors which may also be set to *NULL* meaning to use -the default handling defined for the codec. Default error handling for all -built-in codecs is "strict" (:exc:`ValueError` is raised). - -The codecs all use a similar interface. Only deviation from the following -generic ones are documented for simplicity. - - -Generic Codecs -"""""""""""""" - -These are the generic codec APIs: - - -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ - const char *encoding, const char *errors) - - Create a Unicode object by decoding *size* bytes of the encoded string *s*. - *encoding* and *errors* have the same meaning as the parameters of the same name - in the :func:`str` built-in function. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by - the codec. - - -.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, \ - const char *encoding, const char *errors) - - Encode a Unicode object and return the result as Python bytes object. - *encoding* and *errors* have the same meaning as the parameters of the same - name in the Unicode :meth:`~str.encode` method. The codec to be used is looked up - using the Python codec registry. Return *NULL* if an exception was raised by - the codec. - - -.. c:function:: PyObject* PyUnicode_Encode(const Py_UNICODE *s, Py_ssize_t size, \ - const char *encoding, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python - bytes object. *encoding* and *errors* have the same meaning as the - parameters of the same name in the Unicode :meth:`~str.encode` method. The codec - to be used is looked up using the Python codec registry. Return *NULL* if an - exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsEncodedString`. - - -UTF-8 Codecs -"""""""""""" - -These are the UTF-8 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF8`. If - *consumed* is not *NULL*, trailing incomplete UTF-8 byte sequences will not be - treated as an error. Those bytes will not be decoded and the number of bytes - that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode) - - Encode a Unicode object using UTF-8 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was - raised by the codec. - - -.. c:function:: char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) - - Return a pointer to the UTF-8 encoding of the Unicode object, and - store the size of the encoded representation (in bytes) in *size*. The - *size* argument can be *NULL*; in this case no size will be stored. The - returned buffer always has an extra null byte appended (not included in - *size*), regardless of whether there are any other null code points. - - In the case of an error, *NULL* is returned with an exception set and no - *size* is stored. - - This caches the UTF-8 representation of the string in the Unicode object, and - subsequent calls will return a pointer to the same buffer. The caller is not - responsible for deallocating the buffer. - - .. versionadded:: 3.3 - - -.. c:function:: char* PyUnicode_AsUTF8(PyObject *unicode) - - As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_EncodeUTF8(const Py_UNICODE *s, Py_ssize_t size, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* using UTF-8 and - return a Python bytes object. Return *NULL* if an exception was raised by - the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsUTF8String`, :c:func:`PyUnicode_AsUTF8AndSize` or - :c:func:`PyUnicode_AsEncodedString`. - - -UTF-32 Codecs -""""""""""""" - -These are the UTF-32 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder) - - Decode *size* bytes from a UTF-32 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error - handling. It defaults to "strict". - - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte - order:: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If ``*byteorder`` is zero, and the first four bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If ``*byteorder`` is ``-1`` or - ``1``, any byte order mark is copied to the output. - - After completion, *\*byteorder* is set to the current byte order at the end - of input data. - - If *byteorder* is *NULL*, the codec starts in native order mode. - - Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder, Py_ssize_t *consumed) - - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF32`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat - trailing incomplete UTF-32 byte sequences (such as a number of bytes not divisible - by four) as an error. Those bytes will not be decoded and the number of bytes - that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode) - - Return a Python byte string using the UTF-32 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeUTF32(const Py_UNICODE *s, Py_ssize_t size, \ - const char *errors, int byteorder) - - Return a Python bytes object holding the UTF-32 encoded value of the Unicode - data in *s*. Output is written according to the following byte order:: - - byteorder == -1: little endian - byteorder == 0: native byte order (writes a BOM mark) - byteorder == 1: big endian - - If byteorder is ``0``, the output string will always start with the Unicode BOM - mark (U+FEFF). In the other two modes, no BOM mark is prepended. - - If *Py_UNICODE_WIDE* is not defined, surrogate pairs will be output - as a single code point. - - Return *NULL* if an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsUTF32String` or :c:func:`PyUnicode_AsEncodedString`. - - -UTF-16 Codecs -""""""""""""" - -These are the UTF-16 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder) - - Decode *size* bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-*NULL*) defines the error - handling. It defaults to "strict". - - If *byteorder* is non-*NULL*, the decoder starts decoding using the given byte - order:: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If ``*byteorder`` is zero, and the first two bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If ``*byteorder`` is ``-1`` or - ``1``, any byte order mark is copied to the output (where it will result in - either a ``\ufeff`` or a ``\ufffe`` character). - - After completion, *\*byteorder* is set to the current byte order at the end - of input data. - - If *byteorder* is *NULL*, the codec starts in native order mode. - - Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder, Py_ssize_t *consumed) - - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF16`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat - trailing incomplete UTF-16 byte sequences (such as an odd number of bytes or a - split surrogate pair) as an error. Those bytes will not be decoded and the - number of bytes that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode) - - Return a Python byte string using the UTF-16 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeUTF16(const Py_UNICODE *s, Py_ssize_t size, \ - const char *errors, int byteorder) - - Return a Python bytes object holding the UTF-16 encoded value of the Unicode - data in *s*. Output is written according to the following byte order:: - - byteorder == -1: little endian - byteorder == 0: native byte order (writes a BOM mark) - byteorder == 1: big endian - - If byteorder is ``0``, the output string will always start with the Unicode BOM - mark (U+FEFF). In the other two modes, no BOM mark is prepended. - - If *Py_UNICODE_WIDE* is defined, a single :c:type:`Py_UNICODE` value may get - represented as a surrogate pair. If it is not defined, each :c:type:`Py_UNICODE` - values is interpreted as a UCS-2 character. - - Return *NULL* if an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsUTF16String` or :c:func:`PyUnicode_AsEncodedString`. - - -UTF-7 Codecs -"""""""""""" - -These are the UTF-7 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeUTF7`. If - *consumed* is not *NULL*, trailing incomplete UTF-7 base-64 sections will not - be treated as an error. Those bytes will not be decoded and the number of - bytes that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_EncodeUTF7(const Py_UNICODE *s, Py_ssize_t size, \ - int base64SetO, int base64WhiteSpace, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer of the given size using UTF-7 and - return a Python bytes object. Return *NULL* if an exception was raised by - the codec. - - If *base64SetO* is nonzero, "Set O" (punctuation that has no otherwise - special meaning) will be encoded in base-64. If *base64WhiteSpace* is - nonzero, whitespace will be encoded in base-64. Both are set to zero for the - Python "utf-7" codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsEncodedString`. - - -Unicode-Escape Codecs -""""""""""""""""""""" - -These are the "Unicode Escape" codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ - Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) - - Encode a Unicode object using Unicode-Escape and return the result as a - bytes object. Error handling is "strict". Return *NULL* if an exception was - raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and - return a bytes object. Return *NULL* if an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsUnicodeEscapeString`. - - -Raw-Unicode-Escape Codecs -""""""""""""""""""""""""" - -These are the "Raw Unicode Escape" codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ - Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) - - Encode a Unicode object using Raw-Unicode-Escape and return the result as - a bytes object. Error handling is "strict". Return *NULL* if an exception - was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, \ - Py_ssize_t size) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape - and return a bytes object. Return *NULL* if an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsRawUnicodeEscapeString` or - :c:func:`PyUnicode_AsEncodedString`. - - -Latin-1 Codecs -"""""""""""""" - -These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode -ordinals and only these are accepted by the codecs during encoding. - - -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) - - Encode a Unicode object using Latin-1 and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was - raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeLatin1(const Py_UNICODE *s, Py_ssize_t size, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Latin-1 and - return a Python bytes object. Return *NULL* if an exception was raised by - the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsLatin1String` or - :c:func:`PyUnicode_AsEncodedString`. - - -ASCII Codecs -"""""""""""" - -These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other -codes generate errors. - - -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) - - Encode a Unicode object using ASCII and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was - raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeASCII(const Py_UNICODE *s, Py_ssize_t size, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using ASCII and - return a Python bytes object. Return *NULL* if an exception was raised by - the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsASCIIString` or - :c:func:`PyUnicode_AsEncodedString`. - - -Character Map Codecs -"""""""""""""""""""" - -This codec is special in that it can be used to implement many different codecs -(and this is in fact what was done to obtain most of the standard codecs -included in the :mod:`encodings` package). The codec uses mapping to encode and -decode characters. The mapping objects provided must support the -:meth:`__getitem__` mapping interface; dictionaries and sequences work well. - -These are the mapping codec APIs: - -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ - PyObject *mapping, const char *errors) - - Create a Unicode object by decoding *size* bytes of the encoded string *s* - using the given *mapping* object. Return *NULL* if an exception was raised - by the codec. - - If *mapping* is *NULL*, Latin-1 decoding will be applied. Else - *mapping* must map bytes ordinals (integers in the range from 0 to 255) - to Unicode strings, integers (which are then interpreted as Unicode - ordinals) or ``None``. Unmapped data bytes -- ones which cause a - :exc:`LookupError`, as well as ones which get mapped to ``None``, - ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause - an error. - - -.. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - - Encode a Unicode object using the given *mapping* object and return the - result as a bytes object. Error handling is "strict". Return *NULL* if an - exception was raised by the codec. - - The *mapping* object must map Unicode ordinal integers to bytes objects, - integers in the range from 0 to 255 or ``None``. Unmapped character - ordinals (ones which cause a :exc:`LookupError`) as well as mapped to - ``None`` are treated as "undefined mapping" and cause an error. - - -.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *mapping, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given - *mapping* object and return the result as a bytes object. Return *NULL* if - an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsCharmapString` or - :c:func:`PyUnicode_AsEncodedString`. - - -The following codec API is special in that maps Unicode to Unicode. - -.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ - PyObject *mapping, const char *errors) - - Translate a Unicode object using the given *mapping* object and return the - resulting Unicode object. Return *NULL* if an exception was raised by the - codec. - - The *mapping* object must map Unicode ordinal integers to Unicode strings, - integers (which are then interpreted as Unicode ordinals) or ``None`` - (causing deletion of the character). Unmapped character ordinals (ones - which cause a :exc:`LookupError`) are left untouched and are copied as-is. - - -.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ - PyObject *mapping, const char *errors) - - Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a - character *mapping* table to it and return the resulting Unicode object. - Return *NULL* when an exception was raised by the codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API - ` - - -MBCS codecs for Windows -""""""""""""""""""""""" - -These are the MBCS codec APIs. They are currently only available on Windows and -use the Win32 MBCS converters to implement the conversions. Note that MBCS (or -DBCS) is a class of encodings, not just one. The target encoding is defined by -the user settings on the machine running the codec. - -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. - Return *NULL* if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If - *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode - trailing lead byte and the number of bytes that have been decoded will be stored - in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) - - Encode a Unicode object using MBCS and return the result as Python bytes - object. Error handling is "strict". Return *NULL* if an exception was - raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeCodePage(int code_page, PyObject *unicode, const char *errors) - - Encode the Unicode object using the specified code page and return a Python - bytes object. Return *NULL* if an exception was raised by the codec. Use - :c:data:`CP_ACP` code page to get the MBCS encoder. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_EncodeMBCS(const Py_UNICODE *s, Py_ssize_t size, const char *errors) - - Encode the :c:type:`Py_UNICODE` buffer of the given *size* using MBCS and return - a Python bytes object. Return *NULL* if an exception was raised by the - codec. - - .. deprecated-removed:: 3.3 4.0 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsMBCSString`, :c:func:`PyUnicode_EncodeCodePage` or - :c:func:`PyUnicode_AsEncodedString`. - - -Methods & Slots -""""""""""""""" - - -.. _unicodemethodsandslots: - -Methods and Slot Functions -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following APIs are capable of handling Unicode objects and strings on input -(we refer to them as strings in the descriptions) and return Unicode objects or -integers as appropriate. - -They all return *NULL* or ``-1`` if an exception occurs. - - -.. c:function:: PyObject* PyUnicode_Concat(PyObject *left, PyObject *right) - - Concat two strings giving a new Unicode string. - - -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) - - Split a string giving a list of Unicode strings. If *sep* is *NULL*, splitting - will be done at all whitespace substrings. Otherwise, splits occur at the given - separator. At most *maxsplit* splits will be done. If negative, no limit is - set. Separators are not included in the resulting list. - - -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) - - Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the Line break - characters are not included in the resulting strings. - - -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \ - const char *errors) - - Translate a string by applying a character mapping table to it and return the - resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode ordinal integers - or ``None`` (causing deletion of the character). - - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. - - *errors* has the usual meaning for codecs. It may be *NULL* which indicates to - use the default error handling. - - -.. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) - - Join a sequence of strings using the given *separator* and return the resulting - Unicode string. - - -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end - (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), - ``0`` otherwise. Return ``-1`` if an error occurred. - - -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return the first position of *substr* in ``str[start:end]`` using the given - *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a - backward search). The return value is the index of the first match; a value of - ``-1`` indicates that no match was found, and ``-2`` indicates that an error - occurred and an exception has been set. - - -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return the first position of the character *ch* in ``str[start:end]`` using - the given *direction* (*direction* == ``1`` means to do a forward search, - *direction* == ``-1`` a backward search). The return value is the index of the - first match; a value of ``-1`` indicates that no match was found, and ``-2`` - indicates that an error occurred and an exception has been set. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end) - - Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. - - -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ - PyObject *replstr, Py_ssize_t maxcount) - - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and - return the resulting Unicode object. *maxcount* == ``-1`` means replace all - occurrences. - - -.. c:function:: int PyUnicode_Compare(PyObject *left, PyObject *right) - - Compare two strings and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, - respectively. - - This function returns ``-1`` upon failure, so one should call - :c:func:`PyErr_Occurred` to check for errors. - - -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) - - Compare a unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less - than, equal, and greater than, respectively. It is best to pass only - ASCII-encoded strings, but the function interprets the input string as - ISO-8859-1 if it contains non-ASCII characters. - - This function does not raise exceptions. - - -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) - - Rich compare two unicode strings and return one of the following: - - * ``NULL`` in case an exception was raised - * :const:`Py_True` or :const:`Py_False` for successful comparisons - * :const:`Py_NotImplemented` in case the type combination is unknown - - Possible values for *op* are :const:`Py_GT`, :const:`Py_GE`, :const:`Py_EQ`, - :const:`Py_NE`, :const:`Py_LT`, and :const:`Py_LE`. - - -.. c:function:: PyObject* PyUnicode_Format(PyObject *format, PyObject *args) - - Return a new string object from *format* and *args*; this is analogous to - ``format % args``. - - -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) - - Check whether *element* is contained in *container* and return true or false - accordingly. - - *element* has to coerce to a one element Unicode string. ``-1`` is returned - if there was an error. - - -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) - - Intern the argument *\*string* in place. The argument must be the address of a - pointer variable pointing to a Python unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call - if and only if you owned it before the call.) - - -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) - - A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. diff --git a/third_party/python/Doc/c-api/utilities.rst b/third_party/python/Doc/c-api/utilities.rst deleted file mode 100644 index d4484fb27..000000000 --- a/third_party/python/Doc/c-api/utilities.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. highlightlang:: c - -.. _utilities: - -********* -Utilities -********* - -The functions in this chapter perform various utility tasks, ranging from -helping C code be more portable across platforms, using Python modules from C, -and parsing function arguments and constructing Python values from C values. - -.. toctree:: - - sys.rst - import.rst - marshal.rst - arg.rst - conversion.rst - reflection.rst - codec.rst diff --git a/third_party/python/Doc/c-api/veryhigh.rst b/third_party/python/Doc/c-api/veryhigh.rst deleted file mode 100644 index 535293803..000000000 --- a/third_party/python/Doc/c-api/veryhigh.rst +++ /dev/null @@ -1,390 +0,0 @@ -.. highlightlang:: c - - -.. _veryhigh: - -************************* -The Very High Level Layer -************************* - -The functions in this chapter will let you execute Python source code given in a -file or a buffer, but they will not let you interact in a more detailed way with -the interpreter. - -Several of these functions accept a start symbol from the grammar as a -parameter. The available start symbols are :const:`Py_eval_input`, -:const:`Py_file_input`, and :const:`Py_single_input`. These are described -following the functions which accept them as parameters. - -Note also that several of these functions take :c:type:`FILE\*` parameters. One -particular issue which needs to be handled carefully is that the :c:type:`FILE` -structure for different C libraries can be different and incompatible. Under -Windows (at least), it is possible for dynamically linked extensions to actually -use different libraries, so care should be taken that :c:type:`FILE\*` parameters -are only passed to these functions if it is certain that they were created by -the same library that the Python runtime is using. - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. - - -.. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. - - -.. c:function:: int PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *closeit* argument set to ``0``. - - -.. c:function:: int PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *flags* argument set to *NULL*. - - -.. c:function:: int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) - - If *fp* refers to a file associated with an interactive device (console or - terminal input or Unix pseudo-terminal), return the value of - :c:func:`PyRun_InteractiveLoop`, otherwise return the result of - :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this - function uses ``"???"`` as the filename. - - -.. c:function:: int PyRun_SimpleString(const char *command) - - This is a simplified interface to :c:func:`PyRun_SimpleStringFlags` below, - leaving the *PyCompilerFlags\** argument set to NULL. - - -.. c:function:: int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) - - Executes the Python source code from *command* in the :mod:`__main__` module - according to the *flags* argument. If :mod:`__main__` does not already exist, it - is created. Returns ``0`` on success or ``-1`` if an exception was raised. If - there was an error, there is no way to get the exception information. For the - meaning of *flags*, see below. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``-1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. - - -.. c:function:: int PyRun_SimpleFile(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *closeit* set to ``0`` and *flags* set to *NULL*. - - -.. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) - - This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *flags* set to *NULL*. - - -.. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) - - Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read - from *fp* instead of an in-memory string. *filename* should be the name of - the file, it is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is - closed before PyRun_SimpleFileExFlags returns. - - -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to *NULL*. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - filesystem encoding (:func:`sys.getfilesystemencoding`). - - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - - -.. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, - leaving *flags* set to *NULL*. - - -.. c:function:: int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - Read and execute statements from a file associated with an interactive device - until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative - number upon failure. - - -.. c:var:: int (*PyOS_InputHook)(void) - - Can be set to point to a function with the prototype - ``int func(void)``. The function will be called when Python's - interpreter prompt is about to become idle and wait for user input - from the terminal. The return value is ignored. Overriding this - hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the - Python source code. - - -.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) - - Can be set to point to a function with the prototype - ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, - overriding the default function used to read a single line of input - at the interpreter's prompt. The function is expected to output - the string *prompt* if it's not *NULL*, and then read a line of - input from the provided standard input file, returning the - resulting string. For example, The :mod:`readline` module sets - this hook to provide line-editing and tab-completion features. - - The result must be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. - - .. versionchanged:: 3.4 - The result must be allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, instead of being allocated by - :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. - - -.. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start) - - This is a simplified interface to - :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL* and *flags* set to ``0``. - - -.. c:function:: struct _node* PyParser_SimpleParseStringFlags( const char *str, int start, int flags) - - This is a simplified interface to - :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set - to *NULL*. - - -.. c:function:: struct _node* PyParser_SimpleParseStringFlagsFilename( const char *str, const char *filename, int start, int flags) - - Parse Python source code from *str* using the start token *start* according to - the *flags* argument. The result can be used to create a code object which can - be evaluated efficiently. This is useful if a code fragment must be evaluated - many times. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). - - -.. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) - - This is a simplified interface to :c:func:`PyParser_SimpleParseFileFlags` below, - leaving *flags* set to ``0``. - - -.. c:function:: struct _node* PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags) - - Similar to :c:func:`PyParser_SimpleParseStringFlagsFilename`, but the Python - source code is read from *fp* instead of an in-memory string. - - -.. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving - *flags* set to *NULL*. - - -.. c:function:: PyObject* PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) - - Execute Python source code from *str* in the context specified by the - objects *globals* and *locals* with the compiler flags specified by - *flags*. *globals* must be a dictionary; *locals* can be any object - that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. - - Returns the result of executing the code as a Python object, or *NULL* if an - exception was raised. - - -.. c:function:: PyObject* PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to *NULL*. - - -.. c:function:: PyObject* PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *flags* set to *NULL*. - - -.. c:function:: PyObject* PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0``. - - -.. c:function:: PyObject* PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) - - Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from - *fp* instead of an in-memory string. *filename* should be the name of the file, - it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). - If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` - returns. - - -.. c:function:: PyObject* Py_CompileString(const char *str, const char *filename, int start) - - This is a simplified interface to :c:func:`Py_CompileStringFlags` below, leaving - *flags* set to *NULL*. - - -.. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`Py_CompileStringExFlags` below, with - *optimize* set to ``-1``. - - -.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) - - Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :const:`Py_eval_input`, - :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by - *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns *NULL* if the code - cannot be parsed or compiled. - - The integer *optimize* specifies the optimization level of the compiler; a - value of ``-1`` selects the optimization level of the interpreter as given by - :option:`-O` options. Explicit levels are ``0`` (no optimization; - ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) - or ``2`` (docstrings are removed too). - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) - - Like :c:func:`Py_CompileStringObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). - - .. versionadded:: 3.2 - -.. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just - the code object, and global and local variables. The other arguments are - set to *NULL*. - - -.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject **args, int argcount, PyObject **kws, int kwcount, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) - - Evaluate a precompiled code object, given a particular environment for its - evaluation. This environment consists of a dictionary of global variables, - a mapping object of local variables, arrays of arguments, keywords and - defaults, a dictionary of default values for :ref:`keyword-only - ` arguments and a closure tuple of cells. - - -.. c:type:: PyFrameObject - - The C structure of the objects used to describe frame objects. The - fields of this type are subject to change at any time. - - -.. c:function:: PyObject* PyEval_EvalFrame(PyFrameObject *f) - - Evaluate an execution frame. This is a simplified interface to - :c:func:`PyEval_EvalFrameEx`, for backward compatibility. - - -.. c:function:: PyObject* PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - - This is the main, unvarnished function of Python interpretation. It is - literally 2000 lines long. The code object associated with the execution - frame *f* is executed, interpreting bytecode and executing calls as needed. - The additional *throwflag* parameter can mostly be ignored - if true, then - it causes an exception to immediately be thrown; this is used for the - :meth:`~generator.throw` methods of generator objects. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - - -.. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) - - This function changes the flags of the current evaluation frame, and returns - true on success, false on failure. - - -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - -.. c:type:: struct PyCompilerFlags - - This is the structure used to hold compiler flags. In cases where code is only - being compiled, it is passed as ``int flags``, and in cases where code is being - executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from - __future__ import`` can modify *flags*. - - Whenever ``PyCompilerFlags *flags`` is *NULL*, :attr:`cf_flags` is treated as - equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. :: - - struct PyCompilerFlags { - int cf_flags; - } - - -.. c:var:: int CO_FUTURE_DIVISION - - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. diff --git a/third_party/python/Doc/c-api/weakref.rst b/third_party/python/Doc/c-api/weakref.rst deleted file mode 100644 index 6cb3e33fe..000000000 --- a/third_party/python/Doc/c-api/weakref.rst +++ /dev/null @@ -1,69 +0,0 @@ -.. highlightlang:: c - -.. _weakrefobjects: - -Weak Reference Objects ----------------------- - -Python supports *weak references* as first-class objects. There are two -specific object types which directly implement weak references. The first is a -simple reference object, and the second acts as a proxy for the original object -as much as it can. - - -.. c:function:: int PyWeakref_Check(ob) - - Return true if *ob* is either a reference or proxy object. - - -.. c:function:: int PyWeakref_CheckRef(ob) - - Return true if *ob* is a reference object. - - -.. c:function:: int PyWeakref_CheckProxy(ob) - - Return true if *ob* is a proxy object. - - -.. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback) - - Return a weak reference object for the object *ob*. This will always return - a new reference, but is not guaranteed to create a new object; an existing - reference object may be returned. The second parameter, *callback*, can be a - callable object that receives notification when *ob* is garbage collected; it - should accept a single parameter, which will be the weak reference object - itself. *callback* may also be ``None`` or *NULL*. If *ob* is not a - weakly-referencable object, or if *callback* is not callable, ``None``, or - *NULL*, this will return *NULL* and raise :exc:`TypeError`. - - -.. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) - - Return a weak reference proxy object for the object *ob*. This will always - return a new reference, but is not guaranteed to create a new object; an - existing proxy object may be returned. The second parameter, *callback*, can - be a callable object that receives notification when *ob* is garbage - collected; it should accept a single parameter, which will be the weak - reference object itself. *callback* may also be ``None`` or *NULL*. If *ob* - is not a weakly-referencable object, or if *callback* is not callable, - ``None``, or *NULL*, this will return *NULL* and raise :exc:`TypeError`. - - -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return the referenced object from a weak reference, *ref*. If the referent is - no longer live, returns :const:`Py_None`. - - .. note:: - - This function returns a **borrowed reference** to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except if you know that it cannot be destroyed while you are still - using it. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) - - Similar to :c:func:`PyWeakref_GetObject`, but implemented as a macro that does no - error checking. diff --git a/third_party/python/Doc/conf.py b/third_party/python/Doc/conf.py deleted file mode 100644 index fab963e23..000000000 --- a/third_party/python/Doc/conf.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# Python documentation build configuration file -# -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). - -import sys, os, time -sys.path.append(os.path.abspath('tools/extensions')) - -# General configuration -# --------------------- - -extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations', 'escape4chm'] - -# General substitutions. -project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') - -# We look for the Include/patchlevel.h file in the current Python source tree -# and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -today = '' -# Else, today_fmt is used as the format for a strftime call. -today_fmt = '%B %d, %Y' - -# By default, highlight as Python 3. -highlight_language = 'python3' - -# Require Sphinx 1.2 for build. -needs_sphinx = '1.2' - -# Ignore any .rst files in the venv/ directory. -venvdir = os.getenv('VENVDIR', 'venv') -exclude_patterns = [venvdir+'/*', 'README.rst'] - -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' - -# Options for HTML output -# ----------------------- - -# Use our custom theme. -html_theme = 'pydoctheme' -html_theme_path = ['tools'] -html_theme_options = {'collapsiblesidebar': True} - -# Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' - -# Path to find HTML templates. -templates_path = ['tools/templates'] - -# Custom sidebar templates, filenames relative to this file. -html_sidebars = { - # Defaults taken from http://www.sphinx-doc.org/en/stable/config.html#confval-html_sidebars - # Removes the quick search block - '**': ['localtoc.html', 'relations.html', 'customsourcelink.html'], - 'index': ['indexsidebar.html'], -} - -# Additional templates that should be rendered to pages. -html_additional_pages = { - 'download': 'download.html', - 'index': 'indexcontent.html', -} - -# Output an OpenSearch description file. -html_use_opensearch = 'https://docs.python.org/' + version - -# Additional static files. -html_static_path = ['tools/static'] - -# Output file base name for HTML help builder. -htmlhelp_basename = 'python' + release.replace('.', '') - -# Split the index -html_split_index = True - - -# Options for LaTeX output -# ------------------------ - -latex_engine = 'xelatex' - -# Get LaTeX to handle Unicode correctly -latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' -\authoraddress{ - \sphinxstrong{Python Software Foundation}\\ - Email: \sphinxemail{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = r'Guido van Rossum\\and the Python development team' -latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), -] -# Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') - -# Documents to append as an appendix to all manuals. -latex_appendices = ['glossary', 'about', 'license', 'copyright'] - -# Options for Epub output -# ----------------------- - -epub_author = 'Python Documentation Authors' -epub_publisher = 'Python Software Foundation' - -# Options for the coverage checker -# -------------------------------- - -# The coverage checker will ignore all modules/functions/classes whose names -# match any of the following regexes (using re.match). -coverage_ignore_modules = [ - r'[T|t][k|K]', - r'Tix', - r'distutils.*', -] - -coverage_ignore_functions = [ - 'test($|_)', -] - -coverage_ignore_classes = [ -] - -# Glob patterns for C source files for C API coverage, relative to this directory. -coverage_c_path = [ - '../Include/*.h', -] - -# Regexes to find C items in the source files. -coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), -} - -# The coverage checker will ignore all C items whose names match these regexes -# (using re.match) -- the keys must be the same as in coverage_c_regexes. -coverage_ignore_c_items = { -# 'cfunction': [...] -} - - -# Options for the link checker -# ---------------------------- - -# Ignore certain URLs. -linkcheck_ignore = [r'https://bugs.python.org/(issue)?\d+', - # Ignore PEPs for now, they all have permanent redirects. - r'http://www.python.org/dev/peps/pep-\d+'] - - -# Options for extensions -# ---------------------- - -# Relative filename of the reference count data file. -refcount_file = 'data/refcounts.dat' diff --git a/third_party/python/Doc/contents.rst b/third_party/python/Doc/contents.rst deleted file mode 100644 index 8690de77b..000000000 --- a/third_party/python/Doc/contents.rst +++ /dev/null @@ -1,31 +0,0 @@ -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - Python Documentation contents -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -.. toctree:: - - whatsnew/index.rst - tutorial/index.rst - using/index.rst - reference/index.rst - library/index.rst - extending/index.rst - c-api/index.rst - distributing/index.rst - installing/index.rst - howto/index.rst - faq/index.rst - glossary.rst - - about.rst - bugs.rst - copyright.rst - license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - distutils/index.rst - install/index.rst diff --git a/third_party/python/Doc/copyright.rst b/third_party/python/Doc/copyright.rst deleted file mode 100644 index 4191c0bb6..000000000 --- a/third_party/python/Doc/copyright.rst +++ /dev/null @@ -1,19 +0,0 @@ -********* -Copyright -********* - -Python and this documentation is: - -Copyright © 2001-2021 Python Software Foundation. All rights reserved. - -Copyright © 2000 BeOpen.com. All rights reserved. - -Copyright © 1995-2000 Corporation for National Research Initiatives. All rights -reserved. - -Copyright © 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -------- - -See :ref:`history-and-license` for complete license and permissions information. - diff --git a/third_party/python/Doc/data/refcounts.dat b/third_party/python/Doc/data/refcounts.dat deleted file mode 100644 index 0ac62abc7..000000000 --- a/third_party/python/Doc/data/refcounts.dat +++ /dev/null @@ -1,1881 +0,0 @@ -# Created by Skip Montanaro <skip@mojam.com>. - -# Format: -# function ':' type ':' [param name] ':' [refcount effect] ':' [comment] -# If the param name slot is empty, that line corresponds to the function's -# return value, otherwise it's the type of the named parameter. - -# The first line of a function block gives type/refcount information for the -# function's return value. Successive lines with the same function name -# correspond to the function's parameter list and appear in the order the -# parameters appear in the function's prototype. - -# For readability, each function's lines are surrounded by a blank line. -# The blocks are sorted alphabetically by function name. - -# Refcount behavior is given for all PyObject* types: 0 (no change), +1 -# (increment) and -1 (decrement). A blank refcount field indicates the -# parameter or function value is not a PyObject* and is therefore not -# subject to reference counting. A special case for the value "null" -# (without quotes) is used for functions which return a PyObject* type but -# always return NULL. This is used by some of the PyErr_*() functions, in -# particular. - -# XXX NOTE: the 0/+1/-1 refcount information for arguments is -# confusing! Much more useful would be to indicate whether the -# function "steals" a reference to the argument or not. Take for -# example PyList_SetItem(list, i, item). This lists as a 0 change for -# both the list and the item arguments. However, in fact it steals a -# reference to the item argument! - -# The parameter names are as they appear in the API manual, not the source -# code. - -PyBool_FromLong:PyObject*::+1: -PyBool_FromLong:long:v:0: - -PyBuffer_FromObject:PyObject*::+1: -PyBuffer_FromObject:PyObject*:base:+1: -PyBuffer_FromObject:int:offset:: -PyBuffer_FromObject:int:size:: - -PyBuffer_FromReadWriteObject:PyObject*::+1: -PyBuffer_FromReadWriteObject:PyObject*:base:+1: -PyBuffer_FromReadWriteObject:int:offset:: -PyBuffer_FromReadWriteObject:int:size:: - -PyBuffer_FromMemory:PyObject*::+1: -PyBuffer_FromMemory:void*:ptr:: -PyBuffer_FromMemory:int:size:: - -PyBuffer_FromReadWriteMemory:PyObject*::+1: -PyBuffer_FromReadWriteMemory:void*:ptr:: -PyBuffer_FromReadWriteMemory:int:size:: - -PyBuffer_New:PyObject*::+1: -PyBuffer_New:int:size:: - -PyCapsule_GetContext:void *::: -PyCapsule_GetContext:PyObject*:self:0: - -PyCapsule_GetDestructor:void (*)(PyObject *)::: -PyCapsule_GetDestructor:PyObject*:self:0: - -PyCapsule_GetName:const char *::: -PyCapsule_GetName:PyObject*:self:0: - -PyCapsule_GetPointer:void*::: -PyCapsule_GetPointer:PyObject*:self:0: -PyCapsule_GetPointer:const char *:name:: - -PyCapsule_Import:void *::: -PyCapsule_Import:const char *:name:: -PyCapsule_Import:int:no_block:: - -PyCapsule_New:PyObject*::+1: -PyCapsule_New:void*:pointer:: -PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: - -PyCapsule_SetContext:int::: -PyCapsule_SetContext:PyObject*:self:0: -PyCapsule_SetContext:void *:context:: - -PyCapsule_SetDestructor:int::: -PyCapsule_SetDestructor:PyObject*:self:0: -PyCapsule_SetDestructor:void (*)(PyObject *):destructor:: - -PyCapsule_SetName:int::: -PyCapsule_SetName:PyObject*:self:0: -PyCapsule_SetName:const char *:name:: - -PyCapsule_SetPointer:int::: -PyCapsule_SetPointer:PyObject*:self:0: -PyCapsule_SetPointer:void*:pointer:: - - -PyCObject_AsVoidPtr:void*::: -PyCObject_AsVoidPtr:PyObject*:self:0: - -PyCObject_FromVoidPtr:PyObject*::+1: -PyCObject_FromVoidPtr:void*:cobj:: -PyCObject_FromVoidPtr::void (* destr)(void* ):: - -PyCObject_FromVoidPtrAndDesc:PyObject*::+1: -PyCObject_FromVoidPtrAndDesc:void*:cobj:: -PyCObject_FromVoidPtrAndDesc:void*:desc:: -PyCObject_FromVoidPtrAndDesc:void(*)(void*,void*):destr:: - -PyCObject_GetDesc:void*::: -PyCObject_GetDesc:PyObject*:self:0: - -PyCell_New:PyObject*::+1: -PyCell_New:PyObject*:ob:0: - -PyCell_GET:PyObject*::0: -PyCell_GET:PyObject*:ob:0: - -PyCell_Get:PyObject*::+1: -PyCell_Get:PyObject*:cell:0: - -PyCell_SET:void::: -PyCell_SET:PyObject*:cell:0: -PyCell_SET:PyObject*:value:0: - -PyCell_Set:int::: -PyCell_Set:PyObject*:cell:0: -PyCell_Set:PyObject*:value:0: - -PyCallIter_New:PyObject*::+1: -PyCallIter_New:PyObject*:callable:: -PyCallIter_New:PyObject*:sentinel:: - -PyCallable_Check:int::: -PyCallable_Check:PyObject*:o:0: - -PyComplex_AsCComplex:Py_complex::: -PyComplex_AsCComplex:PyObject*:op:0: - -PyComplex_Check:int::: -PyComplex_Check:PyObject*:p:0: - -PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: - -PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: - -PyComplex_ImagAsDouble:double::: -PyComplex_ImagAsDouble:PyObject*:op:0: - -PyComplex_RealAsDouble:double::: -PyComplex_RealAsDouble:PyObject*:op:0: - -PyDate_FromDate:PyObject*::+1: -PyDate_FromDate:int:year:: -PyDate_FromDate:int:month:: -PyDate_FromDate:int:day:: - -PyDate_FromTimestamp:PyObject*::+1: -PyDate_FromTimestamp:PyObject*:args:0: - -PyDateTime_FromDateAndTime:PyObject*::+1: -PyDateTime_FromDateAndTime:int:year:: -PyDateTime_FromDateAndTime:int:month:: -PyDateTime_FromDateAndTime:int:day:: -PyDateTime_FromDateAndTime:int:hour:: -PyDateTime_FromDateAndTime:int:minute:: -PyDateTime_FromDateAndTime:int:second:: -PyDateTime_FromDateAndTime:int:usecond:: - -PyDateTime_FromTimestamp:PyObject*::+1: -PyDateTime_FromTimestamp:PyObject*:args:0: - -PyDelta_FromDSU:PyObject*::+1: -PyDelta_FromDSU:int:days:: -PyDelta_FromDSU:int:seconds:: -PyDelta_FromDSU:int:useconds:: - -PyDescr_NewClassMethod:PyObject*::+1: -PyDescr_NewClassMethod:PyTypeObject*:type:: -PyDescr_NewClassMethod:PyMethodDef*:method:: - -PyDescr_NewGetSet:PyObject*::+1: -PyDescr_NewGetSet:PyTypeObject*:type:: -PyDescr_NewGetSet:PyGetSetDef*:getset:: - -PyDescr_NewMember:PyObject*::+1: -PyDescr_NewMember:PyTypeObject*:type:: -PyDescr_NewMember:PyMemberDef*:member:: - -PyDescr_NewMethod:PyObject*::+1: -PyDescr_NewMethod:PyTypeObject*:type:: -PyDescr_NewMethod:PyMethodDef*:meth:: - -PyDescr_NewWrapper:PyObject*::+1: -PyDescr_NewWrapper:PyTypeObject*:type:: -PyDescr_NewWrapper:struct wrapperbase*:base:: -PyDescr_NewWrapper:void*:wrapped:: - -PyDict_Check:int::: -PyDict_Check:PyObject*:p:0: - -PyDict_Clear:void::: -PyDict_Clear:PyObject*:p:0: - -PyDict_DelItem:int::: -PyDict_DelItem:PyObject*:p:0: -PyDict_DelItem:PyObject*:key:0: - -PyDict_DelItemString:int::: -PyDict_DelItemString:PyObject*:p:0: -PyDict_DelItemString:const char*:key:: - -PyDict_GetItem:PyObject*::0:0 -PyDict_GetItem:PyObject*:p:0: -PyDict_GetItem:PyObject*:key:0: - -PyDict_GetItemString:PyObject*::0: -PyDict_GetItemString:PyObject*:p:0: -PyDict_GetItemString:const char*:key:: - -PyDict_SetDefault:PyObject*::0: -PyDict_SetDefault:PyObject*:p:0: -PyDict_SetDefault:PyObject*:key:0:conditionally +1 if inserted into the dict -PyDict_SetDefault:PyObject*:default:0:conditionally +1 if inserted into the dict - -PyDict_Items:PyObject*::+1: -PyDict_Items:PyObject*:p:0: - -PyDict_Keys:PyObject*::+1: -PyDict_Keys:PyObject*:p:0: - -PyDict_New:PyObject*::+1: - -PyDict_Copy:PyObject*::+1: -PyDict_Copy:PyObject*:p:0: - -PyDict_Next:int::: -PyDict_Next:PyObject*:p:0: -PyDict_Next:int:ppos:: -PyDict_Next:PyObject**:pkey:0: -PyDict_Next:PyObject**:pvalue:0: - -PyDict_SetItem:int::: -PyDict_SetItem:PyObject*:p:0: -PyDict_SetItem:PyObject*:key:+1: -PyDict_SetItem:PyObject*:val:+1: - -PyDict_SetItemString:int::: -PyDict_SetItemString:PyObject*:p:0: -PyDict_SetItemString:const char*:key:: -PyDict_SetItemString:PyObject*:val:+1: - -PyDict_Size:int::: -PyDict_Size:PyObject*:p:: - -PyDict_Values:PyObject*::+1: -PyDict_Values:PyObject*:p:0: - -PyDictProxy_New:PyObject*::+1: -PyDictProxy_New:PyObject*:dict:0: - -PyErr_BadArgument:int::: - -PyErr_BadInternalCall:void::: - -PyErr_CheckSignals:int::: - -PyErr_Clear:void::: - -PyErr_ExceptionMatches:int::: -PyErr_ExceptionMatches:PyObject*:exc:0: - -PyErr_Fetch:void::: -PyErr_Fetch:PyObject**:ptype:0: -PyErr_Fetch:PyObject**:pvalue:0: -PyErr_Fetch:PyObject**:ptraceback:0: - -PyErr_GivenExceptionMatches:int::: -PyErr_GivenExceptionMatches:PyObject*:given:0: -PyErr_GivenExceptionMatches:PyObject*:exc:0: - -PyErr_NewException:PyObject*::+1: -PyErr_NewException:const char*:name:: -PyErr_NewException:PyObject*:base:0: -PyErr_NewException:PyObject*:dict:0: - -PyErr_NewExceptionWithDoc:PyObject*::+1: -PyErr_NewExceptionWithDoc:const char*:name:: -PyErr_NewExceptionWithDoc:const char*:doc:: -PyErr_NewExceptionWithDoc:PyObject*:base:0: -PyErr_NewExceptionWithDoc:PyObject*:dict:0: - -PyErr_NoMemory:PyObject*::null: - -PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? - -PyErr_Occurred:PyObject*::0: - -PyErr_Print:void::: - -PyErr_Restore:void::: -PyErr_Restore:PyObject*:type:-1: -PyErr_Restore:PyObject*:value:-1: -PyErr_Restore:PyObject*:traceback:-1: - -PyErr_SetExcFromWindowsErr:PyObject*::null: -PyErr_SetExcFromWindowsErr:PyObject*:type:0: -PyErr_SetExcFromWindowsErr:int:ierr:: - -PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null: -PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:0: -PyErr_SetExcFromWindowsErrWithFilename:int:ierr:: -PyErr_SetExcFromWindowsErrWithFilename:const char*:filename:: - -PyErr_SetFromErrno:PyObject*::null: -PyErr_SetFromErrno:PyObject*:type:0: - -PyErr_SetFromErrnoWithFilename:PyObject*::null: -PyErr_SetFromErrnoWithFilename:PyObject*:type:0: -PyErr_SetFromErrnoWithFilename:const char*:filename:: - -PyErr_SetFromWindowsErr:PyObject*::null: -PyErr_SetFromWindowsErr:int:ierr:: - -PyErr_SetFromWindowsErrWithFilename:PyObject*::null: -PyErr_SetFromWindowsErrWithFilename:int:ierr:: -PyErr_SetFromWindowsErrWithFilename:const char*:filename:: - -PyErr_SetInterrupt:void::: - -PyErr_SetNone:void::: -PyErr_SetNone:PyObject*:type:+1: - -PyErr_SetObject:void::: -PyErr_SetObject:PyObject*:type:+1: -PyErr_SetObject:PyObject*:value:+1: - -PyErr_SetString:void::: -PyErr_SetString:PyObject*:type:+1: -PyErr_SetString:const char*:message:: - -PyErr_Format:PyObject*::null: -PyErr_Format:PyObject*:exception:+1: -PyErr_Format:const char*:format:: -PyErr_Format::...:: - -PyErr_FormatV:PyObject*::null: -PyErr_FormatV:PyObject*:exception:+1: -PyErr_FormatV:const char*:format:: -PyErr_FormatV:va_list:vargs:: - -PyErr_WarnEx:int::: -PyErr_WarnEx:PyObject*:category:0: -PyErr_WarnEx:const char*:message:: -PyErr_WarnEx:Py_ssize_t:stack_level:: - -PyEval_AcquireLock:void::: - -PyEval_AcquireThread:void::: -PyEval_AcquireThread:PyThreadState*:tstate:: - -PyEval_GetBuiltins:PyObject*::0: -PyEval_GetLocals:PyObject*::0: -PyEval_GetGlobals:PyObject*::0: -PyEval_GetFrame:PyObject*::0: - -PyEval_InitThreads:void::: - -PyEval_ReleaseLock:void::: - -PyEval_ReleaseThread:void::: -PyEval_ReleaseThread:PyThreadState*:tstate:: - -PyEval_RestoreThread:void::: -PyEval_RestoreThread:PyThreadState*:tstate:: - -PyEval_SaveThread:PyThreadState*::: - -PyEval_EvalCode:PyObject*::+1: -PyEval_EvalCode:PyCodeObject*:co:0: -PyEval_EvalCode:PyObject*:globals:0: -PyEval_EvalCode:PyObject*:locals:0: - -PyException_GetTraceback:PyObject*::+1: - -PyFile_AsFile:FILE*::: -PyFile_AsFile:PyFileObject*:p:0: - -PyFile_Check:int::: -PyFile_Check:PyObject*:p:0: - -PyFile_FromFile:PyObject*::+1: -PyFile_FromFile:FILE*:fp:: -PyFile_FromFile:const char*:name:: -PyFile_FromFile:const char*:mode:: -PyFile_FromFile:int(*:close):: - -PyFile_FromFileEx:PyObject*::+1: -PyFile_FromFileEx:FILE*:fp:: -PyFile_FromFileEx:const char*:name:: -PyFile_FromFileEx:const char*:mode:: -PyFile_FromFileEx:int(*:close):: -PyFile_FromFileEx:int:buffering:: -PyFile_FromFileEx:const char*:encoding:: -PyFile_FromFileEx:const char*:newline:: - -PyFile_FromString:PyObject*::+1: -PyFile_FromString:const char*:name:: -PyFile_FromString:const char*:mode:: - -PyFile_GetLine:PyObject*::+1: -PyFile_GetLine:PyObject*:p:: -PyFile_GetLine:int:n:: - -PyFile_Name:PyObject*::0: -PyFile_Name:PyObject*:p:0: - -PyFile_SetBufSize:void::: -PyFile_SetBufSize:PyFileObject*:p:0: -PyFile_SetBufSize:int:n:: - -PyFile_SoftSpace:int::: -PyFile_SoftSpace:PyFileObject*:p:0: -PyFile_SoftSpace:int:newflag:: - -PyFile_WriteObject:int::: -PyFile_WriteObject:PyObject*:obj:0: -PyFile_WriteObject:PyFileObject*:p:0: -PyFile_WriteObject:int:flags:: - -PyFile_WriteString:int::: -PyFile_WriteString:const char*:s:: -PyFile_WriteString:PyFileObject*:p:0: -PyFile_WriteString:int:flags:: - -PyFloat_AS_DOUBLE:double::: -PyFloat_AS_DOUBLE:PyObject*:pyfloat:0: - -PyFloat_AsDouble:double::: -PyFloat_AsDouble:PyObject*:pyfloat:0: - -PyFloat_Check:int::: -PyFloat_Check:PyObject*:p:0: - -PyFloat_FromDouble:PyObject*::+1: -PyFloat_FromDouble:double:v:: - -PyFloat_FromString:PyObject*::+1: -PyFloat_FromString:PyObject*:str:0: - -PyFrozenSet_New:PyObject*::+1: -PyFrozenSet_New:PyObject*:iterable:0: - -PyFunction_GetClosure:PyObject*::0: -PyFunction_GetClosure:PyObject*:op:0: - -PyFunction_GetCode:PyObject*::0: -PyFunction_GetCode:PyObject*:op:0: - -PyFunction_GetDefaults:PyObject*::0: -PyFunction_GetDefaults:PyObject*:op:0: - -PyFunction_GetGlobals:PyObject*::0: -PyFunction_GetGlobals:PyObject*:op:0: - -PyFunction_GetModule:PyObject*::0: -PyFunction_GetModule:PyObject*:op:0: - -PyFunction_New:PyObject*::+1: -PyFunction_New:PyObject*:code:+1: -PyFunction_New:PyObject*:globals:+1: - -PyFunction_NewWithQualName:PyObject*::+1: -PyFunction_NewWithQualName:PyObject*:code:+1: -PyFunction_NewWithQualName:PyObject*:globals:+1: -PyFunction_NewWithQualName:PyObject*:qualname:+1: - -PyFunction_SetClosure:int::: -PyFunction_SetClosure:PyObject*:op:0: -PyFunction_SetClosure:PyObject*:closure:+1: - -PyFunction_SetDefaults:int::: -PyFunction_SetDefaults:PyObject*:op:0: -PyFunction_SetDefaults:PyObject*:defaults:+1: - -PyGen_New:PyObject*::+1: -PyGen_New:PyFrameObject*:frame:0: - -PyGen_NewWithQualName:PyObject*::+1: -PyGen_NewWithQualName:PyFrameObject*:frame:0: - -PyCoro_New:PyObject*::+1: -PyCoro_New:PyFrameObject*:frame:0: - -Py_InitModule:PyObject*::0: -Py_InitModule:const char*:name:: -Py_InitModule:PyMethodDef[]:methods:: - -Py_InitModule3:PyObject*::0: -Py_InitModule3:const char*:name:: -Py_InitModule3:PyMethodDef[]:methods:: -Py_InitModule3:const char*:doc:: - -Py_InitModule4:PyObject*::0: -Py_InitModule4:const char*:name:: -Py_InitModule4:PyMethodDef[]:methods:: -Py_InitModule4:const char*:doc:: -Py_InitModule4:PyObject*:self:: -Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3 - -PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules -PyImport_AddModule:const char*:name:: - -PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules -PyImport_AddModuleObject:PyObject*:name:0: - -PyImport_Cleanup:void::: - -PyImport_ExecCodeModule:PyObject*::+1: -PyImport_ExecCodeModule:const char*:name:: -PyImport_ExecCodeModule:PyObject*:co:0: - -PyImport_ExecCodeModuleEx:PyObject*::+1: -PyImport_ExecCodeModuleEx:const char*:name:: -PyImport_ExecCodeModuleEx:PyObject*:co:0: -PyImport_ExecCodeModuleEx:const char*:pathname:: - -PyImport_ExecCodeModuleObject:PyObject*::+1: -PyImport_ExecCodeModuleObject:const char*:name:: -PyImport_ExecCodeModuleObject:PyObject*:co:0: -PyImport_ExecCodeModuleObject:PyObject*:pathname:0: -PyImport_ExecCodeModuleObject:PyObject*:cpathname:0: - -PyImport_ExecCodeModuleWithPathnames:PyObject*::+1: -PyImport_ExecCodeModuleWithPathnames:const char*:name:: -PyImport_ExecCodeModuleWithPathnames:PyObject*:co:0: -PyImport_ExecCodeModuleWithPathnames:const char*:pathname:: -PyImport_ExecCodeModuleWithPathnames:const char*:cpathname:: - -PyImport_GetImporter:PyObject*::+1: -PyImport_GetImporter:PyObject*:path:0: - -PyImport_GetMagicNumber:long::: - -PyImport_GetModuleDict:PyObject*::0: - -PyImport_Import:PyObject*::+1: -PyImport_Import:PyObject*:name:0: - -PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:const char*::: - -PyImport_ImportFrozenModuleObject:int::: -PyImport_ImportFrozenModuleObject:PyObject*::+1: - -PyImport_ImportModule:PyObject*::+1: -PyImport_ImportModule:const char*:name:: - -PyImport_ImportModuleEx:PyObject*::+1: -PyImport_ImportModuleEx:const char*:name:: -PyImport_ImportModuleEx:PyObject*:globals:0:??? -PyImport_ImportModuleEx:PyObject*:locals:0:??? -PyImport_ImportModuleEx:PyObject*:fromlist:0:??? - -PyImport_ImportModuleLevel:PyObject*::+1: -PyImport_ImportModuleLevel:const char*:name:: -PyImport_ImportModuleLevel:PyObject*:globals:0:??? -PyImport_ImportModuleLevel:PyObject*:locals:0:??? -PyImport_ImportModuleLevel:PyObject*:fromlist:0:??? -PyImport_ImportModuleLevel:int:level:: - -PyImport_ImportModuleLevelObject:PyObject*::+1: -PyImport_ImportModuleLevelObject:PyObject*:name:0: -PyImport_ImportModuleLevelObject:PyObject*:globals:0:??? -PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? -PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? -PyImport_ImportModuleLevelObject:int:level:: - -PyImport_ReloadModule:PyObject*::+1: -PyImport_ReloadModule:PyObject*:m:0: - -PyInstance_New:PyObject*::+1: -PyInstance_New:PyObject*:klass:+1: -PyInstance_New:PyObject*:arg:0: -PyInstance_New:PyObject*:kw:0: - -PyInstance_NewRaw:PyObject*::+1: -PyInstance_NewRaw:PyObject*:klass:+1: -PyInstance_NewRaw:PyObject*:dict:+1: - -PyInt_AS_LONG:long::: -PyInt_AS_LONG:PyIntObject*:io:0: - -PyInt_AsLong:long::: -PyInt_AsLong:PyObject*:io:0: - -PyInt_Check:int::: -PyInt_Check:PyObject*:op:0: - -PyInt_FromLong:PyObject*::+1: -PyInt_FromLong:long:ival:: - -PyInt_FromString:PyObject*::+1: -PyInt_FromString:char*:str:0: -PyInt_FromString:char**:pend:0: -PyInt_FromString:int:base:0: - -PyInt_FromSsize_t:PyObject*::+1: -PyInt_FromSsize_t:Py_ssize_t:ival:: - -PyInt_GetMax:long::: - -PyInterpreterState_Clear:void::: -PyInterpreterState_Clear:PyInterpreterState*:interp:: - -PyInterpreterState_Delete:void::: -PyInterpreterState_Delete:PyInterpreterState*:interp:: - -PyInterpreterState_New:PyInterpreterState*::: - -PyIter_Check:int:o:0: - -PyIter_Next:PyObject*::+1: -PyIter_Next:PyObject*:o:0: - -PyList_Append:int::: -PyList_Append:PyObject*:list:0: -PyList_Append:PyObject*:item:+1: - -PyList_AsTuple:PyObject*::+1: -PyList_AsTuple:PyObject*:list:0: - -PyList_Check:int::: -PyList_Check:PyObject*:p:0: - -PyList_GET_ITEM:PyObject*::0: -PyList_GET_ITEM:PyObject*:list:0: -PyList_GET_ITEM:int:i:0: - -PyList_GET_SIZE:int::: -PyList_GET_SIZE:PyObject*:list:0: - -PyList_GetItem:PyObject*::0: -PyList_GetItem:PyObject*:list:0: -PyList_GetItem:int:index:: - -PyList_GetSlice:PyObject*::+1: -PyList_GetSlice:PyObject*:list:0: -PyList_GetSlice:int:low:: -PyList_GetSlice:int:high:: - -PyList_Insert:int::: -PyList_Insert:PyObject*:list:0: -PyList_Insert:int:index:: -PyList_Insert:PyObject*:item:+1: - -PyList_New:PyObject*::+1: -PyList_New:int:len:: - -PyList_Reverse:int::: -PyList_Reverse:PyObject*:list:0: - -PyList_SET_ITEM:void::: -PyList_SET_ITEM:PyObject*:list:0: -PyList_SET_ITEM:int:i:: -PyList_SET_ITEM:PyObject*:o:0: - -PyList_SetItem:int::: -PyList_SetItem:PyObject*:list:0: -PyList_SetItem:int:index:: -PyList_SetItem:PyObject*:item:0: - -PyList_SetSlice:int::: -PyList_SetSlice:PyObject*:list:0: -PyList_SetSlice:int:low:: -PyList_SetSlice:int:high:: -PyList_SetSlice:PyObject*:itemlist:0:but increfs its elements? - -PyList_Size:int::: -PyList_Size:PyObject*:list:0: - -PyList_Sort:int::: -PyList_Sort:PyObject*:list:0: - -PyLong_AsDouble:double::: -PyLong_AsDouble:PyObject*:pylong:0: - -PyLong_AsLong:long::: -PyLong_AsLong:PyObject*:pylong:0: - -PyLong_AsUnsignedLong:unsigned long::: -PyLong_AsUnsignedLong:PyObject*:pylong:0: - -PyLong_Check:int::: -PyLong_Check:PyObject*:p:0: - -PyLong_FromDouble:PyObject*::+1: -PyLong_FromDouble:double:v:: - -PyLong_FromLong:PyObject*::+1: -PyLong_FromLong:long:v:: - -PyLong_FromLongLong:PyObject*::+1: -PyLong_FromLongLong:long long:v:: - -PyLong_FromUnsignedLongLong:PyObject*::+1: -PyLong_FromUnsignedLongLong:unsigned long long:v:: - -PyLong_FromString:PyObject*::+1: -PyLong_FromString:const char*:str:: -PyLong_FromString:char**:pend:: -PyLong_FromString:int:base:: - -PyLong_FromUnicode:PyObject*::+1: -PyLong_FromUnicode:Py_UNICODE:u:: -PyLong_FromUnicode:int:length:: -PyLong_FromUnicode:int:base:: - -PyLong_FromUnsignedLong:PyObject*::+1: -PyLong_FromUnsignedLong:unsignedlong:v:: - -PyLong_FromVoidPtr:PyObject*::+1: -PyLong_FromVoidPtr:void*:p:: - -PyMapping_Check:int::: -PyMapping_Check:PyObject*:o:0: - -PyMapping_DelItem:int::: -PyMapping_DelItem:PyObject*:o:0: -PyMapping_DelItem:PyObject*:key:0: - -PyMapping_DelItemString:int::: -PyMapping_DelItemString:PyObject*:o:0: -PyMapping_DelItemString:const char*:key:: - -PyMapping_GetItemString:PyObject*::+1: -PyMapping_GetItemString:PyObject*:o:0: -PyMapping_GetItemString:const char*:key:: - -PyMapping_HasKey:int::: -PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: - -PyMapping_HasKeyString:int::: -PyMapping_HasKeyString:PyObject*:o:0: -PyMapping_HasKeyString:const char*:key:: - -PyMapping_Items:PyObject*::+1: -PyMapping_Items:PyObject*:o:0: - -PyMapping_Keys:PyObject*::+1: -PyMapping_Keys:PyObject*:o:0: - -PyMapping_Length:int::: -PyMapping_Length:PyObject*:o:0: - -PyMapping_SetItemString:int::: -PyMapping_SetItemString:PyObject*:o:0: -PyMapping_SetItemString:const char*:key:: -PyMapping_SetItemString:PyObject*:v:+1: - -PyMapping_Values:PyObject*::+1: -PyMapping_Values:PyObject*:o:0: - -PyMarshal_ReadLastObjectFromFile:PyObject*::+1: -PyMarshal_ReadLastObjectFromFile:FILE*:file:: - -PyMarshal_ReadObjectFromFile:PyObject*::+1: -PyMarshal_ReadObjectFromFile:FILE*:file:: - -PyMarshal_ReadObjectFromString:PyObject*::+1: -PyMarshal_ReadObjectFromString:const char*:string:: -PyMarshal_ReadObjectFromString:int:len:: - -PyMarshal_WriteObjectToString:PyObject*::+1: -PyMarshal_WriteObjectToString:PyObject*:value:0: - -PyMethod_Class:PyObject*::0: -PyMethod_Class:PyObject*:im:0: - -PyMethod_Function:PyObject*::0: -PyMethod_Function:PyObject*:im:0: - -PyMethod_GET_CLASS:PyObject*::0: -PyMethod_GET_CLASS:PyObject*:im:0: - -PyMethod_GET_FUNCTION:PyObject*::0: -PyMethod_GET_FUNCTION:PyObject*:im:0: - -PyMethod_GET_SELF:PyObject*::0: -PyMethod_GET_SELF:PyObject*:im:0: - -PyMethod_New:PyObject*::+1: -PyMethod_New:PyObject*:func:0: -PyMethod_New:PyObject*:self:0: -PyMethod_New:PyObject*:class:0: - -PyMethod_Self:PyObject*::0: -PyMethod_Self:PyObject*:im:0: - -PyModule_GetDict:PyObject*::0: -PyModule_GetDict::PyObject* module:0: - -PyModule_GetFilename:const char*::: -PyModule_GetFilename:PyObject*:module:0: - -PyModule_GetName:const char*::: -PyModule_GetName:PyObject*:module:0: - -PyModule_New:PyObject*::+1: -PyModule_New::char* name:: - -PyNumber_Absolute:PyObject*::+1: -PyNumber_Absolute:PyObject*:o:0: - -PyNumber_Add:PyObject*::+1: -PyNumber_Add:PyObject*:o1:0: -PyNumber_Add:PyObject*:o2:0: - -PyNumber_And:PyObject*::+1: -PyNumber_And:PyObject*:o1:0: -PyNumber_And:PyObject*:o2:0: - -PyNumber_Check:PyObject*:o:0: -PyNumber_Check:int::: - -PyNumber_Divide:PyObject*::+1: -PyNumber_Divide:PyObject*:o1:0: -PyNumber_Divide:PyObject*:o2:0: - -PyNumber_Divmod:PyObject*::+1: -PyNumber_Divmod:PyObject*:o1:0: -PyNumber_Divmod:PyObject*:o2:0: - -PyNumber_Float:PyObject*::+1: -PyNumber_Float:PyObject*:o:0: - -PyNumber_FloorDivide:PyObject*::+1: -PyNumber_FloorDivide:PyObject*:v:0: -PyNumber_FloorDivide:PyObject*:w:0: - -PyNumber_InPlaceAdd:PyObject*::+1: -PyNumber_InPlaceAdd:PyObject*:v:0: -PyNumber_InPlaceAdd:PyObject*:w:0: - -PyNumber_InPlaceAnd:PyObject*::+1: -PyNumber_InPlaceAnd:PyObject*:v:0: -PyNumber_InPlaceAnd:PyObject*:w:0: - -PyNumber_InPlaceDivide:PyObject*::+1: -PyNumber_InPlaceDivide:PyObject*:v:0: -PyNumber_InPlaceDivide:PyObject*:w:0: - -PyNumber_InPlaceFloorDivide:PyObject*::+1: -PyNumber_InPlaceFloorDivide:PyObject*:v:0: -PyNumber_InPlaceFloorDivide:PyObject*:w:0: - -PyNumber_InPlaceLshift:PyObject*::+1: -PyNumber_InPlaceLshift:PyObject*:v:0: -PyNumber_InPlaceLshift:PyObject*:w:0: - -PyNumber_InPlaceMultiply:PyObject*::+1: -PyNumber_InPlaceMultiply:PyObject*:v:0: -PyNumber_InPlaceMultiply:PyObject*:w:0: - -PyNumber_InPlaceOr:PyObject*::+1: -PyNumber_InPlaceOr:PyObject*:v:0: -PyNumber_InPlaceOr:PyObject*:w:0: - -PyNumber_InPlacePower:PyObject*::+1: -PyNumber_InPlacePower:PyObject*:v:0: -PyNumber_InPlacePower:PyObject*:w:0: -PyNumber_InPlacePower:PyObject*:z:0: - -PyNumber_InPlaceRemainder:PyObject*::+1: -PyNumber_InPlaceRemainder:PyObject*:v:0: -PyNumber_InPlaceRemainder:PyObject*:w:0: - -PyNumber_InPlaceRshift:PyObject*::+1: -PyNumber_InPlaceRshift:PyObject*:v:0: -PyNumber_InPlaceRshift:PyObject*:w:0: - -PyNumber_InPlaceSubtract:PyObject*::+1: -PyNumber_InPlaceSubtract:PyObject*:v:0: -PyNumber_InPlaceSubtract:PyObject*:w:0: - -PyNumber_InPlaceTrueDivide:PyObject*::+1: -PyNumber_InPlaceTrueDivide:PyObject*:v:0: -PyNumber_InPlaceTrueDivide:PyObject*:w:0: - -PyNumber_InPlaceXor:PyObject*::+1: -PyNumber_InPlaceXor:PyObject*:v:0: -PyNumber_InPlaceXor:PyObject*:w:0: - -PyNumber_Invert:PyObject*::+1: -PyNumber_Invert:PyObject*:o:0: - -PyNumber_Long:PyObject*::+1: -PyNumber_Long:PyObject*:o:0: - -PyNumber_Lshift:PyObject*::+1: -PyNumber_Lshift:PyObject*:o1:0: -PyNumber_Lshift:PyObject*:o2:0: - -PyNumber_Multiply:PyObject*::+1: -PyNumber_Multiply:PyObject*:o1:0: -PyNumber_Multiply:PyObject*:o2:0: - -PyNumber_Negative:PyObject*::+1: -PyNumber_Negative:PyObject*:o:0: - -PyNumber_Or:PyObject*::+1: -PyNumber_Or:PyObject*:o1:0: -PyNumber_Or:PyObject*:o2:0: - -PyNumber_Positive:PyObject*::+1: -PyNumber_Positive:PyObject*:o:0: - -PyNumber_Power:PyObject*::+1: -PyNumber_Power:PyObject*:o1:0: -PyNumber_Power:PyObject*:o2:0: -PyNumber_Power:PyObject*:o3:0: - -PyNumber_Remainder:PyObject*::+1: -PyNumber_Remainder:PyObject*:o1:0: -PyNumber_Remainder:PyObject*:o2:0: - -PyNumber_Rshift:PyObject*::+1: -PyNumber_Rshift:PyObject*:o1:0: -PyNumber_Rshift:PyObject*:o2:0: - -PyNumber_Subtract:PyObject*::+1: -PyNumber_Subtract:PyObject*:o1:0: -PyNumber_Subtract:PyObject*:o2:0: - -PyNumber_TrueDivide:PyObject*::+1: -PyNumber_TrueDivide:PyObject*:v:0: -PyNumber_TrueDivide:PyObject*:w:0: - -PyNumber_Xor:PyObject*::+1: -PyNumber_Xor:PyObject*:o1:0: -PyNumber_Xor:PyObject*:o2:0: - -PyObject_AsFileDescriptor:int::: -PyObject_AsFileDescriptor:PyObject*:o:0: - -PyOS_FSPath:PyObject*::+1: -PyOS_FSPath:PyObject*:path:0: - -PyObject_Call:PyObject*::+1: -PyObject_Call:PyObject*:callable_object:0: -PyObject_Call:PyObject*:args:0: -PyObject_Call:PyObject*:kw:0: - -PyObject_CallFunction:PyObject*::+1: -PyObject_CallFunction:PyObject*:callable_object:0: -PyObject_CallFunction:const char*:format:: -PyObject_CallFunction::...:: - -PyObject_CallFunctionObjArgs:PyObject*::+1: -PyObject_CallFunctionObjArgs:PyObject*:callable:0: -PyObject_CallFunctionObjArgs::...:: - -PyObject_CallMethod:PyObject*::+1: -PyObject_CallMethod:PyObject*:o:0: -PyObject_CallMethod:const char*:m:: -PyObject_CallMethod:const char*:format:: -PyObject_CallMethod::...:: - -PyObject_CallMethodObjArgs:PyObject*::+1: -PyObject_CallMethodObjArgs:PyObject*:o:0: -PyObject_CallMethodObjArgs:PyObject*:name:0: -PyObject_CallMethodObjArgs::...:: - -PyObject_CallObject:PyObject*::+1: -PyObject_CallObject:PyObject*:callable_object:0: -PyObject_CallObject:PyObject*:args:0: - -PyObject_Cmp:int::: -PyObject_Cmp:PyObject*:o1:0: -PyObject_Cmp:PyObject*:o2:0: -PyObject_Cmp:int*:result:: - -PyObject_Compare:int::: -PyObject_Compare:PyObject*:o1:0: -PyObject_Compare:PyObject*:o2:0: - -PyObject_DelAttr:int::: -PyObject_DelAttr:PyObject*:o:0: -PyObject_DelAttr:PyObject*:attr_name:0: - -PyObject_DelAttrString:int::: -PyObject_DelAttrString:PyObject*:o:0: -PyObject_DelAttrString:const char*:attr_name:: - -PyObject_DelItem:int::: -PyObject_DelItem:PyObject*:o:0: -PyObject_DelItem:PyObject*:key:0: - -PyObject_Dir:PyObject*::+1: -PyObject_Dir:PyObject*:o:0: - -PyObject_GetAttr:PyObject*::+1: -PyObject_GetAttr:PyObject*:o:0: -PyObject_GetAttr:PyObject*:attr_name:0: - -PyObject_GetAttrString:PyObject*::+1: -PyObject_GetAttrString:PyObject*:o:0: -PyObject_GetAttrString:const char*:attr_name:: - -PyObject_GetItem:PyObject*::+1: -PyObject_GetItem:PyObject*:o:0: -PyObject_GetItem:PyObject*:key:0: - -PyObject_GetIter:PyObject*::+1: -PyObject_GetIter:PyObject*:o:0: - -PyObject_HasAttr:int::: -PyObject_HasAttr:PyObject*:o:0: -PyObject_HasAttr:PyObject*:attr_name:0: - -PyObject_HasAttrString:int::: -PyObject_HasAttrString:PyObject*:o:0: -PyObject_HasAttrString:const char*:attr_name:0: - -PyObject_Hash:int::: -PyObject_Hash:PyObject*:o:0: - -PyObject_IsTrue:int::: -PyObject_IsTrue:PyObject*:o:0: - -PyObject_Init:PyObject*::0: -PyObject_Init:PyObject*:op:0: - -PyObject_InitVar:PyVarObject*::0: -PyObject_InitVar:PyVarObject*:op:0: - -PyObject_Length:int::: -PyObject_Length:PyObject*:o:0: - -PyObject_NEW:PyObject*::+1: - -PyObject_New:PyObject*::+1: - -PyObject_NEW_VAR:PyObject*::+1: - -PyObject_NewVar:PyObject*::+1: - -PyObject_Print:int::: -PyObject_Print:PyObject*:o:0: -PyObject_Print:FILE*:fp:: -PyObject_Print:int:flags:: - -PyObject_Repr:PyObject*::+1: -PyObject_Repr:PyObject*:o:0: - -PyObject_RichCompare:PyObject*::+1: -PyObject_RichCompare:PyObject*:o1:0: -PyObject_RichCompare:PyObject*:o2:0: -PyObject_RichCompare:int:opid:: - -PyObject_RichCompareBool:int::: -PyObject_RichCompareBool:PyObject*:o1:0: -PyObject_RichCompareBool:PyObject*:o2:0: -PyObject_RichCompareBool:int:opid:: - -PyObject_SetAttr:int::: -PyObject_SetAttr:PyObject*:o:0: -PyObject_SetAttr:PyObject*:attr_name:0: -PyObject_SetAttr:PyObject*:v:+1: - -PyObject_SetAttrString:int::: -PyObject_SetAttrString:PyObject*:o:0: -PyObject_SetAttrString:const char*:attr_name:: -PyObject_SetAttrString:PyObject*:v:+1: - -PyObject_SetItem:int::: -PyObject_SetItem:PyObject*:o:0: -PyObject_SetItem:PyObject*:key:0: -PyObject_SetItem:PyObject*:v:+1: - -PyObject_Str:PyObject*::+1: -PyObject_Str:PyObject*:o:0: - -PyObject_Type:PyObject*::+1: -PyObject_Type:PyObject*:o:0: - -PyObject_Unicode:PyObject*::+1: -PyObject_Unicode:PyObject*:o:0: - -PyParser_SimpleParseFile:struct _node*::: -PyParser_SimpleParseFile:FILE*:fp:: -PyParser_SimpleParseFile:const char*:filename:: -PyParser_SimpleParseFile:int:start:: - -PyParser_SimpleParseString:struct _node*::: -PyParser_SimpleParseString:const char*:str:: -PyParser_SimpleParseString:int:start:: - -PyRun_AnyFile:int::: -PyRun_AnyFile:FILE*:fp:: -PyRun_AnyFile:const char*:filename:: - -PyRun_File:PyObject*::+1:??? -- same as eval_code2() -PyRun_File:FILE*:fp:: -PyRun_File:const char*:filename:: -PyRun_File:int:start:: -PyRun_File:PyObject*:globals:0: -PyRun_File:PyObject*:locals:0: - -PyRun_FileEx:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileEx:FILE*:fp:: -PyRun_FileEx:const char*:filename:: -PyRun_FileEx:int:start:: -PyRun_FileEx:PyObject*:globals:0: -PyRun_FileEx:PyObject*:locals:0: -PyRun_FileEx:int:closeit:: - -PyRun_FileFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileFlags:FILE*:fp:: -PyRun_FileFlags:const char*:filename:: -PyRun_FileFlags:int:start:: -PyRun_FileFlags:PyObject*:globals:0: -PyRun_FileFlags:PyObject*:locals:0: -PyRun_FileFlags:PyCompilerFlags*:flags:: - -PyRun_FileExFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileExFlags:FILE*:fp:: -PyRun_FileExFlags:const char*:filename:: -PyRun_FileExFlags:int:start:: -PyRun_FileExFlags:PyObject*:globals:0: -PyRun_FileExFlags:PyObject*:locals:0: -PyRun_FileExFlags:int:closeit:: -PyRun_FileExFlags:PyCompilerFlags*:flags:: - -PyRun_InteractiveLoop:int::: -PyRun_InteractiveLoop:FILE*:fp:: -PyRun_InteractiveLoop:const char*:filename:: - -PyRun_InteractiveOne:int::: -PyRun_InteractiveOne:FILE*:fp:: -PyRun_InteractiveOne:const char*:filename:: - -PyRun_SimpleFile:int::: -PyRun_SimpleFile:FILE*:fp:: -PyRun_SimpleFile:const char*:filename:: - -PyRun_SimpleString:int::: -PyRun_SimpleString:const char*:command:: - -PyRun_String:PyObject*::+1:??? -- same as eval_code2() -PyRun_String:const char*:str:: -PyRun_String:int:start:: -PyRun_String:PyObject*:globals:0: -PyRun_String:PyObject*:locals:0: - -PyRun_StringFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_StringFlags:const char*:str:: -PyRun_StringFlags:int:start:: -PyRun_StringFlags:PyObject*:globals:0: -PyRun_StringFlags:PyObject*:locals:0: -PyRun_StringFlags:PyCompilerFlags*:flags:: - -PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: - -PySequence_Check:int::: -PySequence_Check:PyObject*:o:0: - -PySequence_Concat:PyObject*::+1: -PySequence_Concat:PyObject*:o1:0: -PySequence_Concat:PyObject*:o2:0: - -PySequence_Count:int::: -PySequence_Count:PyObject*:o:0: -PySequence_Count:PyObject*:value:0: - -PySequence_DelItem:int::: -PySequence_DelItem:PyObject*:o:0: -PySequence_DelItem:int:i:: - -PySequence_DelSlice:int::: -PySequence_DelSlice:PyObject*:o:0: -PySequence_DelSlice:int:i1:: -PySequence_DelSlice:int:i2:: - -PySequence_Fast:PyObject*::+1: -PySequence_Fast:PyObject*:v:0: -PySequence_Fast:const char*:m:: - -PySequence_Fast_GET_ITEM:PyObject*::0: -PySequence_Fast_GET_ITEM:PyObject*:o:0: -PySequence_Fast_GET_ITEM:int:i:: - -PySequence_GetItem:PyObject*::+1: -PySequence_GetItem:PyObject*:o:0: -PySequence_GetItem:int:i:: - -PySequence_GetSlice:PyObject*::+1: -PySequence_GetSlice:PyObject*:o:0: -PySequence_GetSlice:int:i1:: -PySequence_GetSlice:int:i2:: - -PySequence_In:int::: -PySequence_In:PyObject*:o:0: -PySequence_In:PyObject*:value:0: - -PySequence_Index:int::: -PySequence_Index:PyObject*:o:0: -PySequence_Index:PyObject*:value:0: - -PySequence_InPlaceConcat:PyObject*::+1: -PySequence_InPlaceConcat:PyObject*:s:0: -PySequence_InPlaceConcat:PyObject*:o:0: - -PySequence_InPlaceRepeat:PyObject*::+1: -PySequence_InPlaceRepeat:PyObject*:s:0: -PySequence_InPlaceRepeat:PyObject*:o:0: - -PySequence_ITEM:PyObject*::+1: -PySequence_ITEM:PyObject*:o:0: -PySequence_ITEM:int:i:: - -PySequence_Repeat:PyObject*::+1: -PySequence_Repeat:PyObject*:o:0: -PySequence_Repeat:int:count:: - -PySequence_SetItem:int::: -PySequence_SetItem:PyObject*:o:0: -PySequence_SetItem:int:i:: -PySequence_SetItem:PyObject*:v:+1: - -PySequence_SetSlice:int::: -PySequence_SetSlice:PyObject*:o:0: -PySequence_SetSlice:int:i1:: -PySequence_SetSlice:int:i2:: -PySequence_SetSlice:PyObject*:v:+1: - -PySequence_List:PyObject*::+1: -PySequence_List:PyObject*:o:0: - -PySequence_Tuple:PyObject*::+1: -PySequence_Tuple:PyObject*:o:0: - -PySet_Append:int::: -PySet_Append:PyObject*:set:0: -PySet_Append:PyObject*:key:+1: - -PySet_Contains:int::: -PySet_Contains:PyObject*:anyset:0: -PySet_Contains:PyObject*:key:0: - -PySet_Discard:int::: -PySet_Discard:PyObject*:set:0: -PySet_Discard:PyObject*:key:-1:no effect if key not found - -PySet_New:PyObject*::+1: -PySet_New:PyObject*:iterable:0: - -PySet_Pop:PyObject*::+1:or returns NULL and raises KeyError if set is empty -PySet_Pop:PyObject*:set:0: - -PySet_Size:int::: -PySet_Size:PyObject*:anyset:0: - -PySlice_Check:int::: -PySlice_Check:PyObject*:ob:0: - -PySlice_New:PyObject*::+1: -PySlice_New:PyObject*:start:0: -PySlice_New:PyObject*:stop:0: -PySlice_New:PyObject*:step:0: - -PyString_AS_STRING:const char*::: -PyString_AS_STRING:PyObject*:string:0: - -PyString_AsDecodedObject:PyObject*::+1: -PyString_AsDecodedObject:PyObject*:str:0: -PyString_AsDecodedObject:const char*:encoding:: -PyString_AsDecodedObject:const char*:errors:: - -PyString_AsEncodedObject:PyObject*::+1: -PyString_AsEncodedObject:PyObject*:str:0: -PyString_AsEncodedObject:const char*:encoding:: -PyString_AsEncodedObject:const char*:errors:: - -PyString_AsString:const char*::: -PyString_AsString:PyObject*:string:0: - -PyString_AsStringAndSize:int::: -PyString_AsStringAndSize:PyObject*:obj:0: -PyString_AsStringAndSize:char**:buffer:: -PyString_AsStringAndSize:int*:length:: - -PyString_Check:int::: -PyString_Check:PyObject*:o:0: - -PyString_Concat:void::: -PyString_Concat:PyObject**:string:0:??? -- replaces w/ new string or NULL -PyString_Concat:PyObject*:newpart:0: - -PyString_ConcatAndDel:void::: -PyString_ConcatAndDel:PyObject**:string:0:??? -- replaces w/ new string or NULL -PyString_ConcatAndDel:PyObject*:newpart:-1: - -PyString_Format:PyObject*::+1: -PyString_Format:PyObject*:format:0: -PyString_Format:PyObject*:args:0: - -PyString_FromString:PyObject*::+1: -PyString_FromString:const char*:v:: - -PyString_FromStringAndSize:PyObject*::+1: -PyString_FromStringAndSize:const char*:v:: -PyString_FromStringAndSize:int:len:: - -PyString_FromFormat:PyObject*::+1: -PyString_FromFormat:const char*:format:: -PyString_FromFormat::...:: - -PyString_FromFormatV:PyObject*::+1: -PyString_FromFormatV:const char*:format:: -PyString_FromFormatV:va_list:vargs:: - -PyString_GET_SIZE:int::: -PyString_GET_SIZE:PyObject*:string:0: - -PyString_InternFromString:PyObject*::+1: -PyString_InternFromString:const char*:v:: - -PyString_InternInPlace:void::: -PyString_InternInPlace:PyObject**:string:+1:??? - -PyString_Size:int::: -PyString_Size:PyObject*:string:0: - -PyString_Decode:PyObject*::+1: -PyString_Decode:const char*:s:: -PyString_Decode:int:size:: -PyString_Decode:const char*:encoding:: -PyString_Decode:const char*:errors:: - -PyString_Encode:PyObject*::+1: -PyString_Encode:const char*:s:: -PyString_Encode:int:size:: -PyString_Encode:const char*:encoding:: -PyString_Encode:const char*:errors:: - -PyString_AsEncodedString:PyObject*::+1: -PyString_AsEncodedString:PyObject*:str:: -PyString_AsEncodedString:const char*:encoding:: -PyString_AsEncodedString:const char*:errors:: - -PySys_AddWarnOption:void::: -PySys_AddWarnOption:const char*:s:: - -PySys_AddXOption:void::: -PySys_AddXOption:const wchar_t*:s:: - -PySys_GetObject:PyObject*::0: -PySys_GetObject:const char*:name:: - -PySys_GetXOptions:PyObject*::0: - -PySys_SetArgv:int::: -PySys_SetArgv:int:argc:: -PySys_SetArgv:char**:argv:: - -PySys_SetObject:int::: -PySys_SetObject:const char*:name:: -PySys_SetObject:PyObject*:v:+1: - -PySys_ResetWarnOptions:void::: - -PySys_WriteStdout:void::: -PySys_WriteStdout:const char*:format:: - -PySys_WriteStderr:void::: -PySys_WriteStderr:const char*:format:: - -PyThreadState_Clear:void::: -PyThreadState_Clear:PyThreadState*:tstate:: - -PyThreadState_Delete:void::: -PyThreadState_Delete:PyThreadState*:tstate:: - -PyThreadState_Get:PyThreadState*::: - -PyThreadState_GetDict:PyObject*::0: - -PyThreadState_New:PyThreadState*::: -PyThreadState_New:PyInterpreterState*:interp:: - -PyThreadState_Swap:PyThreadState*::: -PyThreadState_Swap:PyThreadState*:tstate:: - -PyTime_FromTime:PyObject*::+1: -PyTime_FromTime:int:hour:: -PyTime_FromTime:int:minute:: -PyTime_FromTime:int:second:: -PyTime_FromTime:int:usecond:: - -PyTuple_Check:int::: -PyTuple_Check:PyObject*:p:0: - -PyTuple_GET_ITEM:PyObject*::0: -PyTuple_GET_ITEM:PyTupleObject*:p:0: -PyTuple_GET_ITEM:int:pos:: - -PyTuple_GetItem:PyObject*::0: -PyTuple_GetItem:PyTupleObject*:p:0: -PyTuple_GetItem:int:pos:: - -PyTuple_GetSlice:PyObject*::+1: -PyTuple_GetSlice:PyTupleObject*:p:0: -PyTuple_GetSlice:int:low:: -PyTuple_GetSlice:int:high:: - -PyTuple_New:PyObject*::+1: -PyTuple_New:int:len:: - -PyTuple_Pack:PyObject*::+1: -PyTuple_Pack:int:len:: -PyTuple_Pack:PyObject*:...:0: - -PyTuple_SET_ITEM:void::: -PyTuple_SET_ITEM:PyTupleObject*:p:0: -PyTuple_SET_ITEM:int:pos:: -PyTuple_SET_ITEM:PyObject*:o:0: - -PyTuple_SetItem:int::: -PyTuple_SetItem:PyTupleObject*:p:0: -PyTuple_SetItem:int:pos:: -PyTuple_SetItem:PyObject*:o:0: - -PyTuple_Size:int::: -PyTuple_Size:PyTupleObject*:p:0: - -PyType_GenericAlloc:PyObject*::+1: -PyType_GenericAlloc:PyObject*:type:0: -PyType_GenericAlloc:int:nitems:0: - -PyType_GenericNew:PyObject*::+1: -PyType_GenericNew:PyObject*:type:0: -PyType_GenericNew:PyObject*:args:0: -PyType_GenericNew:PyObject*:kwds:0: - -PyUnicode_Check:int::: -PyUnicode_Check:PyObject*:o:0: - -PyUnicode_GET_SIZE:int::: -PyUnicode_GET_SIZE:PyObject*:o:0: - -PyUnicode_GET_DATA_SIZE:int::: -PyUnicode_GET_DATA_SIZE:PyObject*:o:0: - -PyUnicode_AS_UNICODE:Py_UNICODE*::: -PyUnicode_AS_UNICODE:PyObject*:o:0: - -PyUnicode_AS_DATA:const char*::: -PyUnicode_AS_DATA:PyObject*:o:0: - -Py_UNICODE_ISSPACE:int::: -Py_UNICODE_ISSPACE:Py_UNICODE:ch:: - -Py_UNICODE_ISLOWER:int::: -Py_UNICODE_ISLOWER:Py_UNICODE:ch:: - -Py_UNICODE_ISUPPER:int::: -Py_UNICODE_ISUPPER:Py_UNICODE:ch:: - -Py_UNICODE_ISTITLE:int::: -Py_UNICODE_ISTITLE:Py_UNICODE:ch:: - -Py_UNICODE_ISLINEBREAK:int::: -Py_UNICODE_ISLINEBREAK:Py_UNICODE:ch:: - -Py_UNICODE_ISDECIMAL:int::: -Py_UNICODE_ISDECIMAL:Py_UNICODE:ch:: - -Py_UNICODE_ISDIGIT:int::: -Py_UNICODE_ISDIGIT:Py_UNICODE:ch:: - -Py_UNICODE_ISNUMERIC:int::: -Py_UNICODE_ISNUMERIC:Py_UNICODE:ch:: - -Py_UNICODE_TOLOWER:Py_UNICODE::: -Py_UNICODE_TOLOWER:Py_UNICODE:ch:: - -Py_UNICODE_TOUPPER:Py_UNICODE::: -Py_UNICODE_TOUPPER:Py_UNICODE:ch:: - -Py_UNICODE_TOTITLE:Py_UNICODE::: -Py_UNICODE_TOTITLE:Py_UNICODE:ch:: - -Py_UNICODE_TODECIMAL:int::: -Py_UNICODE_TODECIMAL:Py_UNICODE:ch:: - -Py_UNICODE_TODIGIT:int::: -Py_UNICODE_TODIGIT:Py_UNICODE:ch:: - -Py_UNICODE_TONUMERIC:double::: -Py_UNICODE_TONUMERIC:Py_UNICODE:ch:: - -PyUnicode_FromUnicode:PyObject*::+1: -PyUnicode_FromUnicode:const Py_UNICODE*:u:: -PyUnicode_FromUnicode:int:size:: - -PyUnicode_AsUnicode:Py_UNICODE*::: -PyUnicode_AsUnicode:PyObject :*unicode:0: - -PyUnicode_GetSize:int::: -PyUnicode_GetSize:PyObject :*unicode:0: - -PyUnicode_FromObject:PyObject*::+1: -PyUnicode_FromObject:PyObject*:*obj:0: - -PyUnicode_FromEncodedObject:PyObject*::+1: -PyUnicode_FromEncodedObject:PyObject*:*obj:0: -PyUnicode_FromEncodedObject:const char*:encoding:: -PyUnicode_FromEncodedObject:const char*:errors:: - -PyUnicode_FromWideChar:PyObject*::+1: -PyUnicode_FromWideChar:const wchar_t*:w:: -PyUnicode_FromWideChar:int:size:: - -PyUnicode_AsWideChar:int::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: -PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:int:size:: - -PyUnicode_Decode:PyObject*::+1: -PyUnicode_Decode:const char*:s:: -PyUnicode_Decode:int:size:: -PyUnicode_Decode:const char*:encoding:: -PyUnicode_Decode:const char*:errors:: - -PyUnicode_DecodeUTF16Stateful:PyObject*::+1: -PyUnicode_DecodeUTF16Stateful:const char*:s:: -PyUnicode_DecodeUTF16Stateful:int:size:: -PyUnicode_DecodeUTF16Stateful:const char*:errors:: -PyUnicode_DecodeUTF16Stateful:int*:byteorder:: -PyUnicode_DecodeUTF16Stateful:int*:consumed:: - -PyUnicode_DecodeUTF8Stateful:PyObject*::+1: -PyUnicode_DecodeUTF8Stateful:const char*:s:: -PyUnicode_DecodeUTF8Stateful:int:size:: -PyUnicode_DecodeUTF8Stateful:const char*:errors:: -PyUnicode_DecodeUTF8Stateful:int*:consumed:: - -PyUnicode_Encode:PyObject*::+1: -PyUnicode_Encode:const Py_UNICODE*:s:: -PyUnicode_Encode:int:size:: -PyUnicode_Encode:const char*:encoding:: -PyUnicode_Encode:const char*:errors:: - -PyUnicode_AsEncodedString:PyObject*::+1: -PyUnicode_AsEncodedString:PyObject*:unicode:: -PyUnicode_AsEncodedString:const char*:encoding:: -PyUnicode_AsEncodedString:const char*:errors:: - -PyUnicode_DecodeUTF8:PyObject*::+1: -PyUnicode_DecodeUTF8:const char*:s:: -PyUnicode_DecodeUTF8:int:size:: -PyUnicode_DecodeUTF8:const char*:errors:: - -PyUnicode_EncodeUTF8:PyObject*::+1: -PyUnicode_EncodeUTF8:const Py_UNICODE*:s:: -PyUnicode_EncodeUTF8:int:size:: -PyUnicode_EncodeUTF8:const char*:errors:: - -PyUnicode_AsUTF8String:PyObject*::+1: -PyUnicode_AsUTF8String:PyObject*:unicode:: - -PyUnicode_DecodeUTF16:PyObject*::+1: -PyUnicode_DecodeUTF16:const char*:s:: -PyUnicode_DecodeUTF16:int:size:: -PyUnicode_DecodeUTF16:const char*:errors:: -PyUnicode_DecodeUTF16:int*:byteorder:: - -PyUnicode_EncodeUTF16:PyObject*::+1: -PyUnicode_EncodeUTF16:const Py_UNICODE*:s:: -PyUnicode_EncodeUTF16:int:size:: -PyUnicode_EncodeUTF16:const char*:errors:: -PyUnicode_EncodeUTF16:int:byteorder:: - -PyUnicode_AsUTF16String:PyObject*::+1: -PyUnicode_AsUTF16String:PyObject*:unicode:: - -PyUnicode_DecodeUnicodeEscape:PyObject*::+1: -PyUnicode_DecodeUnicodeEscape:const char*:s:: -PyUnicode_DecodeUnicodeEscape:int:size:: -PyUnicode_DecodeUnicodeEscape:const char*:errors:: - -PyUnicode_EncodeUnicodeEscape:PyObject*::+1: -PyUnicode_EncodeUnicodeEscape:const Py_UNICODE*:s:: -PyUnicode_EncodeUnicodeEscape:int:size:: -PyUnicode_EncodeUnicodeEscape:const char*:errors:: - -PyUnicode_AsUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsUnicodeEscapeString:PyObject*:unicode:: - -PyUnicode_DecodeRawUnicodeEscape:PyObject*::+1: -PyUnicode_DecodeRawUnicodeEscape:const char*:s:: -PyUnicode_DecodeRawUnicodeEscape:int:size:: -PyUnicode_DecodeRawUnicodeEscape:const char*:errors:: - -PyUnicode_EncodeRawUnicodeEscape:PyObject*::+1: -PyUnicode_EncodeRawUnicodeEscape:const Py_UNICODE*:s:: -PyUnicode_EncodeRawUnicodeEscape:int:size:: -PyUnicode_EncodeRawUnicodeEscape:const char*:errors:: - -PyUnicode_AsRawUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsRawUnicodeEscapeString:PyObject*:unicode:: - -PyUnicode_DecodeLatin1:PyObject*::+1: -PyUnicode_DecodeLatin1:const char*:s:: -PyUnicode_DecodeLatin1:int:size:: -PyUnicode_DecodeLatin1:const char*:errors:: - -PyUnicode_EncodeLatin1:PyObject*::+1: -PyUnicode_EncodeLatin1:const Py_UNICODE*:s:: -PyUnicode_EncodeLatin1:int:size:: -PyUnicode_EncodeLatin1:const char*:errors:: - -PyUnicode_AsLatin1String:PyObject*::+1: -PyUnicode_AsLatin1String:PyObject*:unicode:: - -PyUnicode_DecodeASCII:PyObject*::+1: -PyUnicode_DecodeASCII:const char*:s:: -PyUnicode_DecodeASCII:int:size:: -PyUnicode_DecodeASCII:const char*:errors:: - -PyUnicode_EncodeASCII:PyObject*::+1: -PyUnicode_EncodeASCII:const Py_UNICODE*:s:: -PyUnicode_EncodeASCII:int:size:: -PyUnicode_EncodeASCII:const char*:errors:: - -PyUnicode_AsASCIIString:PyObject*::+1: -PyUnicode_AsASCIIString:PyObject*:unicode:: - -PyUnicode_DecodeCharmap:PyObject*::+1: -PyUnicode_DecodeCharmap:const char*:s:: -PyUnicode_DecodeCharmap:int:size:: -PyUnicode_DecodeCharmap:PyObject*:mapping:0: -PyUnicode_DecodeCharmap:const char*:errors:: - -PyUnicode_EncodeCharmap:PyObject*::+1: -PyUnicode_EncodeCharmap:const Py_UNICODE*:s:: -PyUnicode_EncodeCharmap:int:size:: -PyUnicode_EncodeCharmap:PyObject*:mapping:0: -PyUnicode_EncodeCharmap:const char*:errors:: - -PyUnicode_AsCharmapString:PyObject*::+1: -PyUnicode_AsCharmapString:PyObject*:unicode:0: -PyUnicode_AsCharmapString:PyObject*:mapping:0: - -PyUnicode_TranslateCharmap:PyObject*::+1: -PyUnicode_TranslateCharmap:const Py_UNICODE*:s:: -PyUnicode_TranslateCharmap:int:size:: -PyUnicode_TranslateCharmap:PyObject*:table:0: -PyUnicode_TranslateCharmap:const char*:errors:: - -PyUnicode_DecodeMBCS:PyObject*::+1: -PyUnicode_DecodeMBCS:const char*:s:: -PyUnicode_DecodeMBCS:int:size:: -PyUnicode_DecodeMBCS:const char*:errors:: - -PyUnicode_EncodeMBCS:PyObject*::+1: -PyUnicode_EncodeMBCS:const Py_UNICODE*:s:: -PyUnicode_EncodeMBCS:int:size:: -PyUnicode_EncodeMBCS:const char*:errors:: - -PyUnicode_AsMBCSString:PyObject*::+1: -PyUnicode_AsMBCSString:PyObject*:unicode:: - -PyUnicode_Concat:PyObject*::+1: -PyUnicode_Concat:PyObject*:left:0: -PyUnicode_Concat:PyObject*:right:0: - -PyUnicode_Split:PyObject*::+1: -PyUnicode_Split:PyObject*:left:0: -PyUnicode_Split:PyObject*:right:0: -PyUnicode_Split:int:maxsplit:: - -PyUnicode_Splitlines:PyObject*::+1: -PyUnicode_Splitlines:PyObject*:s:0: -PyUnicode_Splitlines:int:maxsplit:: - -PyUnicode_Translate:PyObject*::+1: -PyUnicode_Translate:PyObject*:str:0: -PyUnicode_Translate:PyObject*:table:0: -PyUnicode_Translate:const char*:errors:: - -PyUnicode_Join:PyObject*::+1: -PyUnicode_Join:PyObject*:separator:0: -PyUnicode_Join:PyObject*:seq:0: - -PyUnicode_Tailmatch:int::: -PyUnicode_Tailmatch:PyObject*:str:0: -PyUnicode_Tailmatch:PyObject*:substr:0: -PyUnicode_Tailmatch:int:start:: -PyUnicode_Tailmatch:int:end:: -PyUnicode_Tailmatch:int:direction:: - -PyUnicode_Find:int::: -PyUnicode_Find:PyObject*:str:0: -PyUnicode_Find:PyObject*:substr:0: -PyUnicode_Find:int:start:: -PyUnicode_Find:int:end:: -PyUnicode_Find:int:direction:: - -PyUnicode_Count:int::: -PyUnicode_Count:PyObject*:str:0: -PyUnicode_Count:PyObject*:substr:0: -PyUnicode_Count:int:start:: -PyUnicode_Count:int:end:: - -PyUnicode_Replace:PyObject*::+1: -PyUnicode_Replace:PyObject*:str:0: -PyUnicode_Replace:PyObject*:substr:0: -PyUnicode_Replace:PyObject*:replstr:0: -PyUnicode_Replace:int:maxcount:: - -PyUnicode_Compare:int::: -PyUnicode_Compare:PyObject*:left:0: -PyUnicode_Compare:PyObject*:right:0: - -PyUnicode_Format:PyObject*::+1: -PyUnicode_Format:PyObject*:format:0: -PyUnicode_Format:PyObject*:args:0: - -PyUnicode_Contains:int::: -PyUnicode_Contains:PyObject*:container:0: -PyUnicode_Contains:PyObject*:element:0: - -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: - -PyWeakref_NewProxy:PyObject*::+1: -PyWeakref_NewProxy:PyObject*:ob:0: -PyWeakref_NewProxy:PyObject*:callback:0: - -PyWeakref_NewRef:PyObject*::+1: -PyWeakref_NewRef:PyObject*:ob:0: -PyWeakref_NewRef:PyObject*:callback:0: - -PyWrapper_New:PyObject*::+1: -PyWrapper_New:PyObject*:d:0: -PyWrapper_New:PyObject*:self:0: - -Py_AtExit:int::: -Py_AtExit:void (*)():func:: - -Py_BuildValue:PyObject*::+1: -Py_BuildValue:const char*:format:: - -Py_CompileString:PyObject*::+1: -Py_CompileString:const char*:str:: -Py_CompileString:const char*:filename:: -Py_CompileString:int:start:: - -Py_CompileStringFlags:PyObject*::+1: -Py_CompileStringFlags:const char*:str:: -Py_CompileStringFlags:const char*:filename:: -Py_CompileStringFlags:int:start:: -Py_CompileStringFlags:PyCompilerFlags*:flags:: - -Py_DECREF:void::: -Py_DECREF:PyObject*:o:-1: - -Py_EndInterpreter:void::: -Py_EndInterpreter:PyThreadState*:tstate:: - -Py_Exit:void::: -Py_Exit:int:status:: - -Py_FatalError:void::: -Py_FatalError:const char*:message:: - -Py_FdIsInteractive:int::: -Py_FdIsInteractive:FILE*:fp:: -Py_FdIsInteractive:const char*:filename:: - -Py_Finalize:void::: - -Py_GetBuildInfoconst:const char*::: - -Py_GetCompilerconst:const char*::: - -Py_GetCopyrightconst:const char*::: - -Py_GetExecPrefix:const char*::: - -Py_GetPath:const char*::: - -Py_GetPlatformconst:const char*::: - -Py_GetPrefix:const char*::: - -Py_GetProgramFullPath:const char*::: - -Py_GetProgramName:const char*::: - -Py_GetVersionconst:const char*::: - -Py_INCREF:void::: -Py_INCREF:PyObject*:o:+1: - -Py_Initialize:void::: - -Py_IsInitialized:int::: - -Py_NewInterpreter:PyThreadState*::: - -Py_SetProgramName:void::: -Py_SetProgramName:const char*:name:: - -Py_XDECREF:void::: -Py_XDECREF:PyObject*:o:-1:if o is not NULL - -Py_XINCREF:void::: -Py_XINCREF:PyObject*:o:+1:if o is not NULL - -_PyImport_FindExtension:PyObject*::0:??? see PyImport_AddModule -_PyImport_FindExtension:const char*::: -_PyImport_FindExtension:const char*::: - -_PyImport_Fini:void::: - -_PyImport_FixupExtension:PyObject*:::??? -_PyImport_FixupExtension:const char*::: -_PyImport_FixupExtension:const char*::: - -_PyImport_Init:void::: - -_PyObject_New:PyObject*::+1: -_PyObject_New:PyTypeObject*:type:0: - -_PyObject_NewVar:PyObject*::+1: -_PyObject_NewVar:PyTypeObject*:type:0: -_PyObject_NewVar:int:size:: - -_PyString_Resize:int::: -_PyString_Resize:PyObject**:string:+1: -_PyString_Resize:int:newsize:: - -_PyTuple_Resize:int::: -_PyTuple_Resize:PyTupleObject**:p:+1: -_PyTuple_Resize:int:new:: - -_Py_c_diff:Py_complex::: -_Py_c_diff:Py_complex:left:: -_Py_c_diff:Py_complex:right:: - -_Py_c_neg:Py_complex::: -_Py_c_neg:Py_complex:complex:: - -_Py_c_pow:Py_complex::: -_Py_c_pow:Py_complex:num:: -_Py_c_pow:Py_complex:exp:: - -_Py_c_prod:Py_complex::: -_Py_c_prod:Py_complex:left:: -_Py_c_prod:Py_complex:right:: - -_Py_c_quot:Py_complex::: -_Py_c_quot:Py_complex:dividend:: -_Py_c_quot:Py_complex:divisor:: - -_Py_c_sum:Py_complex::: -_Py_c_sum:Py_complex:left:: -_Py_c_sum:Py_complex:right:: diff --git a/third_party/python/Doc/distributing/index.rst b/third_party/python/Doc/distributing/index.rst deleted file mode 100644 index b0dccb3db..000000000 --- a/third_party/python/Doc/distributing/index.rst +++ /dev/null @@ -1,170 +0,0 @@ -.. _distributing-index: - -############################### - Distributing Python Modules -############################### - -:Email: distutils-sig@python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide <installing-index>`. - -.. note:: - - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Packaging Index <https://pypi.org>`__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - <https://www.pypa.io/>`__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub <https://github.com/pypa>`__ and - `BitBucket <https://bitbucket.org/pypa/>`__. -* :mod:`distutils` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of :mod:`distutils` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for :mod:`distutils` first - published in 2004. Its most notable addition over the unmodified - :mod:`distutils` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to :mod:`distutils` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to :mod:`distutils`/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.org - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including Mac OS X and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/en/latest/current/#packaging-tool-recommendations - -Reading the guide -================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Packaging Index`_ - -.. _Project structure: \ - https://packaging.python.org/en/latest/distributing/ -.. _Building and packaging the project: \ - https://packaging.python.org/en/latest/distributing/#packaging-your-project -.. _Uploading the project to the Python Packaging Index: \ - https://packaging.python.org/en/latest/distributing/#uploading-your-project-to-pypi - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Packaging Index to see if the name is already in use -* check popular hosting sites like GitHub, BitBucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - <https://packaging.python.org/en/latest/extensions>`__ - -.. other topics: - - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) diff --git a/third_party/python/Doc/distutils/apiref.rst b/third_party/python/Doc/distutils/apiref.rst deleted file mode 100644 index 207f43864..000000000 --- a/third_party/python/Doc/distutils/apiref.rst +++ /dev/null @@ -1,2030 +0,0 @@ -.. _api-reference: - -************* -API Reference -************* - - -:mod:`distutils.core` --- Core Distutils functionality -====================================================== - -.. module:: distutils.core - :synopsis: The core Distutils functionality - - -The :mod:`distutils.core` module is the only module that needs to be installed -to use the Distutils. It provides the :func:`setup` (which is called from the -setup script). Indirectly provides the :class:`distutils.dist.Distribution` and -:class:`distutils.cmd.Command` class. - - -.. function:: setup(arguments) - - The basic do-everything function that does most everything you could ever ask - for from a Distutils method. - - The setup function takes a large number of arguments. These are laid out in the - following table. - - .. tabularcolumns:: |l|L|L| - - +--------------------+--------------------------------+-------------------------------------------------------------+ - | argument name | value | type | - +====================+================================+=============================================================+ - | *name* | The name of the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *version* | The version number of the | a string | - | | package; see | | - | | :mod:`distutils.version` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *description* | A single line describing the | a string | - | | package | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *long_description* | Longer description of the | a string | - | | package | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *author* | The name of the package author | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *author_email* | The email address of the | a string | - | | package author | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *maintainer* | The name of the current | a string | - | | maintainer, if different from | | - | | the author. Note that if | | - | | the maintainer is provided, | | - | | distutils will use it as the | | - | | author in :file:`PKG-INFO` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *maintainer_email* | The email address of the | a string | - | | current maintainer, if | | - | | different from the author | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *url* | A URL for the package | a string | - | | (homepage) | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *download_url* | A URL to download the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *packages* | A list of Python packages that | a list of strings | - | | distutils will manipulate | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *py_modules* | A list of Python modules that | a list of strings | - | | distutils will manipulate | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *scripts* | A list of standalone script | a list of strings | - | | files to be built and | | - | | installed | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *ext_modules* | A list of Python extensions to | a list of instances of | - | | be built | :class:`distutils.core.Extension` | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI | - | | package | <https://pypi.org/classifiers>`_. | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *distclass* | the :class:`Distribution` | a subclass of | - | | class to use | :class:`distutils.core.Distribution` | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *script_name* | The name of the setup.py | a string | - | | script - defaults to | | - | | ``sys.argv[0]`` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *script_args* | Arguments to supply to the | a list of strings | - | | setup script | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *options* | default options for the setup | a dictionary | - | | script | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *license* | The license for the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *keywords* | Descriptive meta-data, see | a list of strings or a comma-separated string | - | | :pep:`314` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *platforms* | | a list of strings or a comma-separated string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *cmdclass* | A mapping of command names to | a dictionary | - | | :class:`Command` subclasses | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *data_files* | A list of data files to | a list | - | | install | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *package_dir* | A mapping of package to | a dictionary | - | | directory names | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - - - -.. function:: run_setup(script_name[, script_args=None, stop_after='run']) - - Run a setup script in a somewhat controlled environment, and return the - :class:`distutils.dist.Distribution` instance that drives things. This is - useful if you need to find out the distribution meta-data (passed as keyword - args from *script* to :func:`setup`), or the contents of the config files or - command-line. - - *script_name* is a file that will be read and run with :func:`exec`. ``sys.argv[0]`` - will be replaced with *script* for the duration of the call. *script_args* is a - list of strings; if supplied, ``sys.argv[1:]`` will be replaced by *script_args* - for the duration of the call. - - *stop_after* tells :func:`setup` when to stop processing; possible values: - - .. tabularcolumns:: |l|L| - - +---------------+---------------------------------------------+ - | value | description | - +===============+=============================================+ - | *init* | Stop after the :class:`Distribution` | - | | instance has been created and populated | - | | with the keyword arguments to :func:`setup` | - +---------------+---------------------------------------------+ - | *config* | Stop after config files have been parsed | - | | (and their data stored in the | - | | :class:`Distribution` instance) | - +---------------+---------------------------------------------+ - | *commandline* | Stop after the command-line | - | | (``sys.argv[1:]`` or *script_args*) have | - | | been parsed (and the data stored in the | - | | :class:`Distribution` instance.) | - +---------------+---------------------------------------------+ - | *run* | Stop after all commands have been run (the | - | | same as if :func:`setup` had been called | - | | in the usual way). This is the default | - | | value. | - +---------------+---------------------------------------------+ - -In addition, the :mod:`distutils.core` module exposed a number of classes that -live elsewhere. - -* :class:`~distutils.extension.Extension` from :mod:`distutils.extension` - -* :class:`~distutils.cmd.Command` from :mod:`distutils.cmd` - -* :class:`~distutils.dist.Distribution` from :mod:`distutils.dist` - -A short description of each of these follows, but see the relevant module for -the full reference. - - -.. class:: Extension - - The Extension class describes a single C or C++ extension module in a setup - script. It accepts the following keyword arguments in its constructor: - - .. tabularcolumns:: |l|L|l| - - +------------------------+--------------------------------+---------------------------+ - | argument name | value | type | - +========================+================================+===========================+ - | *name* | the full name of the | a string | - | | extension, including any | | - | | packages --- ie. *not* a | | - | | filename or pathname, but | | - | | Python dotted name | | - +------------------------+--------------------------------+---------------------------+ - | *sources* | list of source filenames, | a list of strings | - | | relative to the distribution | | - | | root (where the setup script | | - | | lives), in Unix form | | - | | (slash-separated) for | | - | | portability. | | - | | Source files may be C, C++, | | - | | SWIG (.i), platform-specific | | - | | resource files, or whatever | | - | | else is recognized by the | | - | | :command:`build_ext` command | | - | | as source for a Python | | - | | extension. | | - +------------------------+--------------------------------+---------------------------+ - | *include_dirs* | list of directories to search | a list of strings | - | | for C/C++ header files (in | | - | | Unix form for portability) | | - +------------------------+--------------------------------+---------------------------+ - | *define_macros* | list of macros to define; each | a list of tuples | - | | macro is defined using a | | - | | 2-tuple ``(name, value)``, | | - | | where *value* is | | - | | either the string to define it | | - | | to or ``None`` to define it | | - | | without a particular value | | - | | (equivalent of ``#define FOO`` | | - | | in source or :option:`!-DFOO` | | - | | on Unix C compiler command | | - | | line) | | - +------------------------+--------------------------------+---------------------------+ - | *undef_macros* | list of macros to undefine | a list of strings | - | | explicitly | | - +------------------------+--------------------------------+---------------------------+ - | *library_dirs* | list of directories to search | a list of strings | - | | for C/C++ libraries at link | | - | | time | | - +------------------------+--------------------------------+---------------------------+ - | *libraries* | list of library names (not | a list of strings | - | | filenames or paths) to link | | - | | against | | - +------------------------+--------------------------------+---------------------------+ - | *runtime_library_dirs* | list of directories to search | a list of strings | - | | for C/C++ libraries at run | | - | | time (for shared extensions, | | - | | this is when the extension is | | - | | loaded) | | - +------------------------+--------------------------------+---------------------------+ - | *extra_objects* | list of extra files to link | a list of strings | - | | with (eg. object files not | | - | | implied by 'sources', static | | - | | library that must be | | - | | explicitly specified, binary | | - | | resource files, etc.) | | - +------------------------+--------------------------------+---------------------------+ - | *extra_compile_args* | any extra platform- and | a list of strings | - | | compiler-specific information | | - | | to use when compiling the | | - | | source files in 'sources'. For | | - | | platforms and compilers where | | - | | a command line makes sense, | | - | | this is typically a list of | | - | | command-line arguments, but | | - | | for other platforms it could | | - | | be anything. | | - +------------------------+--------------------------------+---------------------------+ - | *extra_link_args* | any extra platform- and | a list of strings | - | | compiler-specific information | | - | | to use when linking object | | - | | files together to create the | | - | | extension (or to create a new | | - | | static Python interpreter). | | - | | Similar interpretation as for | | - | | 'extra_compile_args'. | | - +------------------------+--------------------------------+---------------------------+ - | *export_symbols* | list of symbols to be exported | a list of strings | - | | from a shared extension. Not | | - | | used on all platforms, and not | | - | | generally necessary for Python | | - | | extensions, which typically | | - | | export exactly one symbol: | | - | | ``init`` + extension_name. | | - +------------------------+--------------------------------+---------------------------+ - | *depends* | list of files that the | a list of strings | - | | extension depends on | | - +------------------------+--------------------------------+---------------------------+ - | *language* | extension language (i.e. | a string | - | | ``'c'``, ``'c++'``, | | - | | ``'objc'``). Will be detected | | - | | from the source extensions if | | - | | not provided. | | - +------------------------+--------------------------------+---------------------------+ - | *optional* | specifies that a build failure | a boolean | - | | in the extension should not | | - | | abort the build process, but | | - | | simply skip the extension. | | - +------------------------+--------------------------------+---------------------------+ - - -.. class:: Distribution - - A :class:`Distribution` describes how to build, install and package up a Python - software package. - - See the :func:`setup` function for a list of keyword arguments accepted by the - Distribution constructor. :func:`setup` creates a Distribution instance. - - -.. class:: Command - - A :class:`Command` class (or rather, an instance of one of its subclasses) - implement a single distutils command. - - -:mod:`distutils.ccompiler` --- CCompiler base class -=================================================== - -.. module:: distutils.ccompiler - :synopsis: Abstract CCompiler class - - -This module provides the abstract base class for the :class:`CCompiler` -classes. A :class:`CCompiler` instance can be used for all the compile and -link steps needed to build a single project. Methods are provided to set -options for the compiler --- macro definitions, include directories, link path, -libraries and the like. - -This module provides the following functions. - - -.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) - - Generate linker options for searching library directories and linking with - specific libraries. *libraries* and *library_dirs* are, respectively, lists of - library names (not filenames!) and search directories. Returns a list of - command-line options suitable for use with some compiler (depending on the two - format strings passed in). - - -.. function:: gen_preprocess_options(macros, include_dirs) - - Generate C pre-processor options (:option:`!-D`, :option:`!-U`, :option:`!-I`) as - used by at least two types of compilers: the typical Unix compiler and Visual - C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)`` - means undefine (:option:`!-U`) macro *name*, and ``(name, value)`` means define - (:option:`!-D`) macro *name* to *value*. *include_dirs* is just a list of - directory names to be added to the header file search path (:option:`!-I`). - Returns a list of command-line options suitable for either Unix compilers or - Visual C++. - - -.. function:: get_default_compiler(osname, platform) - - Determine the default compiler to use for the given platform. - - *osname* should be one of the standard Python OS names (i.e. the ones returned - by ``os.name``) and *platform* the common value returned by ``sys.platform`` for - the platform in question. - - The default values are ``os.name`` and ``sys.platform`` in case the parameters - are not given. - - -.. function:: new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0) - - Factory function to generate an instance of some CCompiler subclass for the - supplied platform/compiler combination. *plat* defaults to ``os.name`` (eg. - ``'posix'``, ``'nt'``), and *compiler* defaults to the default compiler for - that platform. Currently only ``'posix'`` and ``'nt'`` are supported, and the - default compilers are "traditional Unix interface" (:class:`UnixCCompiler` - class) and Visual C++ (:class:`MSVCCompiler` class). Note that it's perfectly - possible to ask for a Unix compiler object under Windows, and a Microsoft - compiler object under Unix---if you supply a value for *compiler*, *plat* is - ignored. - - .. % Is the posix/nt only thing still true? Mac OS X seems to work, and - .. % returns a UnixCCompiler instance. How to document this... hmm. - - -.. function:: show_compilers() - - Print list of available compilers (used by the :option:`!--help-compiler` options - to :command:`build`, :command:`build_ext`, :command:`build_clib`). - - -.. class:: CCompiler([verbose=0, dry_run=0, force=0]) - - The abstract base class :class:`CCompiler` defines the interface that must be - implemented by real compiler classes. The class also has some utility methods - used by several compiler classes. - - The basic idea behind a compiler abstraction class is that each instance can be - used for all the compile/link steps in building a single project. Thus, - attributes common to all of those compile and link steps --- include - directories, macros to define, libraries to link against, etc. --- are - attributes of the compiler instance. To allow for variability in how individual - files are treated, most of those attributes may be varied on a per-compilation - or per-link basis. - - The constructor for each subclass creates an instance of the Compiler object. - Flags are *verbose* (show verbose output), *dry_run* (don't actually execute the - steps) and *force* (rebuild everything, regardless of dependencies). All of - these flags default to ``0`` (off). Note that you probably don't want to - instantiate :class:`CCompiler` or one of its subclasses directly - use the - :func:`distutils.CCompiler.new_compiler` factory function instead. - - The following methods allow you to manually alter compiler options for the - instance of the Compiler class. - - - .. method:: CCompiler.add_include_dir(dir) - - Add *dir* to the list of directories that will be searched for header files. - The compiler is instructed to search directories in the order in which they are - supplied by successive calls to :meth:`add_include_dir`. - - - .. method:: CCompiler.set_include_dirs(dirs) - - Set the list of directories that will be searched to *dirs* (a list of strings). - Overrides any preceding calls to :meth:`add_include_dir`; subsequent calls to - :meth:`add_include_dir` add to the list passed to :meth:`set_include_dirs`. - This does not affect any list of standard include directories that the compiler - may search by default. - - - .. method:: CCompiler.add_library(libname) - - Add *libname* to the list of libraries that will be included in all links driven - by this compiler object. Note that *libname* should \*not\* be the name of a - file containing a library, but the name of the library itself: the actual - filename will be inferred by the linker, the compiler, or the compiler class - (depending on the platform). - - The linker will be instructed to link against libraries in the order they were - supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is perfectly - valid to duplicate library names; the linker will be instructed to link against - libraries as many times as they are mentioned. - - - .. method:: CCompiler.set_libraries(libnames) - - Set the list of libraries to be included in all links driven by this compiler - object to *libnames* (a list of strings). This does not affect any standard - system libraries that the linker may include by default. - - - .. method:: CCompiler.add_library_dir(dir) - - Add *dir* to the list of directories that will be searched for libraries - specified to :meth:`add_library` and :meth:`set_libraries`. The linker will be - instructed to search for libraries in the order they are supplied to - :meth:`add_library_dir` and/or :meth:`set_library_dirs`. - - - .. method:: CCompiler.set_library_dirs(dirs) - - Set the list of library search directories to *dirs* (a list of strings). This - does not affect any standard library search path that the linker may search by - default. - - - .. method:: CCompiler.add_runtime_library_dir(dir) - - Add *dir* to the list of directories that will be searched for shared libraries - at runtime. - - - .. method:: CCompiler.set_runtime_library_dirs(dirs) - - Set the list of directories to search for shared libraries at runtime to *dirs* - (a list of strings). This does not affect any standard search path that the - runtime linker may search by default. - - - .. method:: CCompiler.define_macro(name[, value=None]) - - Define a preprocessor macro for all compilations driven by this compiler object. - The optional parameter *value* should be a string; if it is not supplied, then - the macro will be defined without an explicit value and the exact outcome - depends on the compiler used. - - .. XXX true? does ANSI say anything about this? - - - .. method:: CCompiler.undefine_macro(name) - - Undefine a preprocessor macro for all compilations driven by this compiler - object. If the same macro is defined by :meth:`define_macro` and - undefined by :meth:`undefine_macro` the last call takes precedence - (including multiple redefinitions or undefinitions). If the macro is - redefined/undefined on a per-compilation basis (ie. in the call to - :meth:`compile`), then that takes precedence. - - - .. method:: CCompiler.add_link_object(object) - - Add *object* to the list of object files (or analogues, such as explicitly named - library files or the output of "resource compilers") to be included in every - link driven by this compiler object. - - - .. method:: CCompiler.set_link_objects(objects) - - Set the list of object files (or analogues) to be included in every link to - *objects*. This does not affect any standard object files that the linker may - include by default (such as system libraries). - - The following methods implement methods for autodetection of compiler options, - providing some functionality similar to GNU :program:`autoconf`. - - - .. method:: CCompiler.detect_language(sources) - - Detect the language of a given file, or list of files. Uses the instance - attributes :attr:`language_map` (a dictionary), and :attr:`language_order` (a - list) to do the job. - - - .. method:: CCompiler.find_library_file(dirs, lib[, debug=0]) - - Search the specified list of directories for a static or shared library file - *lib* and return the full path to that file. If *debug* is true, look for a - debugging version (if that makes sense on the current platform). Return - ``None`` if *lib* wasn't found in any of the specified directories. - - - .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None]) - - Return a boolean indicating whether *funcname* is supported on the current - platform. The optional arguments can be used to augment the compilation - environment by providing additional include files and paths and libraries and - paths. - - - .. method:: CCompiler.library_dir_option(dir) - - Return the compiler option to add *dir* to the list of directories searched for - libraries. - - - .. method:: CCompiler.library_option(lib) - - Return the compiler option to add *lib* to the list of libraries linked into the - shared library or executable. - - - .. method:: CCompiler.runtime_library_dir_option(dir) - - Return the compiler option to add *dir* to the list of directories searched for - runtime libraries. - - - .. method:: CCompiler.set_executables(**args) - - Define the executables (and options for them) that will be run to perform the - various stages of compilation. The exact set of executables that may be - specified here depends on the compiler class (via the 'executables' class - attribute), but most will have: - - +--------------+------------------------------------------+ - | attribute | description | - +==============+==========================================+ - | *compiler* | the C/C++ compiler | - +--------------+------------------------------------------+ - | *linker_so* | linker used to create shared objects and | - | | libraries | - +--------------+------------------------------------------+ - | *linker_exe* | linker used to create binary executables | - +--------------+------------------------------------------+ - | *archiver* | static library creator | - +--------------+------------------------------------------+ - - On platforms with a command-line (Unix, DOS/Windows), each of these is a string - that will be split into executable name and (optional) list of arguments. - (Splitting the string is done similarly to how Unix shells operate: words are - delimited by spaces, but quotes and backslashes can override this. See - :func:`distutils.util.split_quoted`.) - - The following methods invoke stages in the build process. - - - .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None]) - - Compile one or more source files. Generates object files (e.g. transforms a - :file:`.c` file to a :file:`.o` file.) - - *sources* must be a list of filenames, most likely C/C++ files, but in reality - anything that can be handled by a particular compiler and compiler class (eg. - :class:`MSVCCompiler` can handle resource files in *sources*). Return a list of - object filenames, one per source filename in *sources*. Depending on the - implementation, not all source files will necessarily be compiled, but all - corresponding object filenames will be returned. - - If *output_dir* is given, object files will be put under it, while retaining - their original path component. That is, :file:`foo/bar.c` normally compiles to - :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then - it would compile to :file:`build/foo/bar.o`. - - *macros*, if given, must be a list of macro definitions. A macro definition is - either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines - a macro; if the value is ``None``, the macro is defined without an explicit - value. The 1-tuple case undefines a macro. Later - definitions/redefinitions/undefinitions take precedence. - - *include_dirs*, if given, must be a list of strings, the directories to add to - the default include file search path for this compilation only. - - *debug* is a boolean; if true, the compiler will be instructed to output debug - symbols in (or alongside) the object file(s). - - *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms - that have the notion of a command-line (e.g. Unix, DOS/Windows), they are most - likely lists of strings: extra command-line arguments to prepend/append to the - compiler command line. On other platforms, consult the implementation class - documentation. In any event, they are intended as an escape hatch for those - occasions when the abstract compiler framework doesn't cut the mustard. - - *depends*, if given, is a list of filenames that all targets depend on. If a - source file is older than any file in depends, then the source file will be - recompiled. This supports dependency tracking, but only at a coarse - granularity. - - Raises :exc:`CompileError` on failure. - - - .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None]) - - Link a bunch of stuff together to create a static library file. The "bunch of - stuff" consists of the list of object files supplied as *objects*, the extra - object files supplied to :meth:`add_link_object` and/or - :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or - :meth:`set_libraries`, and the libraries supplied as *libraries* (if any). - - *output_libname* should be a library name, not a filename; the filename will be - inferred from the library name. *output_dir* is the directory where the library - file will be put. - - .. XXX defaults to what? - - *debug* is a boolean; if true, debugging information will be included in the - library (note that on most platforms, it is the compile step where this matters: - the *debug* flag is included here just for consistency). - - *target_lang* is the target language for which the given objects are being - compiled. This allows specific linkage time treatment of certain languages. - - Raises :exc:`LibError` on failure. - - - .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a bunch of stuff together to create an executable or shared library file. - - The "bunch of stuff" consists of the list of object files supplied as *objects*. - *output_filename* should be a filename. If *output_dir* is supplied, - *output_filename* is relative to it (i.e. *output_filename* can provide - directory components if needed). - - *libraries* is a list of libraries to link against. These are library names, - not filenames, since they're translated into filenames in a platform-specific - way (eg. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on - DOS/Windows). However, they can include a directory component, which means the - linker will look in that specific directory rather than searching all the normal - locations. - - *library_dirs*, if supplied, should be a list of directories to search for - libraries that were specified as bare library names (ie. no directory - component). These are on top of the system default and those supplied to - :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs* - is a list of directories that will be embedded into the shared library and used - to search for other shared libraries that \*it\* depends on at run-time. (This - may only be relevant on Unix.) - - *export_symbols* is a list of symbols that the shared library will export. - (This appears to be relevant only on Windows.) - - *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the - slight distinction that it actually matters on most platforms (as opposed to - :meth:`create_static_lib`, which includes a *debug* flag mostly for form's - sake). - - *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of - course that they supply command-line arguments for the particular linker being - used). - - *target_lang* is the target language for which the given objects are being - compiled. This allows specific linkage time treatment of certain languages. - - Raises :exc:`LinkError` on failure. - - - .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None]) - - Link an executable. *output_progname* is the name of the file executable, while - *objects* are a list of object filenames to link in. Other arguments are as for - the :meth:`link` method. - - - .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a shared library. *output_libname* is the name of the output library, - while *objects* is a list of object filenames to link in. Other arguments are - as for the :meth:`link` method. - - - .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a shared object. *output_filename* is the name of the shared object that - will be created, while *objects* is a list of object filenames to link in. - Other arguments are as for the :meth:`link` method. - - - .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None]) - - Preprocess a single C/C++ source file, named in *source*. Output will be written - to file named *output_file*, or *stdout* if *output_file* not supplied. - *macros* is a list of macro definitions as for :meth:`compile`, which will - augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`. - *include_dirs* is a list of directory names that will be added to the default - list, in the same way as :meth:`add_include_dir`. - - Raises :exc:`PreprocessError` on failure. - - The following utility methods are defined by the :class:`CCompiler` class, for - use by the various concrete subclasses. - - - .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir='']) - - Returns the filename of the executable for the given *basename*. Typically for - non-Windows platforms this is the same as the basename, while Windows will get - a :file:`.exe` added. - - - .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir='']) - - Returns the filename for the given library name on the current platform. On Unix - a library with *lib_type* of ``'static'`` will typically be of the form - :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form - :file:`liblibname.so`. - - - .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir='']) - - Returns the name of the object files for the given source files. - *source_filenames* should be a list of filenames. - - - .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir='']) - - Returns the name of a shared object file for the given file name *basename*. - - - .. method:: CCompiler.execute(func, args[, msg=None, level=1]) - - Invokes :func:`distutils.util.execute`. This method invokes a Python function - *func* with the given arguments *args*, after logging and taking into account - the *dry_run* flag. - - - .. method:: CCompiler.spawn(cmd) - - Invokes :func:`distutils.util.spawn`. This invokes an external process to run - the given command. - - - .. method:: CCompiler.mkpath(name[, mode=511]) - - Invokes :func:`distutils.dir_util.mkpath`. This creates a directory and any - missing ancestor directories. - - - .. method:: CCompiler.move_file(src, dst) - - Invokes :meth:`distutils.file_util.move_file`. Renames *src* to *dst*. - - - .. method:: CCompiler.announce(msg[, level=1]) - - Write a message using :func:`distutils.log.debug`. - - - .. method:: CCompiler.warn(msg) - - Write a warning message *msg* to standard error. - - - .. method:: CCompiler.debug_print(msg) - - If the *debug* flag is set on this :class:`CCompiler` instance, print *msg* to - standard output, otherwise do nothing. - -.. % \subsection{Compiler-specific modules} -.. % -.. % The following modules implement concrete subclasses of the abstract -.. % \class{CCompiler} class. They should not be instantiated directly, but should -.. % be created using \function{distutils.ccompiler.new_compiler()} factory -.. % function. - - -:mod:`distutils.unixccompiler` --- Unix C Compiler -================================================== - -.. module:: distutils.unixccompiler - :synopsis: UNIX C Compiler - - -This module provides the :class:`UnixCCompiler` class, a subclass of -:class:`CCompiler` that handles the typical Unix-style command-line C compiler: - -* macros defined with :option:`!-Dname[=value]` - -* macros undefined with :option:`!-Uname` - -* include search directories specified with :option:`!-Idir` - -* libraries specified with :option:`!-llib` - -* library search directories specified with :option:`!-Ldir` - -* compile handled by :program:`cc` (or similar) executable with :option:`!-c` - option: compiles :file:`.c` to :file:`.o` - -* link static library handled by :program:`ar` command (possibly with - :program:`ranlib`) - -* link shared library handled by :program:`cc` :option:`!-shared` - - -:mod:`distutils.msvccompiler` --- Microsoft Compiler -==================================================== - -.. module:: distutils.msvccompiler - :synopsis: Microsoft Compiler - - -This module provides :class:`MSVCCompiler`, an implementation of the abstract -:class:`CCompiler` class for Microsoft Visual Studio. Typically, extension -modules need to be compiled with the same compiler that was used to compile -Python. For Python 2.3 and earlier, the compiler was Visual Studio 6. For Python -2.4 and 2.5, the compiler is Visual Studio .NET 2003. The AMD64 and Itanium -binaries are created using the Platform SDK. - -:class:`MSVCCompiler` will normally choose the right compiler, linker etc. on -its own. To override this choice, the environment variables *DISTUTILS_USE_SDK* -and *MSSdk* must be both set. *MSSdk* indicates that the current environment has -been setup by the SDK's ``SetEnv.Cmd`` script, or that the environment variables -had been registered when the SDK was installed; *DISTUTILS_USE_SDK* indicates -that the distutils user has made an explicit choice to override the compiler -selection by :class:`MSVCCompiler`. - - -:mod:`distutils.bcppcompiler` --- Borland Compiler -================================================== - -.. module:: distutils.bcppcompiler - - -This module provides :class:`BorlandCCompiler`, a subclass of the abstract -:class:`CCompiler` class for the Borland C++ compiler. - - -:mod:`distutils.cygwincompiler` --- Cygwin Compiler -=================================================== - -.. module:: distutils.cygwinccompiler - - -This module provides the :class:`CygwinCCompiler` class, a subclass of -:class:`UnixCCompiler` that handles the Cygwin port of the GNU C compiler to -Windows. It also contains the Mingw32CCompiler class which handles the mingw32 -port of GCC (same as cygwin in no-cygwin mode). - - -:mod:`distutils.archive_util` --- Archiving utilities -====================================================== - -.. module:: distutils.archive_util - :synopsis: Utility functions for creating archive files (tarballs, zip files, ...) - - -This module provides a few functions for creating archive files, such as -tarballs or zipfiles. - - -.. function:: make_archive(base_name, format[, root_dir=None, base_dir=None, verbose=0, dry_run=0]) - - Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of - the file to create, minus any format-specific extension; *format* is the - archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or - ``ztar``. *root_dir* is a directory that will be the root directory of the - archive; ie. we typically ``chdir`` into *root_dir* before creating the - archive. *base_dir* is the directory where we start archiving from; ie. - *base_dir* will be the common prefix of all files and directories in the - archive. *root_dir* and *base_dir* both default to the current directory. - Returns the name of the archive file. - - .. versionchanged:: 3.5 - Added support for the ``xztar`` format. - - -.. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0]) - - 'Create an (optional compressed) archive as a tar file from all files in and - under *base_dir*. *compress* must be ``'gzip'`` (the default), - ``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'`` - method the compression utility named by :program:`compress` must be on the - default program search path, so this is probably Unix-specific. The output - tar file will be named :file:`base_dir.tar`, possibly plus the appropriate - compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the - output filename. - - .. versionchanged:: 3.5 - Added support for the ``xz`` compression. - - -.. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0]) - - Create a zip file from all files in and under *base_dir*. The output zip file - will be named *base_name* + :file:`.zip`. Uses either the :mod:`zipfile` Python - module (if available) or the InfoZIP :file:`zip` utility (if installed and - found on the default search path). If neither tool is available, raises - :exc:`DistutilsExecError`. Returns the name of the output zip file. - - -:mod:`distutils.dep_util` --- Dependency checking -================================================= - -.. module:: distutils.dep_util - :synopsis: Utility functions for simple dependency checking - - -This module provides functions for performing simple, timestamp-based -dependency of files and groups of files; also, functions based entirely on such -timestamp dependency analysis. - - -.. function:: newer(source, target) - - Return true if *source* exists and is more recently modified than *target*, or - if *source* exists and *target* doesn't. Return false if both exist and *target* - is the same age or newer than *source*. Raise :exc:`DistutilsFileError` if - *source* does not exist. - - -.. function:: newer_pairwise(sources, targets) - - Walk two filename lists in parallel, testing if each source is newer than its - corresponding target. Return a pair of lists (*sources*, *targets*) where - source is newer than target, according to the semantics of :func:`newer`. - - .. % % equivalent to a listcomp... - - -.. function:: newer_group(sources, target[, missing='error']) - - Return true if *target* is out-of-date with respect to any file listed in - *sources*. In other words, if *target* exists and is newer than every file in - *sources*, return false; otherwise return true. *missing* controls what we do - when a source file is missing; the default (``'error'``) is to blow up with an - :exc:`OSError` from inside :func:`os.stat`; if it is ``'ignore'``, we silently - drop any missing source files; if it is ``'newer'``, any missing source files - make us assume that *target* is out-of-date (this is handy in "dry-run" mode: - it'll make you pretend to carry out commands that wouldn't work because inputs - are missing, but that doesn't matter because you're not actually going to run - the commands). - - -:mod:`distutils.dir_util` --- Directory tree operations -======================================================= - -.. module:: distutils.dir_util - :synopsis: Utility functions for operating on directories and directory trees - - -This module provides functions for operating on directories and trees of -directories. - - -.. function:: mkpath(name[, mode=0o777, verbose=0, dry_run=0]) - - Create a directory and any missing ancestor directories. If the directory - already exists (or if *name* is the empty string, which means the current - directory, which of course exists), then do nothing. Raise - :exc:`DistutilsFileError` if unable to create some directory along the way (eg. - some sub-path exists, but is a file rather than a directory). If *verbose* is - true, print a one-line summary of each mkdir to stdout. Return the list of - directories actually created. - - -.. function:: create_tree(base_dir, files[, mode=0o777, verbose=0, dry_run=0]) - - Create all the empty directories under *base_dir* needed to put *files* there. - *base_dir* is just the name of a directory which doesn't necessarily exist - yet; *files* is a list of filenames to be interpreted relative to *base_dir*. - *base_dir* + the directory portion of every file in *files* will be created if - it doesn't already exist. *mode*, *verbose* and *dry_run* flags are as for - :func:`mkpath`. - - -.. function:: copy_tree(src, dst[, preserve_mode=1, preserve_times=1, preserve_symlinks=0, update=0, verbose=0, dry_run=0]) - - Copy an entire directory tree *src* to a new location *dst*. Both *src* and - *dst* must be directory names. If *src* is not a directory, raise - :exc:`DistutilsFileError`. If *dst* does not exist, it is created with - :func:`mkpath`. The end result of the copy is that every file in *src* is - copied to *dst*, and directories under *src* are recursively copied to *dst*. - Return the list of files that were copied or might have been copied, using their - output name. The return value is unaffected by *update* or *dry_run*: it is - simply the list of all files under *src*, with the names changed to be under - *dst*. - - *preserve_mode* and *preserve_times* are the same as for - :func:`distutils.file_util.copy_file`; note that they only apply to - regular files, not to - directories. If *preserve_symlinks* is true, symlinks will be copied as - symlinks (on platforms that support them!); otherwise (the default), the - destination of the symlink will be copied. *update* and *verbose* are the same - as for :func:`copy_file`. - - Files in *src* that begin with :file:`.nfs` are skipped (more information on - these files is available in answer D2 of the `NFS FAQ page - <http://nfs.sourceforge.net/#section_d>`_). - - .. versionchanged:: 3.3.1 - NFS files are ignored. - -.. function:: remove_tree(directory[, verbose=0, dry_run=0]) - - Recursively remove *directory* and all files and directories underneath it. Any - errors are ignored (apart from being reported to ``sys.stdout`` if *verbose* is - true). - - -:mod:`distutils.file_util` --- Single file operations -===================================================== - -.. module:: distutils.file_util - :synopsis: Utility functions for operating on single files - - -This module contains some utility functions for operating on individual files. - - -.. function:: copy_file(src, dst[, preserve_mode=1, preserve_times=1, update=0, link=None, verbose=0, dry_run=0]) - - Copy file *src* to *dst*. If *dst* is a directory, then *src* is copied there - with the same name; otherwise, it must be a filename. (If the file exists, it - will be ruthlessly clobbered.) If *preserve_mode* is true (the default), the - file's mode (type and permission bits, or whatever is analogous on the - current platform) is copied. If *preserve_times* is true (the default), the - last-modified and last-access times are copied as well. If *update* is true, - *src* will only be copied if *dst* does not exist, or if *dst* does exist but - is older than *src*. - - *link* allows you to make hard links (using :func:`os.link`) or symbolic links - (using :func:`os.symlink`) instead of copying: set it to ``'hard'`` or - ``'sym'``; if it is ``None`` (the default), files are copied. Don't set *link* - on systems that don't support it: :func:`copy_file` doesn't check if hard or - symbolic linking is available. It uses :func:`_copy_file_contents` to copy file - contents. - - Return a tuple ``(dest_name, copied)``: *dest_name* is the actual name of the - output file, and *copied* is true if the file was copied (or would have been - copied, if *dry_run* true). - - .. % XXX if the destination file already exists, we clobber it if - .. % copying, but blow up if linking. Hmmm. And I don't know what - .. % macostools.copyfile() does. Should definitely be consistent, and - .. % should probably blow up if destination exists and we would be - .. % changing it (ie. it's not already a hard/soft link to src OR - .. % (not update) and (src newer than dst)). - - -.. function:: move_file(src, dst[, verbose, dry_run]) - - Move file *src* to *dst*. If *dst* is a directory, the file will be moved into - it with the same name; otherwise, *src* is just renamed to *dst*. Returns the - new full name of the file. - - .. warning:: - - Handles cross-device moves on Unix using :func:`copy_file`. What about - other systems? - - -.. function:: write_file(filename, contents) - - Create a file called *filename* and write *contents* (a sequence of strings - without line terminators) to it. - - -:mod:`distutils.util` --- Miscellaneous other utility functions -=============================================================== - -.. module:: distutils.util - :synopsis: Miscellaneous other utility functions - - -This module contains other assorted bits and pieces that don't fit into any -other utility module. - - -.. function:: get_platform() - - Return a string that identifies the current platform. This is used mainly to - distinguish platform-specific build directories and platform-specific built - distributions. Typically includes the OS name and version and the architecture - (as supplied by 'os.uname()'), although the exact information included depends - on the OS; eg. for IRIX the architecture isn't particularly important (IRIX only - runs on SGI hardware), but for Linux the kernel version isn't particularly - important. - - Examples of returned values: - - * ``linux-i586`` - * ``linux-alpha`` - * ``solaris-2.6-sun4u`` - * ``irix-5.3`` - * ``irix64-6.2`` - - For non-POSIX platforms, currently just returns ``sys.platform``. - - For Mac OS X systems the OS version reflects the minimal version on which - binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` - during the build of Python), not the OS version of the current system. - - For universal binary builds on Mac OS X the architecture value reflects - the universal binary status instead of the architecture of the current - processor. For 32-bit universal binaries the architecture is ``fat``, - for 64-bit universal binaries the architecture is ``fat64``, and - for 4-way universal binaries the architecture is ``universal``. Starting - from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for - a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for - a universal build with the i386 and x86_64 architectures - - Examples of returned values on Mac OS X: - - * ``macosx-10.3-ppc`` - - * ``macosx-10.3-fat`` - - * ``macosx-10.5-universal`` - - * ``macosx-10.6-intel`` - - -.. function:: convert_path(pathname) - - Return 'pathname' as a name that will work on the native filesystem, i.e. split - it on '/' and put it back together again using the current directory separator. - Needed because filenames in the setup script are always supplied in Unix style, - and have to be converted to the local convention before we can actually use them - in the filesystem. Raises :exc:`ValueError` on non-Unix-ish systems if - *pathname* either starts or ends with a slash. - - -.. function:: change_root(new_root, pathname) - - Return *pathname* with *new_root* prepended. If *pathname* is relative, this is - equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires making - *pathname* relative and then joining the two, which is tricky on DOS/Windows. - - -.. function:: check_environ() - - Ensure that 'os.environ' has all the environment variables we guarantee that - users can use in config files, command-line options, etc. Currently this - includes: - - * :envvar:`HOME` - user's home directory (Unix only) - * :envvar:`PLAT` - description of the current platform, including hardware and - OS (see :func:`get_platform`) - - -.. function:: subst_vars(s, local_vars) - - Perform shell/Perl-style variable substitution on *s*. Every occurrence of - ``$`` followed by a name is considered a variable, and variable is substituted - by the value found in the *local_vars* dictionary, or in ``os.environ`` if it's - not in *local_vars*. *os.environ* is first checked/augmented to guarantee that - it contains certain values: see :func:`check_environ`. Raise :exc:`ValueError` - for any variables not found in either *local_vars* or ``os.environ``. - - Note that this is not a fully-fledged string interpolation function. A valid - ``$variable`` can consist only of upper and lower case letters, numbers and an - underscore. No { } or ( ) style quoting is available. - - -.. function:: split_quoted(s) - - Split a string up according to Unix shell-like rules for quotes and backslashes. - In short: words are delimited by spaces, as long as those spaces are not escaped - by a backslash, or inside a quoted string. Single and double quotes are - equivalent, and the quote characters can be backslash-escaped. The backslash is - stripped from any two-character escape sequence, leaving only the escaped - character. The quote characters are stripped from any quoted string. Returns a - list of words. - - .. % Should probably be moved into the standard library. - - -.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0]) - - Perform some action that affects the outside world (for instance, writing to the - filesystem). Such actions are special because they are disabled by the - *dry_run* flag. This method takes care of all that bureaucracy for you; all - you have to do is supply the function to call and an argument tuple for it (to - embody the "external action" being performed), and an optional message to print. - - -.. function:: strtobool(val) - - Convert a string representation of truth to true (1) or false (0). - - True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false values - are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises - :exc:`ValueError` if *val* is anything else. - - -.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None]) - - Byte-compile a collection of Python source files to :file:`.pyc` files in a - :file:`__pycache__` subdirectory (see :pep:`3147` and :pep:`488`). - *py_files* is a list of files to compile; any files that don't end in - :file:`.py` are silently skipped. *optimize* must be one of the following: - - * ``0`` - don't optimize - * ``1`` - normal optimization (like ``python -O``) - * ``2`` - extra optimization (like ``python -OO``) - - If *force* is true, all files are recompiled regardless of timestamps. - - The source filename encoded in each :term:`bytecode` file defaults to the filenames - listed in *py_files*; you can modify these with *prefix* and *basedir*. - *prefix* is a string that will be stripped off of each source filename, and - *base_dir* is a directory name that will be prepended (after *prefix* is - stripped). You can supply either or both (or neither) of *prefix* and - *base_dir*, as you wish. - - If *dry_run* is true, doesn't actually do anything that would affect the - filesystem. - - Byte-compilation is either done directly in this interpreter process with the - standard :mod:`py_compile` module, or indirectly by writing a temporary script - and executing it. Normally, you should let :func:`byte_compile` figure out to - use direct compilation or not (see the source for details). The *direct* flag - is used by the script generated in indirect mode; unless you know what you're - doing, leave it set to ``None``. - - .. versionchanged:: 3.2.3 - Create ``.pyc`` files with an :func:`import magic tag - <imp.get_tag>` in their name, in a :file:`__pycache__` subdirectory - instead of files without tag in the current directory. - - .. versionchanged:: 3.5 - Create ``.pyc`` files according to :pep:`488`. - - -.. function:: rfc822_escape(header) - - Return a version of *header* escaped for inclusion in an :rfc:`822` header, by - ensuring there are 8 spaces space after each newline. Note that it does no other - modification of the string. - - .. % this _can_ be replaced - -.. % \subsection{Distutils objects} - - -:mod:`distutils.dist` --- The Distribution class -================================================ - -.. module:: distutils.dist - :synopsis: Provides the Distribution class, which represents the module distribution being - built/installed/distributed - - -This module provides the :class:`~distutils.core.Distribution` class, which -represents the module distribution being built/installed/distributed. - - -:mod:`distutils.extension` --- The Extension class -================================================== - -.. module:: distutils.extension - :synopsis: Provides the Extension class, used to describe C/C++ extension modules in setup - scripts - - -This module provides the :class:`Extension` class, used to describe C/C++ -extension modules in setup scripts. - -.. % \subsection{Ungrouped modules} -.. % The following haven't been moved into a more appropriate section yet. - - -:mod:`distutils.debug` --- Distutils debug mode -=============================================== - -.. module:: distutils.debug - :synopsis: Provides the debug flag for distutils - - -This module provides the DEBUG flag. - - -:mod:`distutils.errors` --- Distutils exceptions -================================================ - -.. module:: distutils.errors - :synopsis: Provides standard distutils exceptions - - -Provides exceptions used by the Distutils modules. Note that Distutils modules -may raise standard exceptions; in particular, SystemExit is usually raised for -errors that are obviously the end-user's fault (eg. bad command-line arguments). - -This module is safe to use in ``from ... import *`` mode; it only exports -symbols whose names start with ``Distutils`` and end with ``Error``. - - -:mod:`distutils.fancy_getopt` --- Wrapper around the standard getopt module -=========================================================================== - -.. module:: distutils.fancy_getopt - :synopsis: Additional getopt functionality - - -This module provides a wrapper around the standard :mod:`getopt` module that -provides the following additional features: - -* short and long options are tied together - -* options have help strings, so :func:`fancy_getopt` could potentially create a - complete usage summary - -* options set attributes of a passed-in object - -* boolean options can have "negative aliases" --- eg. if :option:`!--quiet` is - the "negative alias" of :option:`!--verbose`, then :option:`!--quiet` on the - command line sets *verbose* to false. - -.. function:: fancy_getopt(options, negative_opt, object, args) - - Wrapper function. *options* is a list of ``(long_option, short_option, - help_string)`` 3-tuples as described in the constructor for - :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names - to option names, both the key and value should be in the *options* list. - *object* is an object which will be used to store values (see the :meth:`getopt` - method of the :class:`FancyGetopt` class). *args* is the argument list. Will use - ``sys.argv[1:]`` if you pass ``None`` as *args*. - - -.. function:: wrap_text(text, width) - - Wraps *text* to less than *width* wide. - - -.. class:: FancyGetopt([option_table=None]) - - The option_table is a list of 3-tuples: ``(long_option, short_option, - help_string)`` - - If an option takes an argument, its *long_option* should have ``'='`` appended; - *short_option* should just be a single character, no ``':'`` in any case. - *short_option* should be ``None`` if a *long_option* doesn't have a - corresponding *short_option*. All option tuples must have long options. - -The :class:`FancyGetopt` class provides the following methods: - - -.. method:: FancyGetopt.getopt([args=None, object=None]) - - Parse command-line options in args. Store as attributes on *object*. - - If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is - ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores - option values there, and returns a tuple ``(args, object)``. If *object* is - supplied, it is modified in place and :func:`getopt` just returns *args*; in - both cases, the returned *args* is a modified copy of the passed-in *args* list, - which is left untouched. - - .. % and args returned are? - - -.. method:: FancyGetopt.get_option_order() - - Returns the list of ``(option, value)`` tuples processed by the previous run of - :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called - yet. - - -.. method:: FancyGetopt.generate_help([header=None]) - - Generate help text (a list of strings, one per suggested line of output) from - the option table for this :class:`FancyGetopt` object. - - If supplied, prints the supplied *header* at the top of the help. - - -:mod:`distutils.filelist` --- The FileList class -================================================ - -.. module:: distutils.filelist - :synopsis: The FileList class, used for poking about the file system and - building lists of files. - - -This module provides the :class:`FileList` class, used for poking about the -filesystem and building lists of files. - - -:mod:`distutils.log` --- Simple PEP 282-style logging -===================================================== - -.. module:: distutils.log - :synopsis: A simple logging mechanism, 282-style - - -:mod:`distutils.spawn` --- Spawn a sub-process -============================================== - -.. module:: distutils.spawn - :synopsis: Provides the spawn() function - - -This module provides the :func:`spawn` function, a front-end to various -platform-specific functions for launching another program in a sub-process. -Also provides :func:`find_executable` to search the path for a given executable -name. - - -:mod:`distutils.sysconfig` --- System configuration information -=============================================================== - -.. module:: distutils.sysconfig - :synopsis: Low-level access to configuration information of the Python interpreter. -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. moduleauthor:: Greg Ward <gward@python.net> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - - -The :mod:`distutils.sysconfig` module provides access to Python's low-level -configuration information. The specific configuration variables available -depend heavily on the platform and configuration. The specific variables depend -on the build process for the specific version of Python being run; the variables -are those found in the :file:`Makefile` and configuration header that are -installed with Python on Unix systems. The configuration header is called -:file:`pyconfig.h` for Python versions starting with 2.2, and :file:`config.h` -for earlier versions of Python. - -Some additional functions are provided which perform some useful manipulations -for other parts of the :mod:`distutils` package. - - -.. data:: PREFIX - - The result of ``os.path.normpath(sys.prefix)``. - - -.. data:: EXEC_PREFIX - - The result of ``os.path.normpath(sys.exec_prefix)``. - - -.. function:: get_config_var(name) - - Return the value of a single variable. This is equivalent to - ``get_config_vars().get(name)``. - - -.. function:: get_config_vars(...) - - Return a set of variable definitions. If there are no arguments, this returns a - dictionary mapping names of configuration variables to values. If arguments are - provided, they should be strings, and the return value will be a sequence giving - the associated values. If a given name does not have a corresponding value, - ``None`` will be included for that variable. - - -.. function:: get_config_h_filename() - - Return the full path name of the configuration header. For Unix, this will be - the header generated by the :program:`configure` script; for other platforms the - header will have been supplied directly by the Python source distribution. The - file is a platform-specific text file. - - -.. function:: get_makefile_filename() - - Return the full path name of the :file:`Makefile` used to build Python. For - Unix, this will be a file generated by the :program:`configure` script; the - meaning for other platforms will vary. The file is a platform-specific text - file, if it exists. This function is only useful on POSIX platforms. - - -.. function:: get_python_inc([plat_specific[, prefix]]) - - Return the directory for either the general or platform-dependent C include - files. If *plat_specific* is true, the platform-dependent include directory is - returned; if false or omitted, the platform-independent directory is returned. - If *prefix* is given, it is used as either the prefix instead of - :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if - *plat_specific* is true. - - -.. function:: get_python_lib([plat_specific[, standard_lib[, prefix]]]) - - Return the directory for either the general or platform-dependent library - installation. If *plat_specific* is true, the platform-dependent include - directory is returned; if false or omitted, the platform-independent directory - is returned. If *prefix* is given, it is used as either the prefix instead of - :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if - *plat_specific* is true. If *standard_lib* is true, the directory for the - standard library is returned rather than the directory for the installation of - third-party extensions. - -The following function is only intended for use within the :mod:`distutils` -package. - - -.. function:: customize_compiler(compiler) - - Do any platform-specific customization of a - :class:`distutils.ccompiler.CCompiler` instance. - - This function is only needed on Unix at this time, but should be called - consistently to support forward-compatibility. It inserts the information that - varies across Unix flavors and is stored in Python's :file:`Makefile`. This - information includes the selected compiler, compiler and linker options, and the - extension used by the linker for shared objects. - -This function is even more special-purpose, and should only be used from -Python's own build procedures. - - -.. function:: set_python_build() - - Inform the :mod:`distutils.sysconfig` module that it is being used as part of - the build process for Python. This changes a lot of relative locations for - files, allowing them to be located in the build area rather than in an installed - Python. - - -:mod:`distutils.text_file` --- The TextFile class -================================================= - -.. module:: distutils.text_file - :synopsis: provides the TextFile class, a simple interface to text files - - -This module provides the :class:`TextFile` class, which gives an interface to -text files that (optionally) takes care of stripping comments, ignoring blank -lines, and joining lines with backslashes. - - -.. class:: TextFile([filename=None, file=None, **options]) - - This class provides a file-like object that takes care of all the things you - commonly want to do when processing a text file that has some line-by-line - syntax: strip comments (as long as ``#`` is your comment character), skip blank - lines, join adjacent lines by escaping the newline (ie. backslash at end of - line), strip leading and/or trailing whitespace. All of these are optional and - independently controllable. - - The class provides a :meth:`warn` method so you can generate warning messages - that report physical line number, even if the logical line in question spans - multiple physical lines. Also provides :meth:`unreadline` for implementing - line-at-a-time lookahead. - - :class:`TextFile` instances are create with either *filename*, *file*, or both. - :exc:`RuntimeError` is raised if both are ``None``. *filename* should be a - string, and *file* a file object (or something that provides :meth:`readline` - and :meth:`close` methods). It is recommended that you supply at least - *filename*, so that :class:`TextFile` can include it in warning messages. If - *file* is not supplied, :class:`TextFile` creates its own using the - :func:`open` built-in function. - - The options are all boolean, and affect the values returned by :meth:`readline` - - .. tabularcolumns:: |l|L|l| - - +------------------+--------------------------------+---------+ - | option name | description | default | - +==================+================================+=========+ - | *strip_comments* | strip from ``'#'`` to | true | - | | end-of-line, as well as any | | - | | whitespace leading up to the | | - | | ``'#'``\ ---unless it is | | - | | escaped by a backslash | | - +------------------+--------------------------------+---------+ - | *lstrip_ws* | strip leading whitespace from | false | - | | each line before returning it | | - +------------------+--------------------------------+---------+ - | *rstrip_ws* | strip trailing whitespace | true | - | | (including line terminator!) | | - | | from each line before | | - | | returning it. | | - +------------------+--------------------------------+---------+ - | *skip_blanks* | skip lines that are empty | true | - | | \*after\* stripping comments | | - | | and whitespace. (If both | | - | | lstrip_ws and rstrip_ws are | | - | | false, then some lines may | | - | | consist of solely whitespace: | | - | | these will \*not\* be skipped, | | - | | even if *skip_blanks* is | | - | | true.) | | - +------------------+--------------------------------+---------+ - | *join_lines* | if a backslash is the last | false | - | | non-newline character on a | | - | | line after stripping comments | | - | | and whitespace, join the | | - | | following line to it to form | | - | | one logical line; if N | | - | | consecutive lines end with a | | - | | backslash, then N+1 physical | | - | | lines will be joined to form | | - | | one logical line. | | - +------------------+--------------------------------+---------+ - | *collapse_join* | strip leading whitespace from | false | - | | lines that are joined to their | | - | | predecessor; only matters if | | - | | ``(join_lines and not | | - | | lstrip_ws)`` | | - +------------------+--------------------------------+---------+ - - Note that since *rstrip_ws* can strip the trailing newline, the semantics of - :meth:`readline` must differ from those of the built-in file object's - :meth:`readline` method! In particular, :meth:`readline` returns ``None`` for - end-of-file: an empty string might just be a blank line (or an all-whitespace - line), if *rstrip_ws* is true but *skip_blanks* is not. - - - .. method:: TextFile.open(filename) - - Open a new file *filename*. This overrides any *file* or *filename* - constructor arguments. - - - .. method:: TextFile.close() - - Close the current file and forget everything we know about it (including the - filename and the current line number). - - - .. method:: TextFile.warn(msg[,line=None]) - - Print (to stderr) a warning message tied to the current logical line in the - current file. If the current logical line in the file spans multiple physical - lines, the warning refers to the whole range, such as ``"lines 3-5"``. If - *line* is supplied, it overrides the current line number; it may be a list or - tuple to indicate a range of physical lines, or an integer for a single - physical line. - - - .. method:: TextFile.readline() - - Read and return a single logical line from the current file (or from an internal - buffer if lines have previously been "unread" with :meth:`unreadline`). If the - *join_lines* option is true, this may involve reading multiple physical lines - concatenated into a single string. Updates the current line number, so calling - :meth:`warn` after :meth:`readline` emits a warning about the physical line(s) - just read. Returns ``None`` on end-of-file, since the empty string can occur - if *rstrip_ws* is true but *strip_blanks* is not. - - - .. method:: TextFile.readlines() - - Read and return the list of all logical lines remaining in the current file. - This updates the current line number to the last line of the file. - - - .. method:: TextFile.unreadline(line) - - Push *line* (a string) onto an internal buffer that will be checked by future - :meth:`readline` calls. Handy for implementing a parser with line-at-a-time - lookahead. Note that lines that are "unread" with :meth:`unreadline` are not - subsequently re-cleansed (whitespace stripped, or whatever) when read with - :meth:`readline`. If multiple calls are made to :meth:`unreadline` before a call - to :meth:`readline`, the lines will be returned most in most recent first order. - - -:mod:`distutils.version` --- Version number classes -=================================================== - -.. module:: distutils.version - :synopsis: implements classes that represent module version numbers. - - -.. % todo -.. % \section{Distutils Commands} -.. % -.. % This part of Distutils implements the various Distutils commands, such -.. % as \code{build}, \code{install} \&c. Each command is implemented as a -.. % separate module, with the command name as the name of the module. - - -:mod:`distutils.cmd` --- Abstract base class for Distutils commands -=================================================================== - -.. module:: distutils.cmd - :synopsis: This module provides the abstract base class Command. This class - is subclassed by the modules in the distutils.command subpackage. - - -This module supplies the abstract base class :class:`Command`. - - -.. class:: Command(dist) - - Abstract base class for defining command classes, the "worker bees" of the - Distutils. A useful analogy for command classes is to think of them as - subroutines with local variables called *options*. The options are declared - in :meth:`initialize_options` and defined (given their final values) in - :meth:`finalize_options`, both of which must be defined by every command - class. The distinction between the two is necessary because option values - might come from the outside world (command line, config file, ...), and any - options dependent on other options must be computed after these outside - influences have been processed --- hence :meth:`finalize_options`. The body - of the subroutine, where it does all its work based on the values of its - options, is the :meth:`run` method, which must also be implemented by every - command class. - - The class constructor takes a single argument *dist*, a - :class:`~distutils.core.Distribution` instance. - - -Creating a new Distutils command -================================ - -This section outlines the steps to create a new Distutils command. - -A new command lives in a module in the :mod:`distutils.command` package. There -is a sample template in that directory called :file:`command_template`. Copy -this file to a new module with the same name as the new command you're -implementing. This module should implement a class with the same name as the -module (and the command). So, for instance, to create the command -``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy -:file:`command_template` to :file:`distutils/command/peel_banana.py`, then edit -it so that it's implementing the class :class:`peel_banana`, a subclass of -:class:`distutils.cmd.Command`. - -Subclasses of :class:`Command` must define the following methods. - -.. method:: Command.initialize_options() - - Set default values for all the options that this command supports. Note that - these defaults may be overridden by other commands, by the setup script, by - config files, or by the command-line. Thus, this is not the place to code - dependencies between options; generally, :meth:`initialize_options` - implementations are just a bunch of ``self.foo = None`` assignments. - - -.. method:: Command.finalize_options() - - Set final values for all the options that this command supports. This is - always called as late as possible, ie. after any option assignments from the - command-line or from other commands have been done. Thus, this is the place - to code option dependencies: if *foo* depends on *bar*, then it is safe to - set *foo* from *bar* as long as *foo* still has the same value it was - assigned in :meth:`initialize_options`. - - -.. method:: Command.run() - - A command's raison d'etre: carry out the action it exists to perform, controlled - by the options initialized in :meth:`initialize_options`, customized by other - commands, the setup script, the command-line, and config files, and finalized in - :meth:`finalize_options`. All terminal output and filesystem interaction should - be done by :meth:`run`. - - -.. attribute:: Command.sub_commands - - *sub_commands* formalizes the notion of a "family" of commands, - e.g. ``install`` as the parent with sub-commands ``install_lib``, - ``install_headers``, etc. The parent of a family of commands defines - *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name, - predicate)``, with *command_name* a string and *predicate* a function, a - string or ``None``. *predicate* is a method of the parent command that - determines whether the corresponding command is applicable in the current - situation. (E.g. ``install_headers`` is only applicable if we have any C - header files to install.) If *predicate* is ``None``, that command is always - applicable. - - *sub_commands* is usually defined at the *end* of a class, because - predicates can be methods of the class, so they must already have been - defined. The canonical example is the :command:`install` command. - - -:mod:`distutils.command` --- Individual Distutils commands -========================================================== - -.. module:: distutils.command - :synopsis: This subpackage contains one module for each standard Distutils command. - - -.. % \subsubsection{Individual Distutils commands} -.. % todo - - -:mod:`distutils.command.bdist` --- Build a binary installer -=========================================================== - -.. module:: distutils.command.bdist - :synopsis: Build a binary installer for a package - - -.. % todo - - -:mod:`distutils.command.bdist_packager` --- Abstract base class for packagers -============================================================================= - -.. module:: distutils.command.bdist_packager - :synopsis: Abstract base class for packagers - - -.. % todo - - -:mod:`distutils.command.bdist_dumb` --- Build a "dumb" installer -================================================================ - -.. module:: distutils.command.bdist_dumb - :synopsis: Build a "dumb" installer - a simple archive of files - - -.. % todo - - -:mod:`distutils.command.bdist_msi` --- Build a Microsoft Installer binary package -================================================================================= - -.. module:: distutils.command.bdist_msi - :synopsis: Build a binary distribution as a Windows MSI file - -.. class:: bdist_msi - - Builds a `Windows Installer`_ (.msi) binary package. - - .. _Windows Installer: https://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx - - In most cases, the ``bdist_msi`` installer is a better choice than the - ``bdist_wininst`` installer, because it provides better support for - Win64 platforms, allows administrators to perform non-interactive - installations, and allows installation through group policies. - - -:mod:`distutils.command.bdist_rpm` --- Build a binary distribution as a Redhat RPM and SRPM -=========================================================================================== - -.. module:: distutils.command.bdist_rpm - :synopsis: Build a binary distribution as a Redhat RPM and SRPM - - -.. % todo - - -:mod:`distutils.command.bdist_wininst` --- Build a Windows installer -==================================================================== - -.. module:: distutils.command.bdist_wininst - :synopsis: Build a Windows installer - - -.. % todo - - -:mod:`distutils.command.sdist` --- Build a source distribution -============================================================== - -.. module:: distutils.command.sdist - :synopsis: Build a source distribution - - -.. % todo - - -:mod:`distutils.command.build` --- Build all files of a package -=============================================================== - -.. module:: distutils.command.build - :synopsis: Build all files of a package - - -.. % todo - - -:mod:`distutils.command.build_clib` --- Build any C libraries in a package -========================================================================== - -.. module:: distutils.command.build_clib - :synopsis: Build any C libraries in a package - - -.. % todo - - -:mod:`distutils.command.build_ext` --- Build any extensions in a package -======================================================================== - -.. module:: distutils.command.build_ext - :synopsis: Build any extensions in a package - - -.. % todo - - -:mod:`distutils.command.build_py` --- Build the .py/.pyc files of a package -=========================================================================== - -.. module:: distutils.command.build_py - :synopsis: Build the .py/.pyc files of a package - - -.. class:: build_py - -.. class:: build_py_2to3 - - Alternative implementation of build_py which also runs the - 2to3 conversion library on each .py file that is going to be - installed. To use this in a setup.py file for a distribution - that is designed to run with both Python 2.x and 3.x, add:: - - try: - from distutils.command.build_py import build_py_2to3 as build_py - except ImportError: - from distutils.command.build_py import build_py - - to your setup.py, and later:: - - cmdclass = {'build_py': build_py} - - to the invocation of setup(). - - -:mod:`distutils.command.build_scripts` --- Build the scripts of a package -========================================================================= - -.. module:: distutils.command.build_scripts - :synopsis: Build the scripts of a package - - -.. % todo - - -:mod:`distutils.command.clean` --- Clean a package build area -============================================================= - -.. module:: distutils.command.clean - :synopsis: Clean a package build area - -This command removes the temporary files created by :command:`build` -and its subcommands, like intermediary compiled object files. With -the ``--all`` option, the complete build directory will be removed. - -Extension modules built :ref:`in place <distutils-build-ext-inplace>` -will not be cleaned, as they are not in the build directory. - - -:mod:`distutils.command.config` --- Perform package configuration -================================================================= - -.. module:: distutils.command.config - :synopsis: Perform package configuration - - -.. % todo - - -:mod:`distutils.command.install` --- Install a package -====================================================== - -.. module:: distutils.command.install - :synopsis: Install a package - - -.. % todo - - -:mod:`distutils.command.install_data` --- Install data files from a package -=========================================================================== - -.. module:: distutils.command.install_data - :synopsis: Install data files from a package - - -.. % todo - - -:mod:`distutils.command.install_headers` --- Install C/C++ header files from a package -====================================================================================== - -.. module:: distutils.command.install_headers - :synopsis: Install C/C++ header files from a package - - -.. % todo - - -:mod:`distutils.command.install_lib` --- Install library files from a package -============================================================================= - -.. module:: distutils.command.install_lib - :synopsis: Install library files from a package - - -.. % todo - - -:mod:`distutils.command.install_scripts` --- Install script files from a package -================================================================================ - -.. module:: distutils.command.install_scripts - :synopsis: Install script files from a package - - -.. % todo - - -:mod:`distutils.command.register` --- Register a module with the Python Package Index -===================================================================================== - -.. module:: distutils.command.register - :synopsis: Register a module with the Python Package Index - - -The ``register`` command registers the package with the Python Package Index. -This is described in more detail in :pep:`301`. - -.. % todo - - -:mod:`distutils.command.check` --- Check the meta-data of a package -=================================================================== - -.. module:: distutils.command.check - :synopsis: Check the metadata of a package - - -The ``check`` command performs some tests on the meta-data of a package. -For example, it verifies that all required meta-data are provided as -the arguments passed to the :func:`setup` function. - -.. % todo diff --git a/third_party/python/Doc/distutils/builtdist.rst b/third_party/python/Doc/distutils/builtdist.rst deleted file mode 100644 index 92d796fb7..000000000 --- a/third_party/python/Doc/distutils/builtdist.rst +++ /dev/null @@ -1,459 +0,0 @@ -.. _built-dist: - -**************************** -Creating Built Distributions -**************************** - -A "built distribution" is what you're probably used to thinking of either as a -"binary package" or an "installer" (depending on your background). It's not -necessarily binary, though, because it might contain only Python source code -and/or byte-code; and we don't call it a package, because that word is already -spoken for in Python. (And "installer" is a term specific to the world of -mainstream desktop systems.) - -A built distribution is how you make life as easy as possible for installers of -your module distribution: for users of RPM-based Linux systems, it's a binary -RPM; for Windows users, it's an executable installer; for Debian-based Linux -users, it's a Debian package; and so forth. Obviously, no one person will be -able to create built distributions for every platform under the sun, so the -Distutils are designed to enable module developers to concentrate on their -specialty---writing code and creating source distributions---while an -intermediary species called *packagers* springs up to turn source distributions -into built distributions for as many platforms as there are packagers. - -Of course, the module developer could be their own packager; or the packager could -be a volunteer "out there" somewhere who has access to a platform which the -original developer does not; or it could be software periodically grabbing new -source distributions and turning them into built distributions for as many -platforms as the software has access to. Regardless of who they are, a packager -uses the setup script and the :command:`bdist` command family to generate built -distributions. - -As a simple example, if I run the following command in the Distutils source -tree:: - - python setup.py bdist - -then the Distutils builds my module distribution (the Distutils itself in this -case), does a "fake" installation (also in the :file:`build` directory), and -creates the default type of built distribution for my platform. The default -format for built distributions is a "dumb" tar file on Unix, and a simple -executable installer on Windows. (That tar file is considered "dumb" because it -has to be unpacked in a specific location to work.) - -Thus, the above command on a Unix system creates -:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place -installs the Distutils just as though you had downloaded the source distribution -and run ``python setup.py install``. (The "right place" is either the root of -the filesystem or Python's :file:`{prefix}` directory, depending on the options -given to the :command:`bdist_dumb` command; the default is to make dumb -distributions relative to :file:`{prefix}`.) - -Obviously, for pure Python distributions, this isn't any simpler than just -running ``python setup.py install``\ ---but for non-pure distributions, which -include extensions that would need to be compiled, it can mean the difference -between someone being able to use your extensions or not. And creating "smart" -built distributions, such as an RPM package or an executable installer for -Windows, is far more convenient for users even if your distribution doesn't -include any extensions. - -The :command:`bdist` command has a :option:`!--formats` option, similar to the -:command:`sdist` command, which you can use to select the types of built -distribution to generate: for example, :: - - python setup.py bdist --format=zip - -would, when run on a Unix system, create -:file:`Distutils-1.0.{plat}.zip`\ ---again, this archive would be unpacked -from the root directory to install the Distutils. - -The available formats for built distributions are: - -+-------------+------------------------------+---------+ -| Format | Description | Notes | -+=============+==============================+=========+ -| ``gztar`` | gzipped tar file | \(1) | -| | (:file:`.tar.gz`) | | -+-------------+------------------------------+---------+ -| ``bztar`` | bzipped tar file | | -| | (:file:`.tar.bz2`) | | -+-------------+------------------------------+---------+ -| ``xztar`` | xzipped tar file | | -| | (:file:`.tar.xz`) | | -+-------------+------------------------------+---------+ -| ``ztar`` | compressed tar file | \(3) | -| | (:file:`.tar.Z`) | | -+-------------+------------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | | -+-------------+------------------------------+---------+ -| ``zip`` | zip file (:file:`.zip`) | (2),(4) | -+-------------+------------------------------+---------+ -| ``rpm`` | RPM | \(5) | -+-------------+------------------------------+---------+ -| ``pkgtool`` | Solaris :program:`pkgtool` | | -+-------------+------------------------------+---------+ -| ``sdux`` | HP-UX :program:`swinstall` | | -+-------------+------------------------------+---------+ -| ``wininst`` | self-extracting ZIP file for | \(4) | -| | Windows | | -+-------------+------------------------------+---------+ -| ``msi`` | Microsoft Installer. | | -+-------------+------------------------------+---------+ - -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - - -Notes: - -(1) - default on Unix - -(2) - default on Windows - -(3) - requires external :program:`compress` utility. - -(4) - requires either external :program:`zip` utility or :mod:`zipfile` module (part - of the standard Python library since Python 1.6) - -(5) - requires external :program:`rpm` utility, version 3.0.4 or better (use ``rpm - --version`` to find out which version you have) - -You don't have to use the :command:`bdist` command with the :option:`!--formats` -option; you can also use the command that directly implements the format you're -interested in. Some of these :command:`bdist` "sub-commands" actually generate -several similar formats; for instance, the :command:`bdist_dumb` command -generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``, -``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both -binary and source RPMs. The :command:`bdist` sub-commands, and the formats -generated by each, are: - -+--------------------------+-------------------------------------+ -| Command | Formats | -+==========================+=====================================+ -| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip | -+--------------------------+-------------------------------------+ -| :command:`bdist_rpm` | rpm, srpm | -+--------------------------+-------------------------------------+ -| :command:`bdist_wininst` | wininst | -+--------------------------+-------------------------------------+ -| :command:`bdist_msi` | msi | -+--------------------------+-------------------------------------+ - -The following sections give details on the individual :command:`bdist_\*` -commands. - - -.. .. _creating-dumb: - -.. Creating dumb built distributions -.. ================================= - -.. XXX Need to document absolute vs. prefix-relative packages here, but first - I have to implement it! - - -.. _creating-rpms: - -Creating RPM packages -===================== - -The RPM format is used by many popular Linux distributions, including Red Hat, -SuSE, and Mandrake. If one of these (or any of the other RPM-based Linux -distributions) is your usual environment, creating RPM packages for other users -of that same distribution is trivial. Depending on the complexity of your module -distribution and differences between Linux distributions, you may also be able -to create RPMs that work on different RPM-based distributions. - -The usual way to create an RPM of your module distribution is to run the -:command:`bdist_rpm` command:: - - python setup.py bdist_rpm - -or the :command:`bdist` command with the :option:`!--format` option:: - - python setup.py bdist --formats=rpm - -The former allows you to specify RPM-specific options; the latter allows you to -easily specify multiple formats in one run. If you need to do both, you can -explicitly specify multiple :command:`bdist_\*` commands and their options:: - - python setup.py bdist_rpm --packager="John Doe <jdoe@example.org>" \ - bdist_wininst --target-version="2.0" - -Creating RPM packages is driven by a :file:`.spec` file, much as using the -Distutils is driven by the setup script. To make your life easier, the -:command:`bdist_rpm` command normally creates a :file:`.spec` file based on the -information you supply in the setup script, on the command line, and in any -Distutils configuration files. Various options and sections in the -:file:`.spec` file are derived from options in the setup script as follows: - -+------------------------------------------+----------------------------------------------+ -| RPM :file:`.spec` file option or section | Distutils setup script option | -+==========================================+==============================================+ -| Name | ``name`` | -+------------------------------------------+----------------------------------------------+ -| Summary (in preamble) | ``description`` | -+------------------------------------------+----------------------------------------------+ -| Version | ``version`` | -+------------------------------------------+----------------------------------------------+ -| Vendor | ``author`` and ``author_email``, | -| | or --- & ``maintainer`` and | -| | ``maintainer_email`` | -+------------------------------------------+----------------------------------------------+ -| Copyright | ``license`` | -+------------------------------------------+----------------------------------------------+ -| Url | ``url`` | -+------------------------------------------+----------------------------------------------+ -| %description (section) | ``long_description`` | -+------------------------------------------+----------------------------------------------+ - -Additionally, there are many options in :file:`.spec` files that don't have -corresponding options in the setup script. Most of these are handled through -options to the :command:`bdist_rpm` command as follows: - -+-------------------------------+-----------------------------+-------------------------+ -| RPM :file:`.spec` file option | :command:`bdist_rpm` option | default value | -| or section | | | -+===============================+=============================+=========================+ -| Release | ``release`` | "1" | -+-------------------------------+-----------------------------+-------------------------+ -| Group | ``group`` | "Development/Libraries" | -+-------------------------------+-----------------------------+-------------------------+ -| Vendor | ``vendor`` | (see above) | -+-------------------------------+-----------------------------+-------------------------+ -| Packager | ``packager`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Provides | ``provides`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Requires | ``requires`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Conflicts | ``conflicts`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Obsoletes | ``obsoletes`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Distribution | ``distribution_name`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| BuildRequires | ``build_requires`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Icon | ``icon`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ - -Obviously, supplying even a few of these options on the command-line would be -tedious and error-prone, so it's usually best to put them in the setup -configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If -you distribute or package many Python module distributions, you might want to -put options that apply to all of them in your personal Distutils configuration -file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable -this file, you can pass the :option:`!--no-user-cfg` option to :file:`setup.py`. - -There are three steps to building a binary RPM package, all of which are -handled automatically by the Distutils: - -#. create a :file:`.spec` file, which describes the package (analogous to the - Distutils setup script; in fact, much of the information in the setup script - winds up in the :file:`.spec` file) - -#. create the source RPM - -#. create the "binary" RPM (which may or may not contain binary code, depending - on whether your module distribution contains Python extensions) - -Normally, RPM bundles the last two steps together; when you use the Distutils, -all three steps are typically bundled together. - -If you wish, you can separate these three steps. You can use the -:option:`!--spec-only` option to make :command:`bdist_rpm` just create the -:file:`.spec` file and exit; in this case, the :file:`.spec` file will be -written to the "distribution directory"---normally :file:`dist/`, but -customizable with the :option:`!--dist-dir` option. (Normally, the :file:`.spec` -file winds up deep in the "build tree," in a temporary directory created by -:command:`bdist_rpm`.) - -.. % \XXX{this isn't implemented yet---is it needed?!} -.. % You can also specify a custom \file{.spec} file with the -.. % \longprogramopt{spec-file} option; used in conjunction with -.. % \longprogramopt{spec-only}, this gives you an opportunity to customize -.. % the \file{.spec} file manually: -.. % -.. % \ begin{verbatim} -.. % > python setup.py bdist_rpm --spec-only -.. % # ...edit dist/FooBar-1.0.spec -.. % > python setup.py bdist_rpm --spec-file=dist/FooBar-1.0.spec -.. % \ end{verbatim} -.. % -.. % (Although a better way to do this is probably to override the standard -.. % \command{bdist\_rpm} command with one that writes whatever else you want -.. % to the \file{.spec} file.) - - -.. _creating-wininst: - -Creating Windows Installers -=========================== - -Executable installers are the natural format for binary distributions on -Windows. They display a nice graphical user interface, display some information -about the module distribution to be installed taken from the metadata in the -setup script, let the user select a few options, and start or cancel the -installation. - -Since the metadata is taken from the setup script, creating Windows installers -is usually as easy as running:: - - python setup.py bdist_wininst - -or the :command:`bdist` command with the :option:`!--formats` option:: - - python setup.py bdist --formats=wininst - -If you have a pure module distribution (only containing pure Python modules and -packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix -platforms or Mac OS X. - -If you have a non-pure distribution, the extensions can only be created on a -Windows platform, and will be Python version dependent. The installer filename -will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You -have to create a separate installer for every Python version you want to -support. - -The installer will try to compile pure modules into :term:`bytecode` after installation -on the target system in normal and optimizing mode. If you don't want this to -happen for some reason, you can run the :command:`bdist_wininst` command with -the :option:`!--no-target-compile` and/or the :option:`!--no-target-optimize` -option. - -By default the installer will display the cool "Python Powered" logo when it is -run, but you can also supply your own 152x261 bitmap which must be a Windows -:file:`.bmp` file with the :option:`!--bitmap` option. - -The installer will also display a large title on the desktop background window -when it is run, which is constructed from the name of your distribution and the -version number. This can be changed to another text by using the -:option:`!--title` option. - -The installer file will be written to the "distribution directory" --- normally -:file:`dist/`, but customizable with the :option:`!--dist-dir` option. - -.. _cross-compile-windows: - -Cross-compiling on Windows -========================== - -Starting with Python 2.6, distutils is capable of cross-compiling between -Windows platforms. In practice, this means that with the correct tools -installed, you can use a 32bit version of Windows to create 64bit extensions -and vice-versa. - -To build for an alternate platform, specify the :option:`!--plat-name` option -to the build command. Valid values are currently 'win32', 'win-amd64' and -'win-ia64'. For example, on a 32bit version of Windows, you could execute:: - - python setup.py build --plat-name=win-amd64 - -to build a 64bit version of your extension. The Windows Installers also -support this option, so the command:: - - python setup.py build --plat-name=win-amd64 bdist_wininst - -would create a 64bit installation executable on your 32bit version of Windows. - -To cross-compile, you must download the Python source code and cross-compile -Python itself for the platform you are targeting - it is not possible from a -binary installation of Python (as the .lib etc file for other platforms are -not included.) In practice, this means the user of a 32 bit operating -system will need to use Visual Studio 2008 to open the -:file:`PCBuild/PCbuild.sln` solution in the Python source tree and build the -"x64" configuration of the 'pythoncore' project before cross-compiling -extensions is possible. - -Note that by default, Visual Studio 2008 does not install 64bit compilers or -tools. You may need to reexecute the Visual Studio setup process and select -these tools (using Control Panel->[Add/Remove] Programs is a convenient way to -check or modify your existing install.) - -.. _postinstallation-script: - -The Postinstallation script ---------------------------- - -Starting with Python 2.3, a postinstallation script can be specified with the -:option:`!--install-script` option. The basename of the script must be -specified, and the script filename must also be listed in the scripts argument -to the setup function. - -This script will be run at installation time on the target system after all the -files have been copied, with ``argv[1]`` set to :option:`!-install`, and again at -uninstallation time before the files are removed with ``argv[1]`` set to -:option:`!-remove`. - -The installation script runs embedded in the windows installer, every output -(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be -displayed in the GUI after the script has finished. - -Some functions especially useful in this context are available as additional -built-in functions in the installation script. - - -.. function:: directory_created(path) - file_created(path) - - These functions should be called when a directory or file is created by the - postinstall script at installation time. It will register *path* with the - uninstaller, so that it will be removed when the distribution is uninstalled. - To be safe, directories are only removed if they are empty. - - -.. function:: get_special_folder_path(csidl_string) - - This function can be used to retrieve special folder locations on Windows like - the Start Menu or the Desktop. It returns the full path to the folder. - *csidl_string* must be one of the following strings:: - - "CSIDL_APPDATA" - - "CSIDL_COMMON_STARTMENU" - "CSIDL_STARTMENU" - - "CSIDL_COMMON_DESKTOPDIRECTORY" - "CSIDL_DESKTOPDIRECTORY" - - "CSIDL_COMMON_STARTUP" - "CSIDL_STARTUP" - - "CSIDL_COMMON_PROGRAMS" - "CSIDL_PROGRAMS" - - "CSIDL_FONTS" - - If the folder cannot be retrieved, :exc:`OSError` is raised. - - Which folders are available depends on the exact Windows version, and probably - also the configuration. For details refer to Microsoft's documentation of the - :c:func:`SHGetSpecialFolderPath` function. - - -.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]]) - - This function creates a shortcut. *target* is the path to the program to be - started by the shortcut. *description* is the description of the shortcut. - *filename* is the title of the shortcut that the user will see. *arguments* - specifies the command line arguments, if any. *workdir* is the working directory - for the program. *iconpath* is the file containing the icon for the shortcut, - and *iconindex* is the index of the icon in the file *iconpath*. Again, for - details consult the Microsoft documentation for the :class:`IShellLink` - interface. - - -Vista User Access Control (UAC) -=============================== - -Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-control` -option. The default is 'none' (meaning no UAC handling is done), and other -valid values are 'auto' (meaning prompt for UAC elevation if Python was -installed for all users) and 'force' (meaning always prompt for elevation). diff --git a/third_party/python/Doc/distutils/commandref.rst b/third_party/python/Doc/distutils/commandref.rst deleted file mode 100644 index 6a2ac960f..000000000 --- a/third_party/python/Doc/distutils/commandref.rst +++ /dev/null @@ -1,104 +0,0 @@ -.. _reference: - -***************** -Command Reference -***************** - -.. % \section{Building modules: the \protect\command{build} command family} -.. % \label{build-cmds} -.. % \subsubsection{\protect\command{build}} -.. % \label{build-cmd} -.. % \subsubsection{\protect\command{build\_py}} -.. % \label{build-py-cmd} -.. % \subsubsection{\protect\command{build\_ext}} -.. % \label{build-ext-cmd} -.. % \subsubsection{\protect\command{build\_clib}} -.. % \label{build-clib-cmd} - - -.. _install-cmd: - -Installing modules: the :command:`install` command family -========================================================= - -The install command ensures that the build commands have been run and then runs -the subcommands :command:`install_lib`, :command:`install_data` and -:command:`install_scripts`. - -.. % \subsubsection{\protect\command{install\_lib}} -.. % \label{install-lib-cmd} - - -.. _install-data-cmd: - -:command:`install_data` ------------------------ - -This command installs all data files provided with the distribution. - - -.. _install-scripts-cmd: - -:command:`install_scripts` --------------------------- - -This command installs all (Python) scripts in the distribution. - -.. % \subsection{Cleaning up: the \protect\command{clean} command} -.. % \label{clean-cmd} - - -.. _sdist-cmd: - -Creating a source distribution: the :command:`sdist` command -============================================================ - -.. XXX fragment moved down from above: needs context! - -The manifest template commands are: - -+-------------------------------------------+-----------------------------------------------+ -| Command | Description | -+===========================================+===============================================+ -| :command:`include pat1 pat2 ...` | include all files matching any of the listed | -| | patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed | -| | patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of | -| ...` | the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of | -| ...` | the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree | -| | matching --- & any of the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree | -| | matching --- & any of the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`prune dir` | exclude all files under *dir* | -+-------------------------------------------+-----------------------------------------------+ -| :command:`graft dir` | include all files under *dir* | -+-------------------------------------------+-----------------------------------------------+ - -The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of -regular filename characters, ``?`` matches any single regular filename -character, and ``[range]`` matches any of the characters in *range* (e.g., -``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename -character" is platform-specific: on Unix it is anything except slash; on Windows -anything except backslash or colon. - -.. XXX Windows support not there yet - -.. % \section{Creating a built distribution: the -.. % \protect\command{bdist} command family} -.. % \label{bdist-cmds} - -.. % \subsection{\protect\command{bdist}} -.. % \subsection{\protect\command{bdist\_dumb}} -.. % \subsection{\protect\command{bdist\_rpm}} -.. % \subsection{\protect\command{bdist\_wininst}} - - diff --git a/third_party/python/Doc/distutils/configfile.rst b/third_party/python/Doc/distutils/configfile.rst deleted file mode 100644 index 0874d05fe..000000000 --- a/third_party/python/Doc/distutils/configfile.rst +++ /dev/null @@ -1,142 +0,0 @@ -.. _setup-config: - -************************************ -Writing the Setup Configuration File -************************************ - -Often, it's not possible to write down everything needed to build a distribution -*a priori*: you may need to get some information from the user, or from the -user's system, in order to proceed. As long as that information is fairly -simple---a list of directories to search for C header files or libraries, for -example---then providing a configuration file, :file:`setup.cfg`, for users to -edit is a cheap and easy way to solicit it. Configuration files also let you -provide default values for any command option, which the installer can then -override either on the command-line or by editing the config file. - -The setup configuration file is a useful middle-ground between the setup -script---which, ideally, would be opaque to installers [#]_---and the command-line to -the setup script, which is outside of your control and entirely up to the -installer. In fact, :file:`setup.cfg` (and any other Distutils configuration -files present on the target system) are processed after the contents of the -setup script, but before the command-line. This has several useful -consequences: - -.. % (If you have more advanced needs, such as determining which extensions -.. % to build based on what capabilities are present on the target system, -.. % then you need the Distutils ``auto-configuration'' facility. This -.. % started to appear in Distutils 0.9 but, as of this writing, isn't mature -.. % or stable enough yet for real-world use.) - -* installers can override some of what you put in :file:`setup.py` by editing - :file:`setup.cfg` - -* you can provide non-standard defaults for options that are not easily set in - :file:`setup.py` - -* installers can override anything in :file:`setup.cfg` using the command-line - options to :file:`setup.py` - -The basic syntax of the configuration file is simple: - -.. code-block:: ini - - [command] - option=value - ... - -where *command* is one of the Distutils commands (e.g. :command:`build_py`, -:command:`install`), and *option* is one of the options that command supports. -Any number of options can be supplied for each command, and any number of -command sections can be included in the file. Blank lines are ignored, as are -comments, which run from a ``'#'`` character until the end of the line. Long -option values can be split across multiple lines simply by indenting the -continuation lines. - -You can find out the list of options supported by a particular command with the -universal :option:`!--help` option, e.g. - -.. code-block:: shell-session - - $ python setup.py --help build_ext - [...] - Options for 'build_ext' command: - --build-lib (-b) directory for compiled extension modules - --build-temp (-t) directory for temporary files (build by-products) - --inplace (-i) ignore build-lib and put compiled extensions into the - source directory alongside your pure Python modules - --include-dirs (-I) list of directories to search for header files - --define (-D) C preprocessor macros to define - --undef (-U) C preprocessor macros to undefine - --swig-opts list of SWIG command line options - [...] - -Note that an option spelled :option:`!--foo-bar` on the command-line is spelled -``foo_bar`` in configuration files. - -.. _distutils-build-ext-inplace: - -For example, say you want your extensions to be built "in-place"---that is, you -have an extension :mod:`pkg.ext`, and you want the compiled extension file -(:file:`ext.so` on Unix, say) to be put in the same source directory as your -pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the -:option:`!--inplace` option on the command-line to ensure this: - -.. code-block:: sh - - python setup.py build_ext --inplace - -But this requires that you always specify the :command:`build_ext` command -explicitly, and remember to provide :option:`!--inplace`. An easier way is to -"set and forget" this option, by encoding it in :file:`setup.cfg`, the -configuration file for this distribution: - -.. code-block:: ini - - [build_ext] - inplace=1 - -This will affect all builds of this module distribution, whether or not you -explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in -your source distribution, it will also affect end-user builds---which is -probably a bad idea for this option, since always building extensions in-place -would break installation of the module distribution. In certain peculiar cases, -though, modules are built right in their installation directory, so this is -conceivably a useful ability. (Distributing extensions that expect to be built -in their installation directory is almost always a bad idea, though.) - -Another example: certain commands take a lot of options that don't change from -run to run; for example, :command:`bdist_rpm` needs to know everything required -to generate a "spec" file for creating an RPM distribution. Some of this -information comes from the setup script, and some is automatically generated by -the Distutils (such as the list of files installed). But some of it has to be -supplied as options to :command:`bdist_rpm`, which would be very tedious to do -on the command-line for every run. Hence, here is a snippet from the Distutils' -own :file:`setup.cfg`: - -.. code-block:: ini - - [bdist_rpm] - release = 1 - packager = Greg Ward <gward@python.net> - doc_files = CHANGES.txt - README.txt - USAGE.txt - doc/ - examples/ - -Note that the ``doc_files`` option is simply a whitespace-separated string -split across multiple lines for readability. - - -.. seealso:: - - :ref:`inst-config-syntax` in "Installing Python Modules" - More information on the configuration files is available in the manual for - system administrators. - - -.. rubric:: Footnotes - -.. [#] This ideal probably won't be achieved until auto-configuration is fully - supported by the Distutils. - diff --git a/third_party/python/Doc/distutils/examples.rst b/third_party/python/Doc/distutils/examples.rst deleted file mode 100644 index 4e2761d8a..000000000 --- a/third_party/python/Doc/distutils/examples.rst +++ /dev/null @@ -1,338 +0,0 @@ -.. _examples: - -******** -Examples -******** - -This chapter provides a number of basic examples to help get started with -distutils. Additional information about using distutils can be found in the -Distutils Cookbook. - - -.. seealso:: - - `Distutils Cookbook <https://wiki.python.org/moin/Distutils/Cookbook>`_ - Collection of recipes showing how to achieve more control over distutils. - - -.. _pure-mod: - -Pure Python distribution (by module) -==================================== - -If you're just distributing a couple of modules, especially if they don't live -in a particular package, you can specify them individually using the -``py_modules`` option in the setup script. - -In the simplest case, you'll have two files to worry about: a setup script and -the single module you're distributing, :file:`foo.py` in this example:: - - <root>/ - setup.py - foo.py - -(In all diagrams in this section, *<root>* will refer to the distribution root -directory.) A minimal setup script to describe this situation would be:: - - from distutils.core import setup - setup(name='foo', - version='1.0', - py_modules=['foo'], - ) - -Note that the name of the distribution is specified independently with the -``name`` option, and there's no rule that says it has to be the same as -the name of the sole module in the distribution (although that's probably a good -convention to follow). However, the distribution name is used to generate -filenames, so you should stick to letters, digits, underscores, and hyphens. - -Since ``py_modules`` is a list, you can of course specify multiple -modules, eg. if you're distributing modules :mod:`foo` and :mod:`bar`, your -setup might look like this:: - - <root>/ - setup.py - foo.py - bar.py - -and the setup script might be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - py_modules=['foo', 'bar'], - ) - -You can put module source files into another directory, but if you have enough -modules to do that, it's probably easier to specify modules by package rather -than listing them individually. - - -.. _pure-pkg: - -Pure Python distribution (by package) -===================================== - -If you have more than a couple of modules to distribute, especially if they are -in multiple packages, it's probably easier to specify whole packages rather than -individual modules. This works even if your modules are not in a package; you -can just tell the Distutils to process modules from the root package, and that -works the same as any other package (except that you don't have to have an -:file:`__init__.py` file). - -The setup script from the last example could also be written as :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=[''], - ) - -(The empty string stands for the root package.) - -If those two files are moved into a subdirectory, but remain in the root -package, e.g.:: - - <root>/ - setup.py - src/ foo.py - bar.py - -then you would still specify the root package, but you have to tell the -Distutils where source files in the root package live:: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'': 'src'}, - packages=[''], - ) - -More typically, though, you will want to distribute multiple modules in the same -package (or in sub-packages). For example, if the :mod:`foo` and :mod:`bar` -modules belong in package :mod:`foobar`, one way to layout your source tree is -:: - - <root>/ - setup.py - foobar/ - __init__.py - foo.py - bar.py - -This is in fact the default layout expected by the Distutils, and the one that -requires the least work to describe in your setup script:: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=['foobar'], - ) - -If you want to put modules in directories not named for their package, then you -need to use the ``package_dir`` option again. For example, if the -:file:`src` directory holds modules in the :mod:`foobar` package:: - - <root>/ - setup.py - src/ - __init__.py - foo.py - bar.py - -an appropriate setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'foobar': 'src'}, - packages=['foobar'], - ) - -Or, you might put modules from your main package right in the distribution -root:: - - <root>/ - setup.py - __init__.py - foo.py - bar.py - -in which case your setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'foobar': ''}, - packages=['foobar'], - ) - -(The empty string also stands for the current directory.) - -If you have sub-packages, they must be explicitly listed in ``packages``, -but any entries in ``package_dir`` automatically extend to sub-packages. -(In other words, the Distutils does *not* scan your source tree, trying to -figure out which directories correspond to Python packages by looking for -:file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: - - <root>/ - setup.py - foobar/ - __init__.py - foo.py - bar.py - subfoo/ - __init__.py - blah.py - -then the corresponding setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=['foobar', 'foobar.subfoo'], - ) - - -.. _single-ext: - -Single extension module -======================= - -Extension modules are specified using the ``ext_modules`` option. -``package_dir`` has no effect on where extension source files are found; -it only affects the source for pure Python modules. The simplest case, a -single extension module in a single C source file, is:: - - <root>/ - setup.py - foo.c - -If the :mod:`foo` extension belongs in the root package, the setup script for -this could be :: - - from distutils.core import setup - from distutils.extension import Extension - setup(name='foobar', - version='1.0', - ext_modules=[Extension('foo', ['foo.c'])], - ) - -If the extension actually belongs in a package, say :mod:`foopkg`, then - -With exactly the same source tree layout, this extension can be put in the -:mod:`foopkg` package simply by changing the name of the extension:: - - from distutils.core import setup - from distutils.extension import Extension - setup(name='foobar', - version='1.0', - ext_modules=[Extension('foopkg.foo', ['foo.c'])], - ) - -Checking a package -================== - -The ``check`` command allows you to verify if your package meta-data -meet the minimum requirements to build a distribution. - -To run it, just call it using your :file:`setup.py` script. If something is -missing, ``check`` will display a warning. - -Let's take an example with a simple script:: - - from distutils.core import setup - - setup(name='foobar') - -Running the ``check`` command will display some warnings: - -.. code-block:: shell-session - - $ python setup.py check - running check - warning: check: missing required meta-data: version, url - warning: check: missing meta-data: either (author and author_email) or - (maintainer and maintainer_email) must be supplied - - -If you use the reStructuredText syntax in the ``long_description`` field and -`docutils`_ is installed you can check if the syntax is fine with the -``check`` command, using the ``restructuredtext`` option. - -For example, if the :file:`setup.py` script is changed like this:: - - from distutils.core import setup - - desc = """\ - My description - ============== - - This is the description of the ``foobar`` package. - """ - - setup(name='foobar', version='1', author='tarek', - author_email='tarek@ziade.org', - url='http://example.com', long_description=desc) - -Where the long description is broken, ``check`` will be able to detect it -by using the :mod:`docutils` parser: - -.. code-block:: shell-session - - $ python setup.py check --restructuredtext - running check - warning: check: Title underline too short. (line 2) - warning: check: Could not finish the parsing. - -Reading the metadata -===================== - -The :func:`distutils.core.setup` function provides a command-line interface -that allows you to query the metadata fields of a project through the -``setup.py`` script of a given project: - -.. code-block:: shell-session - - $ python setup.py --name - distribute - -This call reads the ``name`` metadata by running the -:func:`distutils.core.setup` function. Although, when a source or binary -distribution is created with Distutils, the metadata fields are written -in a static file called :file:`PKG-INFO`. When a Distutils-based project is -installed in Python, the :file:`PKG-INFO` file is copied alongside the modules -and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, -where ``NAME`` is the name of the project, ``VERSION`` its version as defined -in the Metadata, and ``pyX.X`` the major and minor version of Python like -``2.7`` or ``3.2``. - -You can read back this static file, by using the -:class:`distutils.dist.DistributionMetadata` class and its -:func:`read_pkg_file` method:: - - >>> from distutils.dist import DistributionMetadata - >>> metadata = DistributionMetadata() - >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) - >>> metadata.name - 'distribute' - >>> metadata.version - '0.6.8' - >>> metadata.description - 'Easily download, build, install, upgrade, and uninstall Python packages' - -Notice that the class can also be instantiated with a metadata file path to -loads its values:: - - >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' - >>> DistributionMetadata(pkg_info_path).name - 'distribute' - - -.. % \section{Multiple extension modules} -.. % \label{multiple-ext} - -.. % \section{Putting it all together} - - -.. _docutils: http://docutils.sourceforge.net diff --git a/third_party/python/Doc/distutils/extending.rst b/third_party/python/Doc/distutils/extending.rst deleted file mode 100644 index 501fd7c56..000000000 --- a/third_party/python/Doc/distutils/extending.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. _extending-distutils: - -******************* -Extending Distutils -******************* - -Distutils can be extended in various ways. Most extensions take the form of new -commands or replacements for existing commands. New commands may be written to -support new types of platform-specific packaging, for example, while -replacements for existing commands may be made to modify details of how the -command operates on a package. - -Most extensions of the distutils are made within :file:`setup.py` scripts that -want to modify existing commands; many simply add a few file extensions that -should be copied into packages in addition to :file:`.py` files as a -convenience. - -Most distutils command implementations are subclasses of the -:class:`distutils.cmd.Command` class. New commands may directly inherit from -:class:`Command`, while replacements often derive from :class:`Command` -indirectly, directly subclassing the command they are replacing. Commands are -required to derive from :class:`Command`. - -.. % \section{Extending existing commands} -.. % \label{extend-existing} - -.. % \section{Writing new commands} -.. % \label{new-commands} -.. % \XXX{Would an uninstall command be a good example here?} - - -Integrating new commands -======================== - -There are different ways to integrate new command implementations into -distutils. The most difficult is to lobby for the inclusion of the new features -in distutils itself, and wait for (and require) a version of Python that -provides that support. This is really hard for many reasons. - -The most common, and possibly the most reasonable for most needs, is to include -the new implementations with your :file:`setup.py` script, and cause the -:func:`distutils.core.setup` function use them:: - - from distutils.command.build_py import build_py as _build_py - from distutils.core import setup - - class build_py(_build_py): - """Specialized Python source builder.""" - - # implement whatever needs to be different... - - setup(cmdclass={'build_py': build_py}, - ...) - -This approach is most valuable if the new implementations must be used to use a -particular package, as everyone interested in the package will need to have the -new command implementation. - -Beginning with Python 2.4, a third option is available, intended to allow new -commands to be added which can support existing :file:`setup.py` scripts without -requiring modifications to the Python installation. This is expected to allow -third-party extensions to provide support for additional packaging systems, but -the commands can be used for anything distutils commands can be used for. A new -configuration option, ``command_packages`` (command-line option -:option:`!--command-packages`), can be used to specify additional packages to be -searched for modules implementing commands. Like all distutils options, this -can be specified on the command line or in a configuration file. This option -can only be set in the ``[global]`` section of a configuration file, or before -any commands on the command line. If set in a configuration file, it can be -overridden from the command line; setting it to an empty string on the command -line causes the default to be used. This should never be set in a configuration -file provided with a package. - -This new option can be used to add any number of packages to the list of -packages searched for command implementations; multiple package names should be -separated by commas. When not specified, the search is only performed in the -:mod:`distutils.command` package. When :file:`setup.py` is run with the option -``--command-packages distcmds,buildcmds``, however, the packages -:mod:`distutils.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched -in that order. New commands are expected to be implemented in modules of the -same name as the command by classes sharing the same name. Given the example -command line option above, the command :command:`bdist_openpkg` could be -implemented by the class :class:`distcmds.bdist_openpkg.bdist_openpkg` or -:class:`buildcmds.bdist_openpkg.bdist_openpkg`. - - -Adding new distribution types -============================= - -Commands that create distributions (files in the :file:`dist/` directory) need -to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that -:command:`upload` can upload it to PyPI. The *filename* in the pair contains no -path information, only the name of the file itself. In dry-run mode, pairs -should still be added to represent what would have been created. - - diff --git a/third_party/python/Doc/distutils/index.rst b/third_party/python/Doc/distutils/index.rst deleted file mode 100644 index c565bcc56..000000000 --- a/third_party/python/Doc/distutils/index.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _distutils-index: - -############################################## - Distributing Python Modules (Legacy version) -############################################## - -:Authors: Greg Ward, Anthony Baxter -:Email: distutils-sig@python.org - -.. seealso:: - - :ref:`distributing-index` - The up to date module distribution documentations - -This document describes the Python Distribution Utilities ("Distutils") from -the module developer's point of view, describing how to use the Distutils to -make Python modules and extensions easily available to a wider audience with -very little overhead for build/release/install mechanics. - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section <https://packaging.python.org/en/latest/current/>`__ - in the Python Packaging User Guide for more information. - -.. toctree:: - :maxdepth: 2 - :numbered: - - introduction.rst - setupscript.rst - configfile.rst - sourcedist.rst - builtdist.rst - packageindex.rst - examples.rst - extending.rst - commandref.rst - apiref.rst diff --git a/third_party/python/Doc/distutils/introduction.rst b/third_party/python/Doc/distutils/introduction.rst deleted file mode 100644 index 7721484fe..000000000 --- a/third_party/python/Doc/distutils/introduction.rst +++ /dev/null @@ -1,212 +0,0 @@ -.. _distutils-intro: - -**************************** -An Introduction to Distutils -**************************** - -This document covers using the Distutils to distribute your Python modules, -concentrating on the role of developer/distributor: if you're looking for -information on installing Python modules, you should refer to the -:ref:`install-index` chapter. - - -.. _distutils-concepts: - -Concepts & Terminology -====================== - -Using the Distutils is quite simple, both for module developers and for -users/administrators installing third-party modules. As a developer, your -responsibilities (apart from writing solid, well-documented and well-tested -code, of course!) are: - -* write a setup script (:file:`setup.py` by convention) - -* (optional) write a setup configuration file - -* create a source distribution - -* (optional) create one or more built (binary) distributions - -Each of these tasks is covered in this document. - -Not all module developers have access to a multitude of platforms, so it's not -always feasible to expect them to create a multitude of built distributions. It -is hoped that a class of intermediaries, called *packagers*, will arise to -address this need. Packagers will take source distributions released by module -developers, build them on one or more platforms, and release the resulting built -distributions. Thus, users on the most popular platforms will be able to -install most popular Python module distributions in the most natural way for -their platform, without having to run a single setup script or compile a line of -code. - - -.. _distutils-simple-example: - -A Simple Example -================ - -The setup script is usually quite simple, although since it's written in Python, -there are no arbitrary limits to what you can do with it, though you should be -careful about putting arbitrarily expensive operations in your setup script. -Unlike, say, Autoconf-style configure scripts, the setup script may be run -multiple times in the course of building and installing your module -distribution. - -If all you want to do is distribute a module called :mod:`foo`, contained in a -file :file:`foo.py`, then your setup script can be as simple as this:: - - from distutils.core import setup - setup(name='foo', - version='1.0', - py_modules=['foo'], - ) - -Some observations: - -* most information that you supply to the Distutils is supplied as keyword - arguments to the :func:`setup` function - -* those keyword arguments fall into two categories: package metadata (name, - version number) and information about what's in the package (a list of pure - Python modules, in this case) - -* modules are specified by module name, not filename (the same will hold true - for packages and extensions) - -* it's recommended that you supply a little more metadata, in particular your - name, email address and a URL for the project (see section :ref:`setup-script` - for an example) - -To create a source distribution for this module, you would create a setup -script, :file:`setup.py`, containing the above code, and run this command from a -terminal:: - - python setup.py sdist - -For Windows, open a command prompt window (:menuselection:`Start --> -Accessories`) and change the command to:: - - setup.py sdist - -:command:`sdist` will create an archive file (e.g., tarball on Unix, ZIP file on Windows) -containing your setup script :file:`setup.py`, and your module :file:`foo.py`. -The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and -will unpack into a directory :file:`foo-1.0`. - -If an end-user wishes to install your :mod:`foo` module, all they have to do is -download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the -:file:`foo-1.0` directory---run :: - - python setup.py install - -which will ultimately copy :file:`foo.py` to the appropriate directory for -third-party modules in their Python installation. - -This simple example demonstrates some fundamental concepts of the Distutils. -First, both developers and installers have the same basic user interface, i.e. -the setup script. The difference is which Distutils *commands* they use: the -:command:`sdist` command is almost exclusively for module developers, while -:command:`install` is more often for installers (although most developers will -want to install their own code occasionally). - -If you want to make things really easy for your users, you can create one or -more built distributions for them. For instance, if you are running on a -Windows machine, and want to make things easy for other Windows users, you can -create an executable installer (the most appropriate type of built distribution -for this platform) with the :command:`bdist_wininst` command. For example:: - - python setup.py bdist_wininst - -will create an executable installer, :file:`foo-1.0.win32.exe`, in the current -directory. - -Other useful built distribution formats are RPM, implemented by the -:command:`bdist_rpm` command, Solaris :program:`pkgtool` -(:command:`bdist_pkgtool`), and HP-UX :program:`swinstall` -(:command:`bdist_sdux`). For example, the following command will create an RPM -file called :file:`foo-1.0.noarch.rpm`:: - - python setup.py bdist_rpm - -(The :command:`bdist_rpm` command uses the :command:`rpm` executable, therefore -this has to be run on an RPM-based system such as Red Hat Linux, SuSE Linux, or -Mandrake Linux.) - -You can find out what distribution formats are available at any time by running -:: - - python setup.py bdist --help-formats - - -.. _python-terms: - -General Python terminology -========================== - -If you're reading this document, you probably have a good idea of what modules, -extensions, and so forth are. Nevertheless, just to be sure that everyone is -operating from a common starting point, we offer the following glossary of -common Python terms: - -module - the basic unit of code reusability in Python: a block of code imported by some - other code. Three types of modules concern us here: pure Python modules, - extension modules, and packages. - -pure Python module - a module written in Python and contained in a single :file:`.py` file (and - possibly associated :file:`.pyc` files). Sometimes referred to as a - "pure module." - -extension module - a module written in the low-level language of the Python implementation: C/C++ - for Python, Java for Jython. Typically contained in a single dynamically - loadable pre-compiled file, e.g. a shared object (:file:`.so`) file for Python - extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python - extensions on Windows, or a Java class file for Jython extensions. (Note that - currently, the Distutils only handles C/C++ extensions for Python.) - -package - a module that contains other modules; typically contained in a directory in the - filesystem and distinguished from other directories by the presence of a file - :file:`__init__.py`. - -root package - the root of the hierarchy of packages. (This isn't really a package, since it - doesn't have an :file:`__init__.py` file. But we have to call it something.) - The vast majority of the standard library is in the root package, as are many - small, standalone third-party modules that don't belong to a larger module - collection. Unlike regular packages, modules in the root package can be found in - many directories: in fact, every directory listed in ``sys.path`` contributes - modules to the root package. - - -.. _distutils-term: - -Distutils-specific terminology -============================== - -The following terms apply more specifically to the domain of distributing Python -modules using the Distutils: - -module distribution - a collection of Python modules distributed together as a single downloadable - resource and meant to be installed *en masse*. Examples of some well-known - module distributions are NumPy, SciPy, Pillow, - or mxBase. (This would be called a *package*, except that term is - already taken in the Python context: a single module distribution may contain - zero, one, or many Python packages.) - -pure module distribution - a module distribution that contains only pure Python modules and packages. - Sometimes referred to as a "pure distribution." - -non-pure module distribution - a module distribution that contains at least one extension module. Sometimes - referred to as a "non-pure distribution." - -distribution root - the top-level directory of your source tree (or source distribution); the - directory where :file:`setup.py` exists. Generally :file:`setup.py` will be - run from this directory. diff --git a/third_party/python/Doc/distutils/packageindex.rst b/third_party/python/Doc/distutils/packageindex.rst deleted file mode 100644 index 50cb74f2b..000000000 --- a/third_party/python/Doc/distutils/packageindex.rst +++ /dev/null @@ -1,253 +0,0 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _package-index: - -******************************* -The Python Package Index (PyPI) -******************************* - -The `Python Package Index (PyPI)`_ stores :ref:`meta-data <meta-data>` -describing distributions packaged with distutils, as well as package data like -distribution files if a package author wishes. - -Distutils provides the :command:`register` and :command:`upload` commands for -pushing meta-data and distribution files to PyPI, respectively. See -:ref:`package-commands` for information on these commands. - - -PyPI overview -============= - -PyPI lets you submit any number of versions of your distribution to the index. -If you alter the meta-data for a particular version, you can submit it again -and the index will be updated. - -PyPI holds a record for each (name, version) combination submitted. The first -user to submit information for a given name is designated the Owner of that -name. Changes can be submitted through the :command:`register` command or -through the web interface. Owners can designate other users as Owners or -Maintainers. Maintainers can edit the package information, but not designate -new Owners or Maintainers. - -By default PyPI displays only the newest version of a given package. The web -interface lets one change this default behavior and manually select which -versions to display and hide. - -For each version, PyPI displays a home page. The home page is created from -the ``long_description`` which can be submitted via the :command:`register` -command. See :ref:`package-display` for more information. - - -.. _package-commands: - -Distutils commands -================== - -Distutils exposes two commands for submitting package data to PyPI: the -:ref:`register <package-register>` command for submitting meta-data to PyPI -and the :ref:`upload <package-upload>` command for submitting distribution -files. Both commands read configuration data from a special file called a -:ref:`.pypirc file <pypirc>`. - - -.. _package-register: - -The ``register`` command ------------------------- - -The distutils command :command:`register` is used to submit your distribution's -meta-data to an index server. It is invoked as follows:: - - python setup.py register - -Distutils will respond with the following prompt:: - - running register - We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit - Your selection [default 1]: - -Note: if your username and password are saved locally, you will not see this -menu. Also, refer to :ref:`pypirc` for how to store your credentials in a -:file:`.pypirc` file. - -If you have not registered with PyPI, then you will need to do so now. You -should choose option 2, and enter your details as required. Soon after -submitting your details, you will receive an email which will be used to confirm -your registration. - -Once you are registered, you may choose option 1 from the menu. You will be -prompted for your PyPI username and password, and :command:`register` will then -submit your meta-data to the index. - -See :ref:`package-cmdoptions` for options to the :command:`register` command. - - -.. _package-upload: - -The ``upload`` command ----------------------- - -The distutils command :command:`upload` pushes the distribution files to PyPI. - -The command is invoked immediately after building one or more distribution -files. For example, the command :: - - python setup.py sdist bdist_wininst upload - -will cause the source distribution and the Windows installer to be uploaded to -PyPI. Note that these will be uploaded even if they are built using an earlier -invocation of :file:`setup.py`, but that only distributions named on the command -line for the invocation including the :command:`upload` command are uploaded. - -If a :command:`register` command was previously called in the same command, -and if the password was entered in the prompt, :command:`upload` will reuse the -entered password. This is useful if you do not want to store a password in -clear text in a :file:`.pypirc` file. - -You can use the ``--sign`` option to tell :command:`upload` to sign each -uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must -be available for execution on the system :envvar:`PATH`. You can also specify -which key to use for signing using the ``--identity=name`` option. - -See :ref:`package-cmdoptions` for additional options to the :command:`upload` -command. - - -.. _package-cmdoptions: - -Additional command options --------------------------- - -This section describes options common to both the :command:`register` and -:command:`upload` commands. - -The ``--repository`` or ``-r`` option lets you specify a PyPI server -different from the default. For example:: - - python setup.py sdist bdist_wininst upload -r https://example.com/pypi - -For convenience, a name can be used in place of the URL when the -:file:`.pypirc` file is configured to do so. For example:: - - python setup.py register -r other - -See :ref:`pypirc` for more information on defining alternate servers. - -The ``--show-response`` option displays the full response text from the PyPI -server, which is useful when debugging problems with registering and uploading. - - -.. index:: - single: .pypirc file - single: Python Package Index (PyPI); .pypirc file - -.. _pypirc: - -The ``.pypirc`` file --------------------- - -The :command:`register` and :command:`upload` commands both check for the -existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. -If this file exists, the command uses the username, password, and repository -URL configured in the file. The format of a :file:`.pypirc` file is as -follows: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - - [pypi] - repository: <repository-url> - username: <username> - password: <password> - -The *distutils* section defines an *index-servers* variable that lists the -name of all sections describing a repository. - -Each section describing a repository defines three variables: - -- *repository*, that defines the url of the PyPI server. Defaults to - ``https://upload.pypi.org/legacy/``. -- *username*, which is the registered username on the PyPI server. -- *password*, that will be used to authenticate. If omitted the user - will be prompt to type it when needed. - -If you want to define another server a new section can be created and -listed in the *index-servers* variable: - -.. code-block:: ini - - [distutils] - index-servers = - pypi - other - - [pypi] - repository: <repository-url> - username: <username> - password: <password> - - [other] - repository: https://example.com/pypi - username: <username> - password: <password> - -This allows the :command:`register` and :command:`upload` commands to be -called with the ``--repository`` option as described in -:ref:`package-cmdoptions`. - -Specifically, you might want to add the `PyPI Test Repository -<https://wiki.python.org/moin/TestPyPI>`_ to your ``.pypirc`` to facilitate -testing before doing your first upload to ``PyPI`` itself. - - -.. _package-display: - -PyPI package display -==================== - -The ``long_description`` field plays a special role at PyPI. It is used by -the server to display a home page for the registered package. - -If you use the `reStructuredText <http://docutils.sourceforge.net/rst.html>`_ -syntax for this field, PyPI will parse it and display an HTML output for -the package home page. - -The ``long_description`` field can be attached to a text file located -in the package:: - - from distutils.core import setup - - with open('README.txt') as file: - long_description = file.read() - - setup(name='Distutils', - long_description=long_description) - -In that case, :file:`README.txt` is a regular reStructuredText text file located -in the root of the package besides :file:`setup.py`. - -To prevent registering broken reStructuredText content, you can use the -:program:`rst2html` program that is provided by the :mod:`docutils` package and -check the ``long_description`` from the command line: - -.. code-block:: shell-session - - $ python setup.py --long-description | rst2html.py > output.html - -:mod:`docutils` will display a warning if there's something wrong with your -syntax. Because PyPI applies additional checks (e.g. by passing ``--no-raw`` -to ``rst2html.py`` in the command above), being able to run the command above -without warnings does not guarantee that PyPI will convert the content -successfully. - - -.. _Python Package Index (PyPI): https://pypi.org diff --git a/third_party/python/Doc/distutils/setupscript.rst b/third_party/python/Doc/distutils/setupscript.rst deleted file mode 100644 index 21d569b65..000000000 --- a/third_party/python/Doc/distutils/setupscript.rst +++ /dev/null @@ -1,695 +0,0 @@ -.. _setup-script: - -************************ -Writing the Setup Script -************************ - -The setup script is the centre of all activity in building, distributing, and -installing modules using the Distutils. The main purpose of the setup script is -to describe your module distribution to the Distutils, so that the various -commands that operate on your modules do the right thing. As we saw in section -:ref:`distutils-simple-example` above, the setup script consists mainly of a call to -:func:`setup`, and most information supplied to the Distutils by the module -developer is supplied as keyword arguments to :func:`setup`. - -Here's a slightly more involved example, which we'll follow for the next couple -of sections: the Distutils' own setup script. (Keep in mind that although the -Distutils are included with Python 1.6 and later, they also have an independent -existence so that Python 1.5.2 users can use them to install other module -distributions. The Distutils' own setup script, shown here, is used to install -the package into Python 1.5.2.) :: - - #!/usr/bin/env python - - from distutils.core import setup - - setup(name='Distutils', - version='1.0', - description='Python Distribution Utilities', - author='Greg Ward', - author_email='gward@python.net', - url='https://www.python.org/sigs/distutils-sig/', - packages=['distutils', 'distutils.command'], - ) - -There are only two differences between this and the trivial one-file -distribution presented in section :ref:`distutils-simple-example`: more metadata, and the -specification of pure Python modules by package, rather than by module. This is -important since the Distutils consist of a couple of dozen modules split into -(so far) two packages; an explicit list of every module would be tedious to -generate and difficult to maintain. For more information on the additional -meta-data, see section :ref:`meta-data`. - -Note that any pathnames (files or directories) supplied in the setup script -should be written using the Unix convention, i.e. slash-separated. The -Distutils will take care of converting this platform-neutral representation into -whatever is appropriate on your current platform before actually using the -pathname. This makes your setup script portable across operating systems, which -of course is one of the major goals of the Distutils. In this spirit, all -pathnames in this document are slash-separated. - -This, of course, only applies to pathnames given to Distutils functions. If -you, for example, use standard Python functions such as :func:`glob.glob` or -:func:`os.listdir` to specify files, you should be careful to write portable -code instead of hardcoding path separators:: - - glob.glob(os.path.join('mydir', 'subdir', '*.html')) - os.listdir(os.path.join('mydir', 'subdir')) - - -.. _listing-packages: - -Listing whole packages -====================== - -The ``packages`` option tells the Distutils to process (build, distribute, -install, etc.) all pure Python modules found in each package mentioned in the -``packages`` list. In order to do this, of course, there has to be a -correspondence between package names and directories in the filesystem. The -default correspondence is the most obvious one, i.e. package :mod:`distutils` is -found in the directory :file:`distutils` relative to the distribution root. -Thus, when you say ``packages = ['foo']`` in your setup script, you are -promising that the Distutils will find a file :file:`foo/__init__.py` (which -might be spelled differently on your system, but you get the idea) relative to -the directory where your setup script lives. If you break this promise, the -Distutils will issue a warning but still process the broken package anyway. - -If you use a different convention to lay out your source directory, that's no -problem: you just have to supply the ``package_dir`` option to tell the -Distutils about your convention. For example, say you keep all Python source -under :file:`lib`, so that modules in the "root package" (i.e., not in any -package at all) are in :file:`lib`, modules in the :mod:`foo` package are in -:file:`lib/foo`, and so forth. Then you would put :: - - package_dir = {'': 'lib'} - -in your setup script. The keys to this dictionary are package names, and an -empty package name stands for the root package. The values are directory names -relative to your distribution root. In this case, when you say ``packages = -['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists. - -Another possible convention is to put the :mod:`foo` package right in -:file:`lib`, the :mod:`foo.bar` package in :file:`lib/bar`, etc. This would be -written in the setup script as :: - - package_dir = {'foo': 'lib'} - -A ``package: dir`` entry in the ``package_dir`` dictionary implicitly -applies to all packages below *package*, so the :mod:`foo.bar` case is -automatically handled here. In this example, having ``packages = ['foo', -'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and -:file:`lib/bar/__init__.py`. (Keep in mind that although ``package_dir`` -applies recursively, you must explicitly list all packages in -``packages``: the Distutils will *not* recursively scan your source tree -looking for any directory with an :file:`__init__.py` file.) - - -.. _listing-modules: - -Listing individual modules -========================== - -For a small module distribution, you might prefer to list all modules rather -than listing packages---especially the case of a single module that goes in the -"root package" (i.e., no package at all). This simplest case was shown in -section :ref:`distutils-simple-example`; here is a slightly more involved example:: - - py_modules = ['mod1', 'pkg.mod2'] - -This describes two modules, one of them in the "root" package, the other in the -:mod:`pkg` package. Again, the default package/directory layout implies that -these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and -that :file:`pkg/__init__.py` exists as well. And again, you can override the -package/directory correspondence using the ``package_dir`` option. - - -.. _describing-extensions: - -Describing extension modules -============================ - -Just as writing Python extension modules is a bit more complicated than writing -pure Python modules, describing them to the Distutils is a bit more complicated. -Unlike pure modules, it's not enough just to list modules or packages and expect -the Distutils to go out and find the right files; you have to specify the -extension name, source file(s), and any compile/link requirements (include -directories, libraries to link with, etc.). - -.. XXX read over this section - -All of this is done through another keyword argument to :func:`setup`, the -``ext_modules`` option. ``ext_modules`` is just a list of -:class:`~distutils.core.Extension` instances, each of which describes a -single extension module. -Suppose your distribution includes a single extension, called :mod:`foo` and -implemented by :file:`foo.c`. If no additional instructions to the -compiler/linker are needed, describing this extension is quite simple:: - - Extension('foo', ['foo.c']) - -The :class:`Extension` class can be imported from :mod:`distutils.core` along -with :func:`setup`. Thus, the setup script for a module distribution that -contains only this one extension and nothing else might be:: - - from distutils.core import setup, Extension - setup(name='foo', - version='1.0', - ext_modules=[Extension('foo', ['foo.c'])], - ) - -The :class:`Extension` class (actually, the underlying extension-building -machinery implemented by the :command:`build_ext` command) supports a great deal -of flexibility in describing Python extensions, which is explained in the -following sections. - - -Extension names and packages ----------------------------- - -The first argument to the :class:`~distutils.core.Extension` constructor is -always the name of the extension, including any package names. For example, :: - - Extension('foo', ['src/foo1.c', 'src/foo2.c']) - -describes an extension that lives in the root package, while :: - - Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c']) - -describes the same extension in the :mod:`pkg` package. The source files and -resulting object code are identical in both cases; the only difference is where -in the filesystem (and therefore where in Python's namespace hierarchy) the -resulting extension lives. - -If you have a number of extensions all in the same package (or all under the -same base package), use the ``ext_package`` keyword argument to -:func:`setup`. For example, :: - - setup(..., - ext_package='pkg', - ext_modules=[Extension('foo', ['foo.c']), - Extension('subpkg.bar', ['bar.c'])], - ) - -will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to -:mod:`pkg.subpkg.bar`. - - -Extension source files ----------------------- - -The second argument to the :class:`~distutils.core.Extension` constructor is -a list of source -files. Since the Distutils currently only support C, C++, and Objective-C -extensions, these are normally C/C++/Objective-C source files. (Be sure to use -appropriate extensions to distinguish C++ source files: :file:`.cc` and -:file:`.cpp` seem to be recognized by both Unix and Windows compilers.) - -However, you can also include SWIG interface (:file:`.i`) files in the list; the -:command:`build_ext` command knows how to deal with SWIG extensions: it will run -SWIG on the interface file and compile the resulting C/C++ file into your -extension. - -.. XXX SWIG support is rough around the edges and largely untested! - -This warning notwithstanding, options to SWIG can be currently passed like -this:: - - setup(..., - ext_modules=[Extension('_foo', ['foo.i'], - swig_opts=['-modern', '-I../include'])], - py_modules=['foo'], - ) - -Or on the commandline like this:: - - > python setup.py build_ext --swig-opts="-modern -I../include" - -On some platforms, you can include non-source files that are processed by the -compiler and included in your extension. Currently, this just means Windows -message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for -Visual C++. These will be compiled to binary resource (:file:`.res`) files and -linked into the executable. - - -Preprocessor options --------------------- - -Three optional arguments to :class:`~distutils.core.Extension` will help if -you need to specify include directories to search or preprocessor macros to -define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. - -For example, if your extension requires header files in the :file:`include` -directory under your distribution root, use the ``include_dirs`` option:: - - Extension('foo', ['foo.c'], include_dirs=['include']) - -You can specify absolute directories there; if you know that your extension will -only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get -away with :: - - Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11']) - -You should avoid this sort of non-portable usage if you plan to distribute your -code: it's probably better to write C code like :: - - #include <X11/Xlib.h> - -If you need to include header files from some other Python extension, you can -take advantage of the fact that header files are installed in a consistent way -by the Distutils :command:`install_headers` command. For example, the Numerical -Python header files are installed (on a standard Unix installation) to -:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ -according to your platform and Python installation.) Since the Python include -directory---\ :file:`/usr/local/include/python1.5` in this case---is always -included in the search path when building Python extensions, the best approach -is to write C code like :: - - #include <Numerical/arrayobject.h> - -If you must put the :file:`Numerical` include directory right into your header -search path, though, you can find that directory using the Distutils -:mod:`distutils.sysconfig` module:: - - from distutils.sysconfig import get_python_inc - incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical') - setup(..., - Extension(..., include_dirs=[incdir]), - ) - -Even though this is quite portable---it will work on any Python installation, -regardless of platform---it's probably easier to just write your C code in the -sensible way. - -You can define and undefine pre-processor macros with the ``define_macros`` and -``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)`` -tuples, where ``name`` is the name of the macro to define (a string) and -``value`` is its value: either a string or ``None``. (Defining a macro ``FOO`` -to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with -most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is -just a list of macros to undefine. - -For example:: - - Extension(..., - define_macros=[('NDEBUG', '1'), - ('HAVE_STRFTIME', None)], - undef_macros=['HAVE_FOO', 'HAVE_BAR']) - -is the equivalent of having this at the top of every C source file:: - - #define NDEBUG 1 - #define HAVE_STRFTIME - #undef HAVE_FOO - #undef HAVE_BAR - - -Library options ---------------- - -You can also specify the libraries to link against when building your extension, -and the directories to search for those libraries. The ``libraries`` option is -a list of libraries to link against, ``library_dirs`` is a list of directories -to search for libraries at link-time, and ``runtime_library_dirs`` is a list of -directories to search for shared (dynamically loaded) libraries at run-time. - -For example, if you need to link against libraries known to be in the standard -library search path on target systems :: - - Extension(..., - libraries=['gdbm', 'readline']) - -If you need to link with libraries in a non-standard location, you'll have to -include the location in ``library_dirs``:: - - Extension(..., - library_dirs=['/usr/X11R6/lib'], - libraries=['X11', 'Xt']) - -(Again, this sort of non-portable construct should be avoided if you intend to -distribute your code.) - -.. XXX Should mention clib libraries here or somewhere else! - - -Other options -------------- - -There are still some other options which can be used to handle special cases. - -The ``optional`` option is a boolean; if it is true, -a build failure in the extension will not abort the build process, but -instead simply not install the failing extension. - -The ``extra_objects`` option is a list of object files to be passed to the -linker. These files must not have extensions, as the default extension for the -compiler is used. - -``extra_compile_args`` and ``extra_link_args`` can be used to -specify additional command line options for the respective compiler and linker -command lines. - -``export_symbols`` is only useful on Windows. It can contain a list of -symbols (functions or variables) to be exported. This option is not needed when -building compiled extensions: Distutils will automatically add ``initmodule`` -to the list of exported symbols. - -The ``depends`` option is a list of files that the extension depends on -(for example header files). The build command will call the compiler on the -sources to rebuild extension if any on this files has been modified since the -previous build. - -Relationships between Distributions and Packages -================================================ - -A distribution may relate to packages in three specific ways: - -#. It can require packages or modules. - -#. It can provide packages or modules. - -#. It can obsolete packages or modules. - -These relationships can be specified using keyword arguments to the -:func:`distutils.core.setup` function. - -Dependencies on other Python modules and packages can be specified by supplying -the *requires* keyword argument to :func:`setup`. The value must be a list of -strings. Each string specifies a package that is required, and optionally what -versions are sufficient. - -To specify that any version of a module or package is required, the string -should consist entirely of the module or package name. Examples include -``'mymodule'`` and ``'xml.parsers.expat'``. - -If specific versions are required, a sequence of qualifiers can be supplied in -parentheses. Each qualifier may consist of a comparison operator and a version -number. The accepted comparison operators are:: - - < > == - <= >= != - -These can be combined by using multiple qualifiers separated by commas (and -optional whitespace). In this case, all of the qualifiers must be matched; a -logical AND is used to combine the evaluations. - -Let's look at a bunch of examples: - -+-------------------------+----------------------------------------------+ -| Requires Expression | Explanation | -+=========================+==============================================+ -| ``==1.0`` | Only version ``1.0`` is compatible | -+-------------------------+----------------------------------------------+ -| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` | -| | is compatible, except ``1.5.1`` | -+-------------------------+----------------------------------------------+ - -Now that we can specify dependencies, we also need to be able to specify what we -provide that other distributions can require. This is done using the *provides* -keyword argument to :func:`setup`. The value for this keyword is a list of -strings, each of which names a Python module or package, and optionally -identifies the version. If the version is not specified, it is assumed to match -that of the distribution. - -Some examples: - -+---------------------+----------------------------------------------+ -| Provides Expression | Explanation | -+=====================+==============================================+ -| ``mypkg`` | Provide ``mypkg``, using the distribution | -| | version | -+---------------------+----------------------------------------------+ -| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of | -| | the distribution version | -+---------------------+----------------------------------------------+ - -A package can declare that it obsoletes other packages using the *obsoletes* -keyword argument. The value for this is similar to that of the *requires* -keyword: a list of strings giving module or package specifiers. Each specifier -consists of a module or package name optionally followed by one or more version -qualifiers. Version qualifiers are given in parentheses after the module or -package name. - -The versions identified by the qualifiers are those that are obsoleted by the -distribution being described. If no qualifiers are given, all versions of the -named module or package are understood to be obsoleted. - -.. _distutils-installing-scripts: - -Installing Scripts -================== - -So far we have been dealing with pure and non-pure Python modules, which are -usually not run by themselves but imported by scripts. - -Scripts are files containing Python source code, intended to be started from the -command line. Scripts don't require Distutils to do anything very complicated. -The only clever feature is that if the first line of the script starts with -``#!`` and contains the word "python", the Distutils will adjust the first line -to refer to the current interpreter location. By default, it is replaced with -the current interpreter location. The :option:`!--executable` (or :option:`!-e`) -option will allow the interpreter path to be explicitly overridden. - -The ``scripts`` option simply is a list of files to be handled in this -way. From the PyXML setup script:: - - setup(..., - scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val'] - ) - -.. versionchanged:: 3.1 - All the scripts will also be added to the ``MANIFEST`` file if no template is - provided. See :ref:`manifest`. - - -.. _distutils-installing-package-data: - -Installing Package Data -======================= - -Often, additional files need to be installed into a package. These files are -often data that's closely related to the package's implementation, or text files -containing documentation that might be of interest to programmers using the -package. These files are called :dfn:`package data`. - -Package data can be added to packages using the ``package_data`` keyword -argument to the :func:`setup` function. The value must be a mapping from -package name to a list of relative path names that should be copied into the -package. The paths are interpreted as relative to the directory containing the -package (information from the ``package_dir`` mapping is used if appropriate); -that is, the files are expected to be part of the package in the source -directories. They may contain glob patterns as well. - -The path names may contain directory portions; any necessary directories will be -created in the installation. - -For example, if a package should contain a subdirectory with several data files, -the files can be arranged like this in the source tree:: - - setup.py - src/ - mypkg/ - __init__.py - module.py - data/ - tables.dat - spoons.dat - forks.dat - -The corresponding call to :func:`setup` might be:: - - setup(..., - packages=['mypkg'], - package_dir={'mypkg': 'src/mypkg'}, - package_data={'mypkg': ['data/*.dat']}, - ) - - -.. versionchanged:: 3.1 - All the files that match ``package_data`` will be added to the ``MANIFEST`` - file if no template is provided. See :ref:`manifest`. - - -.. _distutils-additional-files: - -Installing Additional Files -=========================== - -The ``data_files`` option can be used to specify additional files needed -by the module distribution: configuration files, message catalogs, data files, -anything which doesn't fit in the previous categories. - -``data_files`` specifies a sequence of (*directory*, *files*) pairs in the -following way:: - - setup(..., - data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), - ('config', ['cfg/data.cfg']), - ('/etc/init.d', ['init-script'])] - ) - -Note that you can specify the directory names where the data files will be -installed, but you cannot rename the data files themselves. - -Each (*directory*, *files*) pair in the sequence specifies the installation -directory and the files to install there. If *directory* is a relative path, it -is interpreted relative to the installation prefix (Python's ``sys.prefix`` for -pure-Python packages, ``sys.exec_prefix`` for packages that contain extension -modules). Each file name in *files* is interpreted relative to the -:file:`setup.py` script at the top of the package source distribution. No -directory information from *files* is used to determine the final location of -the installed file; only the name of the file is used. - -You can specify the ``data_files`` options as a simple sequence of files -without specifying a target directory, but this is not recommended, and the -:command:`install` command will print a warning in this case. To install data -files directly in the target directory, an empty string should be given as the -directory. - -.. versionchanged:: 3.1 - All the files that match ``data_files`` will be added to the ``MANIFEST`` - file if no template is provided. See :ref:`manifest`. - - -.. _meta-data: - -Additional meta-data -==================== - -The setup script may include additional meta-data beyond the name and version. -This information includes: - -+----------------------+---------------------------+-----------------+--------+ -| Meta-Data | Description | Value | Notes | -+======================+===========================+=================+========+ -| ``name`` | name of the package | short string | \(1) | -+----------------------+---------------------------+-----------------+--------+ -| ``version`` | version of this release | short string | (1)(2) | -+----------------------+---------------------------+-----------------+--------+ -| ``author`` | package author's name | short string | \(3) | -+----------------------+---------------------------+-----------------+--------+ -| ``author_email`` | email address of the | email address | \(3) | -| | package author | | | -+----------------------+---------------------------+-----------------+--------+ -| ``maintainer`` | package maintainer's name | short string | \(3) | -+----------------------+---------------------------+-----------------+--------+ -| ``maintainer_email`` | email address of the | email address | \(3) | -| | package maintainer | | | -+----------------------+---------------------------+-----------------+--------+ -| ``url`` | home page for the package | URL | \(1) | -+----------------------+---------------------------+-----------------+--------+ -| ``description`` | short, summary | short string | | -| | description of the | | | -| | package | | | -+----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(5) | -| | package | | | -+----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | \(4) | -| | package may be downloaded | | | -+----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | \(4) | -+----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | | -+----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(6) | -+----------------------+---------------------------+-----------------+--------+ - -Notes: - -(1) - These fields are required. - -(2) - It is recommended that versions take the form *major.minor[.patch[.sub]]*. - -(3) - Either the author or the maintainer must be identified. If maintainer is - provided, distutils lists it as the author in :file:`PKG-INFO`. - -(4) - These fields should not be used if your package is to be compatible with Python - versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - <https://pypi.org/>`_. - -(5) - The ``long_description`` field is used by PyPI when you are - :ref:`registering <package-register>` a package, to - :ref:`build its home page <package-display>`. - -(6) - The ``license`` field is a text indicating the license covering the - package where the license is not a selection from the "License" Trove - classifiers. See the ``Classifier`` field. Notice that - there's a ``licence`` distribution option which is deprecated but still - acts as an alias for ``license``. - -'short string' - A single line of text, not more than 200 characters. - -'long string' - Multiple lines of plain text in reStructuredText format (see - http://docutils.sourceforge.net/). - -'list of strings' - See below. - -Encoding the version information is an art in itself. Python packages generally -adhere to the version format *major.minor[.patch][sub]*. The major number is 0 -for initial, experimental releases of software. It is incremented for releases -that represent major milestones in a package. The minor number is incremented -when important new features are added to the package. The patch number -increments when bug-fix releases are made. Additional trailing version -information is sometimes used to indicate sub-releases. These are -"a1,a2,...,aN" (for alpha releases, where functionality and API may change), -"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN" -(for final pre-release release testing). Some examples: - -0.1.0 - the first, experimental release of a package - -1.0.1a2 - the second alpha release of the first patch version of 1.0 - -``classifiers`` are specified in a Python list:: - - setup(..., - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Environment :: Web Environment', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Python Software Foundation License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Programming Language :: Python', - 'Topic :: Communications :: Email', - 'Topic :: Office/Business', - 'Topic :: Software Development :: Bug Tracking', - ], - ) - -.. _debug-setup-script: - -Debugging the setup script -========================== - -Sometimes things go wrong, and the setup script doesn't do what the developer -wants. - -Distutils catches any exceptions when running the setup script, and print a -simple error message before the script is terminated. The motivation for this -behaviour is to not confuse administrators who don't know much about Python and -are trying to install a package. If they get a big long traceback from deep -inside the guts of Distutils, they may think the package or the Python -installation is broken because they don't read all the way down to the bottom -and see that it's a permission problem. - -On the other hand, this doesn't help the developer to find the cause of the -failure. For this purpose, the :envvar:`DISTUTILS_DEBUG` environment variable can be set -to anything except an empty string, and distutils will now print detailed -information about what it is doing, dump the full traceback when an exception -occurs, and print the whole command line when an external program (like a C -compiler) fails. diff --git a/third_party/python/Doc/distutils/sourcedist.rst b/third_party/python/Doc/distutils/sourcedist.rst deleted file mode 100644 index cc289c9b2..000000000 --- a/third_party/python/Doc/distutils/sourcedist.rst +++ /dev/null @@ -1,236 +0,0 @@ -.. _source-dist: - -****************************** -Creating a Source Distribution -****************************** - -As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command -to create a source distribution. In the simplest case, :: - - python setup.py sdist - -(assuming you haven't specified any :command:`sdist` options in the setup script -or config file), :command:`sdist` creates the archive of the default format for -the current platform. The default format is a gzip'ed tar file -(:file:`.tar.gz`) on Unix, and ZIP file on Windows. - -You can specify as many formats as you like using the :option:`!--formats` -option, for example:: - - python setup.py sdist --formats=gztar,zip - -to create a gzipped tarball and a zip file. The available formats are: - -+-----------+-------------------------+---------+ -| Format | Description | Notes | -+===========+=========================+=========+ -| ``zip`` | zip file (:file:`.zip`) | (1),(3) | -+-----------+-------------------------+---------+ -| ``gztar`` | gzip'ed tar file | \(2) | -| | (:file:`.tar.gz`) | | -+-----------+-------------------------+---------+ -| ``bztar`` | bzip2'ed tar file | | -| | (:file:`.tar.bz2`) | | -+-----------+-------------------------+---------+ -| ``xztar`` | xz'ed tar file | | -| | (:file:`.tar.xz`) | | -+-----------+-------------------------+---------+ -| ``ztar`` | compressed tar file | \(4) | -| | (:file:`.tar.Z`) | | -+-----------+-------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | | -+-----------+-------------------------+---------+ - -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - -Notes: - -(1) - default on Windows - -(2) - default on Unix - -(3) - requires either external :program:`zip` utility or :mod:`zipfile` module (part - of the standard Python library since Python 1.6) - -(4) - requires the :program:`compress` program. Notice that this format is now - pending for deprecation and will be removed in the future versions of Python. - -When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or -``tar``), under Unix you can specify the ``owner`` and ``group`` names -that will be set for each member of the archive. - -For example, if you want all files of the archive to be owned by root:: - - python setup.py sdist --owner=root --group=root - - -.. _manifest: - -Specifying the files to distribute -================================== - -If you don't supply an explicit list of files (or instructions on how to -generate one), the :command:`sdist` command puts a minimal default set into the -source distribution: - -* all Python source files implied by the ``py_modules`` and - ``packages`` options - -* all C source files mentioned in the ``ext_modules`` or - ``libraries`` options - - .. XXX getting C library sources currently broken---no - :meth:`get_source_files` method in :file:`build_clib.py`! - -* scripts identified by the ``scripts`` option - See :ref:`distutils-installing-scripts`. - -* anything that looks like a test script: :file:`test/test\*.py` (currently, the - Distutils don't do anything with test scripts except include them in source - distributions, but in the future there will be a standard for testing Python - module distributions) - -* :file:`README.txt` (or :file:`README`), :file:`setup.py` (or whatever you - called your setup script), and :file:`setup.cfg` - -* all files that matches the ``package_data`` metadata. - See :ref:`distutils-installing-package-data`. - -* all files that matches the ``data_files`` metadata. - See :ref:`distutils-additional-files`. - -Sometimes this is enough, but usually you will want to specify additional files -to distribute. The typical way to do this is to write a *manifest template*, -called :file:`MANIFEST.in` by default. The manifest template is just a list of -instructions for how to generate your manifest file, :file:`MANIFEST`, which is -the exact list of files to include in your source distribution. The -:command:`sdist` command processes this template and generates a manifest based -on its instructions and what it finds in the filesystem. - -If you prefer to roll your own manifest file, the format is simple: one filename -per line, regular files (or symlinks to them) only. If you do supply your own -:file:`MANIFEST`, you must specify everything: the default set of files -described above does not apply in this case. - -.. versionchanged:: 3.1 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. - -.. versionchanged:: 3.1.3 - :file:`MANIFEST` files start with a comment indicating they are generated. - Files without this comment are not overwritten or removed. - -.. versionchanged:: 3.2.2 - :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` - exists, like it used to do. - - -The manifest template has one command per line, where each command specifies a -set of files to include or exclude from the source distribution. For an -example, again we turn to the Distutils' own manifest template: - -.. code-block:: none - - include *.txt - recursive-include examples *.txt *.py - prune examples/sample?/build - -The meanings should be fairly clear: include all files in the distribution root -matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory -matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching -:file:`examples/sample?/build`. All of this is done *after* the standard -include set, so you can exclude files from the standard set with explicit -instructions in the manifest template. (Or, you can use the -:option:`!--no-defaults` option to disable the standard set entirely.) There are -several other commands available in the manifest template mini-language; see -section :ref:`sdist-cmd`. - -The order of commands in the manifest template matters: initially, we have the -list of default files as described above, and each command in the template adds -to or removes from that list of files. Once we have fully processed the -manifest template, we remove files that should not be included in the source -distribution: - -* all files in the Distutils "build" tree (default :file:`build/`) - -* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`, - :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs` - -Now we have our complete list of files, which is written to the manifest for -future reference, and then used to build the source distribution archive(s). - -You can disable the default set of included files with the -:option:`!--no-defaults` option, and you can disable the standard exclude set -with :option:`!--no-prune`. - -Following the Distutils' own manifest template, let's trace how the -:command:`sdist` command builds the list of files to include in the Distutils -source distribution: - -#. include all Python source files in the :file:`distutils` and - :file:`distutils/command` subdirectories (because packages corresponding to - those two directories were mentioned in the ``packages`` option in the - setup script---see section :ref:`setup-script`) - -#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard - files) - -#. include :file:`test/test\*.py` (standard files) - -#. include :file:`\*.txt` in the distribution root (this will find - :file:`README.txt` a second time, but such redundancies are weeded out later) - -#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree - under :file:`examples`, - -#. exclude all files in the sub-trees starting at directories matching - :file:`examples/sample?/build`\ ---this may exclude files included by the - previous two steps, so it's important that the ``prune`` command in the manifest - template comes after the ``recursive-include`` command - -#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`, - :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs` - directories - -Just like in the setup script, file and directory names in the manifest template -should always be slash-separated; the Distutils will take care of converting -them to the standard representation on your platform. That way, the manifest -template is portable across operating systems. - - -.. _manifest-options: - -Manifest-related options -======================== - -The normal course of operations for the :command:`sdist` command is as follows: - -* if the manifest file (:file:`MANIFEST` by default) exists and the first line - does not have a comment indicating it is generated from :file:`MANIFEST.in`, - then it is used as is, unaltered - -* if the manifest file doesn't exist or has been previously automatically - generated, read :file:`MANIFEST.in` and create the manifest - -* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest - with just the default file set - -* use the list of files now in :file:`MANIFEST` (either just generated or read - in) to create the source distribution archive(s) - -There are a couple of options that modify this behaviour. First, use the -:option:`!--no-defaults` and :option:`!--no-prune` to disable the standard -"include" and "exclude" sets. - -Second, you might just want to (re)generate the manifest, but not create a source -distribution:: - - python setup.py sdist --manifest-only - -:option:`!-o` is a shortcut for :option:`!--manifest-only`. diff --git a/third_party/python/Doc/distutils/uploading.rst b/third_party/python/Doc/distutils/uploading.rst deleted file mode 100644 index 4bce6997f..000000000 --- a/third_party/python/Doc/distutils/uploading.rst +++ /dev/null @@ -1,7 +0,0 @@ -:orphan: - -*************************************** -Uploading Packages to the Package Index -*************************************** - -The contents of this page have moved to the section :ref:`package-index`. diff --git a/third_party/python/Doc/docutils.conf b/third_party/python/Doc/docutils.conf deleted file mode 100644 index bda4f5dc2..000000000 --- a/third_party/python/Doc/docutils.conf +++ /dev/null @@ -1,2 +0,0 @@ -[restructuredtext parser] -smartquotes-locales: ja: ""'' diff --git a/third_party/python/Doc/extending/building.rst b/third_party/python/Doc/extending/building.rst deleted file mode 100644 index 9fe12c242..000000000 --- a/third_party/python/Doc/extending/building.rst +++ /dev/null @@ -1,167 +0,0 @@ -.. highlightlang:: c - -.. _building: - -***************************** -Building C and C++ Extensions -***************************** - -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. - -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using distutils, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully-initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlightlang:: python - -For modules with ASCII-only names, the function must be named -``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. - - -.. highlightlang:: c - -Building C and C++ Extensions with distutils -============================================ - -.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> - -Extension modules can be built using distutils, which is included in Python. -Since distutils also supports creation of binary packages, users don't -necessarily need a compiler and distutils to install the extension. - -A distutils package contains a driver script, :file:`setup.py`. This is a plain -Python file, which, in the most simple case, could look like this: - -.. code-block:: python3 - - from distutils.core import setup, Extension - - module1 = Extension('demo', - sources = ['demo.c']) - - setup (name = 'PackageName', - version = '1.0', - description = 'This is a demo package', - ext_modules = [module1]) - - -With this :file:`setup.py`, and a file :file:`demo.c`, running :: - - python setup.py build - -will compile :file:`demo.c`, and produce an extension module named ``demo`` in -the :file:`build` directory. Depending on the system, the module file will end -up in a subdirectory :file:`build/lib.system`, and may have a name like -:file:`demo.so` or :file:`demo.pyd`. - -In the :file:`setup.py`, all execution is performed by calling the ``setup`` -function. This takes a variable number of keyword arguments, of which the -example above uses only a subset. Specifically, the example specifies -meta-information to build packages, and it specifies the contents of the -package. Normally, a package will contain additional modules, like Python -source modules, documentation, subpackages, etc. Please refer to the distutils -documentation in :ref:`distutils-index` to learn more about the features of -distutils; this section explains building extension modules only. - -It is common to pre-compute arguments to :func:`setup`, to better structure the -driver script. In the example above, the ``ext_modules`` argument to -:func:`~distutils.core.setup` is a list of extension modules, each of which is -an instance of -the :class:`~distutils.extension.Extension`. In the example, the instance -defines an extension named ``demo`` which is build by compiling a single source -file, :file:`demo.c`. - -In many cases, building an extension is more complex, since additional -preprocessor defines and libraries may be needed. This is demonstrated in the -example below. - -.. code-block:: python3 - - from distutils.core import setup, Extension - - module1 = Extension('demo', - define_macros = [('MAJOR_VERSION', '1'), - ('MINOR_VERSION', '0')], - include_dirs = ['/usr/local/include'], - libraries = ['tcl83'], - library_dirs = ['/usr/local/lib'], - sources = ['demo.c']) - - setup (name = 'PackageName', - version = '1.0', - description = 'This is a demo package', - author = 'Martin v. Loewis', - author_email = 'martin@v.loewis.de', - url = 'https://docs.python.org/extending/building', - long_description = ''' - This is really just a demo package. - ''', - ext_modules = [module1]) - - -In this example, :func:`~distutils.core.setup` is called with additional -meta-information, which -is recommended when distribution packages have to be built. For the extension -itself, it specifies preprocessor defines, include directories, library -directories, and libraries. Depending on the compiler, distutils passes this -information in different ways to the compiler. For example, on Unix, this may -result in the compilation commands :: - - gcc -DNDEBUG -g -O3 -Wall -Wstrict-prototypes -fPIC -DMAJOR_VERSION=1 -DMINOR_VERSION=0 -I/usr/local/include -I/usr/local/include/python2.2 -c demo.c -o build/temp.linux-i686-2.2/demo.o - - gcc -shared build/temp.linux-i686-2.2/demo.o -L/usr/local/lib -ltcl83 -o build/lib.linux-i686-2.2/demo.so - -These lines are for demonstration purposes only; distutils users should trust -that distutils gets the invocations right. - - -.. _distributing: - -Distributing your extension modules -=================================== - -When an extension has been successfully build, there are three ways to use it. - -End-users will typically want to install the module, they do so by running :: - - python setup.py install - -Module maintainers should produce source packages; to do so, they run :: - - python setup.py sdist - -In some cases, additional files need to be included in a source distribution; -this is done through a :file:`MANIFEST.in` file; see :ref:`manifest` for details. - -If the source distribution has been build successfully, maintainers can also -create binary distributions. Depending on the platform, one of the following -commands can be used to do so. :: - - python setup.py bdist_wininst - python setup.py bdist_rpm - python setup.py bdist_dumb diff --git a/third_party/python/Doc/extending/embedding.rst b/third_party/python/Doc/extending/embedding.rst deleted file mode 100644 index 7e4fc19db..000000000 --- a/third_party/python/Doc/extending/embedding.rst +++ /dev/null @@ -1,335 +0,0 @@ -.. highlightlang:: c - - -.. _embedding: - -*************************************** -Embedding Python in Another Application -*************************************** - -The previous chapters discussed how to extend Python, that is, how to extend the -functionality of Python by attaching a library of C functions to it. It is also -possible to do it the other way around: enrich your C/C++ application by -embedding Python in it. Embedding provides your application with the ability to -implement some of the functionality of your application in Python rather than C -or C++. This can be used for many purposes; one example would be to allow users -to tailor the application to their needs by writing some scripts in Python. You -can also use it yourself if some of the functionality can be written in Python -more easily. - -Embedding Python is similar to extending it, but not quite. The difference is -that when you extend Python, the main program of the application is still the -Python interpreter, while if you embed Python, the main program may have nothing -to do with Python --- instead, some parts of the application occasionally call -the Python interpreter to run some Python code. - -So if you are embedding Python, you are providing your own main program. One of -the things this main program has to do is initialize the Python interpreter. At -the very least, you have to call the function :c:func:`Py_Initialize`. There are -optional calls to pass command line arguments to Python. Then later you can -call the interpreter from any part of the application. - -There are several different ways to call the interpreter: you can pass a string -containing Python statements to :c:func:`PyRun_SimpleString`, or you can pass a -stdio file pointer and a file name (for identification in error messages only) -to :c:func:`PyRun_SimpleFile`. You can also call the lower-level operations -described in the previous chapters to construct and use Python objects. - - -.. seealso:: - - :ref:`c-api-index` - The details of Python's C interface are given in this manual. A great deal of - necessary information can be found here. - - -.. _high-level-embedding: - -Very High Level Embedding -========================= - -The simplest form of embedding Python is the use of the very high level -interface. This interface is intended to execute a Python script without needing -to interact with the application directly. This can for example be used to -perform some operation on a file. :: - - #include <Python.h> - - int - main(int argc, char *argv[]) - { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - Py_SetProgramName(program); /* optional but recommended */ - Py_Initialize(); - PyRun_SimpleString("from time import time,ctime\n" - "print('Today is', ctime(time()))\n"); - if (Py_FinalizeEx() < 0) { - exit(120); - } - PyMem_RawFree(program); - return 0; - } - -The :c:func:`Py_SetProgramName` function should be called before -:c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time -libraries. Next, the Python interpreter is initialized with -:c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts -the interpreter down, followed by the end of the program. In a real program, -you may want to get the Python script from another source, perhaps a text-editor -routine, a file, or a database. Getting the Python code from a file can better -be done by using the :c:func:`PyRun_SimpleFile` function, which saves you the -trouble of allocating memory space and loading the file contents. - - -.. _lower-level-embedding: - -Beyond Very High Level Embedding: An overview -============================================= - -The high level interface gives you the ability to execute arbitrary pieces of -Python code from your application, but exchanging data values is quite -cumbersome to say the least. If you want that, you should use lower level calls. -At the cost of having to write more C code, you can achieve almost anything. - -It should be noted that extending Python and embedding Python is quite the same -activity, despite the different intent. Most topics discussed in the previous -chapters are still valid. To show this, consider what the extension code from -Python to C really does: - -#. Convert data values from Python to C, - -#. Perform a function call to a C routine using the converted values, and - -#. Convert the data values from the call from C to Python. - -When embedding Python, the interface code does: - -#. Convert data values from C to Python, - -#. Perform a function call to a Python interface routine using the converted - values, and - -#. Convert the data values from the call from Python to C. - -As you can see, the data conversion steps are simply swapped to accommodate the -different direction of the cross-language transfer. The only difference is the -routine that you call between both data conversions. When extending, you call a -C routine, when embedding, you call a Python routine. - -This chapter will not discuss how to convert data from Python to C and vice -versa. Also, proper use of references and dealing with errors is assumed to be -understood. Since these aspects do not differ from extending the interpreter, -you can refer to earlier chapters for the required information. - - -.. _pure-embedding: - -Pure Embedding -============== - -The first program aims to execute a function in a Python script. Like in the -section about the very high level interface, the Python interpreter does not -directly interact with the application (but that will change in the next -section). - -The code to run a function defined in a Python script is: - -.. literalinclude:: ../includes/run-func.c - - -This code loads a Python script using ``argv[1]``, and calls the function named -in ``argv[2]``. Its integer arguments are the other values of the ``argv`` -array. If you :ref:`compile and link <compiling>` this program (let's call -the finished executable :program:`call`), and use it to execute a Python -script, such as: - -.. code-block:: python - - def multiply(a,b): - print("Will compute", a, "times", b) - c = 0 - for i in range(0, a): - c = c + b - return c - -then the result should be: - -.. code-block:: shell-session - - $ call multiply multiply 3 2 - Will compute 3 times 2 - Result of call: 6 - -Although the program is quite large for its functionality, most of the code is -for data conversion between Python and C, and for error reporting. The -interesting part with respect to embedding Python starts with :: - - Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); - /* Error checking of pName left out */ - pModule = PyImport_Import(pName); - -After initializing the interpreter, the script is loaded using -:c:func:`PyImport_Import`. This routine needs a Python string as its argument, -which is constructed using the :c:func:`PyUnicode_FromString` data conversion -routine. :: - - pFunc = PyObject_GetAttrString(pModule, argv[2]); - /* pFunc is a new reference */ - - if (pFunc && PyCallable_Check(pFunc)) { - ... - } - Py_XDECREF(pFunc); - -Once the script is loaded, the name we're looking for is retrieved using -:c:func:`PyObject_GetAttrString`. If the name exists, and the object returned is -callable, you can safely assume that it is a function. The program then -proceeds by constructing a tuple of arguments as normal. The call to the Python -function is then made with:: - - pValue = PyObject_CallObject(pFunc, pArgs); - -Upon return of the function, ``pValue`` is either *NULL* or it contains a -reference to the return value of the function. Be sure to release the reference -after examining the value. - - -.. _extending-with-embedding: - -Extending Embedded Python -========================= - -Until now, the embedded Python interpreter had no access to functionality from -the application itself. The Python API allows this by extending the embedded -interpreter. That is, the embedded interpreter gets extended with routines -provided by the application. While it sounds complex, it is not so bad. Simply -forget for a while that the application starts the Python interpreter. Instead, -consider the application to be a set of subroutines, and write some glue code -that gives Python access to those routines, just like you would write a normal -Python extension. For example:: - - static int numargs=0; - - /* Return the number of arguments of the application command line */ - static PyObject* - emb_numargs(PyObject *self, PyObject *args) - { - if(!PyArg_ParseTuple(args, ":numargs")) - return NULL; - return PyLong_FromLong(numargs); - } - - static PyMethodDef EmbMethods[] = { - {"numargs", emb_numargs, METH_VARARGS, - "Return the number of arguments received by the process."}, - {NULL, NULL, 0, NULL} - }; - - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL - }; - - static PyObject* - PyInit_emb(void) - { - return PyModule_Create(&EmbModule); - } - -Insert the above code just above the :c:func:`main` function. Also, insert the -following two statements before the call to :c:func:`Py_Initialize`:: - - numargs = argc; - PyImport_AppendInittab("emb", &PyInit_emb); - -These two lines initialize the ``numargs`` variable, and make the -:func:`emb.numargs` function accessible to the embedded Python interpreter. -With these extensions, the Python script can do things like - -.. code-block:: python - - import emb - print("Number of arguments", emb.numargs()) - -In a real application, the methods will expose an API of the application to -Python. - -.. TODO: threads, code examples do not really behave well if errors happen - (what to watch out for) - - -.. _embeddingincplusplus: - -Embedding Python in C++ -======================= - -It is also possible to embed Python in a C++ program; precisely how this is done -will depend on the details of the C++ system used; in general you will need to -write the main program in C++, and use the C++ compiler to compile and link your -program. There is no need to recompile Python itself using C++. - - -.. _compiling: - -Compiling and Linking under Unix-like systems -============================================= - -It is not necessarily trivial to find the right flags to pass to your -compiler (and linker) in order to embed the Python interpreter into your -application, particularly because Python needs to load library modules -implemented as C dynamic extensions (:file:`.so` files) linked against -it. - -To find out the required compiler and linker flags, you can execute the -:file:`python{X.Y}-config` script which is generated as part of the -installation process (a :file:`python3-config` script may also be -available). This script has several options, of which the following will -be directly useful to you: - -* ``pythonX.Y-config --cflags`` will give you the recommended flags when - compiling: - - .. code-block:: shell-session - - $ /opt/bin/python3.4-config --cflags - -I/opt/include/python3.4m -I/opt/include/python3.4m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes - -* ``pythonX.Y-config --ldflags`` will give you the recommended flags when - linking: - - .. code-block:: shell-session - - $ /opt/bin/python3.4-config --ldflags - -L/opt/lib/python3.4/config-3.4m -lpthread -ldl -lutil -lm -lpython3.4m -Xlinker -export-dynamic - -.. note:: - To avoid confusion between several Python installations (and especially - between the system Python and your own compiled Python), it is recommended - that you use the absolute path to :file:`python{X.Y}-config`, as in the above - example. - -If this procedure doesn't work for you (it is not guaranteed to work for -all Unix-like platforms; however, we welcome :ref:`bug reports <reporting-bugs>`) -you will have to read your system's documentation about dynamic linking and/or -examine Python's :file:`Makefile` (use :func:`sysconfig.get_makefile_filename` -to find its location) and compilation -options. In this case, the :mod:`sysconfig` module is a useful tool to -programmatically extract the configuration values that you will want to -combine together. For example: - -.. code-block:: pycon - - >>> import sysconfig - >>> sysconfig.get_config_var('LIBS') - '-lpthread -ldl -lutil' - >>> sysconfig.get_config_var('LINKFORSHARED') - '-Xlinker -export-dynamic' - - -.. XXX similar documentation for Windows missing diff --git a/third_party/python/Doc/extending/extending.rst b/third_party/python/Doc/extending/extending.rst deleted file mode 100644 index 2a288404f..000000000 --- a/third_party/python/Doc/extending/extending.rst +++ /dev/null @@ -1,1359 +0,0 @@ -.. highlightlang:: c - - -.. _extending-intro: - -****************************** -Extending Python with C or C++ -****************************** - -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - <https://cffi.readthedocs.org>`_ library rather than writing custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first line of our file can be:: - - #include <Python.h> - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ``<stdio.h>``, ``<string.h>``, -``<errno.h>``, and ``<stdlib.h>``. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return *NULL* immediately (as we saw in the example). - - -.. _extending-errors: - -Intermezzo: Errors and Exceptions -================================= - -An important convention throughout the Python interpreter is the following: when -a function fails, it should set an exception condition and return an error value -(usually a *NULL* pointer). Exceptions are stored in a static global variable -inside the interpreter; if this variable is *NULL* no exception has occurred. A -second global variable stores the "associated value" of the exception (the -second argument to :keyword:`raise`). A third variable contains the stack -traceback in case the error originated in Python code. These three variables -are the C equivalents of the result in Python of :meth:`sys.exc_info` (see the -section on module :mod:`sys` in the Python Library Reference). It is important -to know about them to understand how errors are passed around. - -The Python API defines a number of functions to set various types of exceptions. - -The most common one is :c:func:`PyErr_SetString`. Its arguments are an exception -object and a C string. The exception object is usually a predefined object like -:c:data:`PyExc_ZeroDivisionError`. The C string indicates the cause of the error -and is converted to a Python string object and stored as the "associated value" -of the exception. - -Another useful function is :c:func:`PyErr_SetFromErrno`, which only takes an -exception argument and constructs the associated value by inspection of the -global variable :c:data:`errno`. The most general function is -:c:func:`PyErr_SetObject`, which takes two object arguments, the exception and -its associated value. You don't need to :c:func:`Py_INCREF` the objects passed -to any of these functions. - -You can test non-destructively whether an exception has been set with -:c:func:`PyErr_Occurred`. This returns the current exception object, or *NULL* -if no exception has occurred. You normally don't need to call -:c:func:`PyErr_Occurred` to see whether an error occurred in a function call, -since you should be able to tell from the return value. - -When a function *f* that calls another function *g* detects that the latter -fails, *f* should itself return an error value (usually *NULL* or ``-1``). It -should *not* call one of the :c:func:`PyErr_\*` functions --- one has already -been called by *g*. *f*'s caller is then supposed to also return an error -indication to *its* caller, again *without* calling :c:func:`PyErr_\*`, and so on ---- the most detailed cause of the error was already reported by the function -that first detected it. Once the error reaches the Python interpreter's main -loop, this aborts the currently executing Python code and tries to find an -exception handler specified by the Python programmer. - -(There are situations where a module can actually give a more detailed error -message by calling another :c:func:`PyErr_\*` function, and in such cases it is -fine to do so. As a general rule, however, this is not necessary, and can cause -information about the cause of the error to be lost: most operations can fail -for a variety of reasons.) - -To ignore an exception set by a function call that failed, the exception -condition must be cleared explicitly by calling :c:func:`PyErr_Clear`. The only -time C code should call :c:func:`PyErr_Clear` is if it doesn't want to pass the -error on to the interpreter but wants to handle it completely by itself -(possibly by trying something else, or pretending nothing went wrong). - -Every failing :c:func:`malloc` call must be turned into an exception --- the -direct caller of :c:func:`malloc` (or :c:func:`realloc`) must call -:c:func:`PyErr_NoMemory` and return a failure indicator itself. All the -object-creating functions (for example, :c:func:`PyLong_FromLong`) already do -this, so this note is only relevant to those who call :c:func:`malloc` directly. - -Also note that, with the important exception of :c:func:`PyArg_ParseTuple` and -friends, functions that return an integer status usually return a positive value -or zero for success and ``-1`` for failure, like Unix system calls. - -Finally, be careful to clean up garbage (by making :c:func:`Py_XDECREF` or -:c:func:`Py_DECREF` calls for objects you have already created) when you return -an error indicator! - -The choice of which exception to raise is entirely yours. There are predeclared -C objects corresponding to all built-in Python exceptions, such as -:c:data:`PyExc_ZeroDivisionError`, which you can use directly. Of course, you -should choose exceptions wisely --- don't use :c:data:`PyExc_TypeError` to mean -that a file couldn't be opened (that should probably be :c:data:`PyExc_IOError`). -If something's wrong with the argument list, the :c:func:`PyArg_ParseTuple` -function usually raises :c:data:`PyExc_TypeError`. If you have an argument whose -value must be in a particular range or must satisfy other conditions, -:c:data:`PyExc_ValueError` is appropriate. - -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: - - static PyObject *SpamError; - -and initialize it in your module's initialization function (:c:func:`PyInit_spam`) -with an exception object (leaving out the error checking for now):: - - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; - - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - - SpamError = PyErr_NewException("spam.error", NULL, NULL); - Py_INCREF(SpamError); - PyModule_AddObject(m, "error", SpamError); - return m; - } - -Note that the Python name for the exception object is :exc:`spam.error`. The -:c:func:`PyErr_NewException` function may create a class with the base class -being :exc:`Exception` (unless another class is passed in instead of *NULL*), -described in :ref:`bltin-exceptions`. - -Note also that the :c:data:`SpamError` variable retains a reference to the newly -created exception class; this is intentional! Since the exception could be -removed from the module by external code, an owned reference to the class is -needed to ensure that it will not be discarded, causing :c:data:`SpamError` to -become a dangling pointer. Should it become a dangling pointer, C code which -raises the exception could cause a core dump or other unintended side effects. - -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this -sample. - -The :exc:`spam.error` exception can be raised in your extension module using a -call to :c:func:`PyErr_SetString` as shown below:: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - if (sts < 0) { - PyErr_SetString(SpamError, "System command failed"); - return NULL; - } - return PyLong_FromLong(sts); - } - - -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns *NULL* (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`spam.system` function must return the value of :c:data:`sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:type:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a *NULL* pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef SpamMethods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :const:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spammodule); - } - -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -When the Python program imports module :mod:`spam` for the first time, -:c:func:`PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return *NULL* if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. - -When embedding Python, the :c:func:`PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - int - main(int argc, char *argv[]) - { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - - /* Add a built-in module, before Py_Initialize */ - PyImport_AppendInittab("spam", PyInit_spam); - - /* Pass argv[0] to the Python interpreter */ - Py_SetProgramName(program); - - /* Initialize the Python interpreter. Required. */ - Py_Initialize(); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyImport_ImportModule("spam"); - - ... - - PyMem_RawFree(program); - return 0; - } - -.. note:: - - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply -read as an example. - -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - - -.. _compilation: - -Compilation and Linkage -======================= - -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. - -If you can't use dynamic loading, or if you want to make your module a permanent -part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place -your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory -of an unpacked source distribution, add a line to the file -:file:`Modules/Setup.local` describing your file: - -.. code-block:: sh - - spam spammodule.o - -and rebuild the interpreter by running :program:`make` in the toplevel -directory. You can also run :program:`make` in the :file:`Modules/` -subdirectory, but then you must first rebuild :file:`Makefile` there by running -':program:`make` Makefile'. (This is necessary each time you change the -:file:`Setup` file.) - -If your module requires additional libraries to link with, these can be listed -on the line in the configuration file as well, for instance: - -.. code-block:: sh - - spam spammodule.o -lX11 - - -.. _callingpython: - -Calling Python Functions from C -=============================== - -So far we have concentrated on making C functions callable from Python. The -reverse is also useful: calling Python functions from C. This is especially the -case for libraries that support so-called "callback" functions. If a C -interface makes use of callbacks, the equivalent Python often needs to provide a -callback mechanism to the Python programmer; the implementation will require -calling the Python callback functions from a C callback. Other uses are also -imaginable. - -Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) - -Calling a Python function is easy. First, the Python program must somehow pass -you the Python function object. You should provide a function (or some other -interface) to do this. When this function is called, save a pointer to the -Python function object (be careful to :c:func:`Py_INCREF` it!) in a global -variable --- or wherever you see fit. For example, the following function might -be part of a module definition:: - - static PyObject *my_callback = NULL; - - static PyObject * - my_set_callback(PyObject *dummy, PyObject *args) - { - PyObject *result = NULL; - PyObject *temp; - - if (PyArg_ParseTuple(args, "O:set_callback", &temp)) { - if (!PyCallable_Check(temp)) { - PyErr_SetString(PyExc_TypeError, "parameter must be callable"); - return NULL; - } - Py_XINCREF(temp); /* Add a reference to new callback */ - Py_XDECREF(my_callback); /* Dispose of previous callback */ - my_callback = temp; /* Remember new callback */ - /* Boilerplate to return "None" */ - Py_INCREF(Py_None); - result = Py_None; - } - return result; - } - -This function must be registered with the interpreter using the -:const:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The -:c:func:`PyArg_ParseTuple` function and its arguments are documented in section -:ref:`parsetuple`. - -The macros :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` increment/decrement the -reference count of an object and are safe in the presence of *NULL* pointers -(but note that *temp* will not be *NULL* in this context). More info on them -in section :ref:`refcounts`. - -.. index:: single: PyObject_CallObject() - -Later, when it is time to call the function, you call the C function -:c:func:`PyObject_CallObject`. This function has two arguments, both pointers to -arbitrary Python objects: the Python function, and the argument list. The -argument list must always be a tuple object, whose length is the number of -arguments. To call the Python function with no arguments, pass in NULL, or -an empty tuple; to call it with one argument, pass a singleton tuple. -:c:func:`Py_BuildValue` returns a tuple when its format string consists of zero -or more format codes between parentheses. For example:: - - int arg; - PyObject *arglist; - PyObject *result; - ... - arg = 123; - ... - /* Time to call the callback */ - arglist = Py_BuildValue("(i)", arg); - result = PyObject_CallObject(my_callback, arglist); - Py_DECREF(arglist); - -:c:func:`PyObject_CallObject` returns a Python object pointer: this is the return -value of the Python function. :c:func:`PyObject_CallObject` is -"reference-count-neutral" with respect to its arguments. In the example a new -tuple was created to serve as the argument list, which is -:c:func:`Py_DECREF`\ -ed immediately after the :c:func:`PyObject_CallObject` -call. - -The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand -new object, or it is an existing object whose reference count has been -incremented. So, unless you want to save it in a global variable, you should -somehow :c:func:`Py_DECREF` the result, even (especially!) if you are not -interested in its value. - -Before you do this, however, it is important to check that the return value -isn't *NULL*. If it is, the Python function terminated by raising an exception. -If the C code that called :c:func:`PyObject_CallObject` is called from Python, it -should now return an error indication to its Python caller, so the interpreter -can print a stack trace, or the calling Python code can handle the exception. -If this is not possible or desirable, the exception should be cleared by calling -:c:func:`PyErr_Clear`. For example:: - - if (result == NULL) - return NULL; /* Pass error back */ - ...use result... - Py_DECREF(result); - -Depending on the desired interface to the Python callback function, you may also -have to provide an argument list to :c:func:`PyObject_CallObject`. In some cases -the argument list is also provided by the Python program, through the same -interface that specified the callback function. It can then be saved and used -in the same manner as the function object. In other cases, you may have to -construct a new tuple to pass as the argument list. The simplest way to do this -is to call :c:func:`Py_BuildValue`. For example, if you want to pass an integral -event code, you might use the following code:: - - PyObject *arglist; - ... - arglist = Py_BuildValue("(l)", eventcode); - result = PyObject_CallObject(my_callback, arglist); - Py_DECREF(arglist); - if (result == NULL) - return NULL; /* Pass error back */ - /* Here maybe use the result */ - Py_DECREF(result); - -Note the placement of ``Py_DECREF(arglist)`` immediately after the call, before -the error check! Also note that strictly speaking this code is not complete: -:c:func:`Py_BuildValue` may run out of memory, and this should be checked. - -You may also call a function with keyword arguments by using -:c:func:`PyObject_Call`, which supports arguments and keyword arguments. As in -the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. :: - - PyObject *dict; - ... - dict = Py_BuildValue("{s:i}", "name", val); - result = PyObject_Call(my_callback, NULL, dict); - Py_DECREF(dict); - if (result == NULL) - return NULL; /* Pass error back */ - /* Here maybe use the result */ - Py_DECREF(result); - - -.. _parsetuple: - -Extracting Parameters in Extension Functions -============================================ - -.. index:: single: PyArg_ParseTuple() - -The :c:func:`PyArg_ParseTuple` function is declared as follows:: - - int PyArg_ParseTuple(PyObject *arg, const char *format, ...); - -The *arg* argument must be a tuple object containing an argument list passed -from Python to a C function. The *format* argument must be a format string, -whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference -Manual. The remaining arguments must be addresses of variables whose type is -determined by the format string. - -Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have -the required types, it cannot check the validity of the addresses of C variables -passed to the call: if you make mistakes there, your code will probably crash or -at least overwrite random bits in memory. So be careful! - -Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! - -Some example calls:: - - #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ - #include <Python.h> - -:: - - int ok; - int i, j; - long k, l; - const char *s; - Py_ssize_t size; - - ok = PyArg_ParseTuple(args, ""); /* No arguments */ - /* Python call: f() */ - -:: - - ok = PyArg_ParseTuple(args, "s", &s); /* A string */ - /* Possible Python call: f('whoops!') */ - -:: - - ok = PyArg_ParseTuple(args, "lls", &k, &l, &s); /* Two longs and a string */ - /* Possible Python call: f(1, 2, 'three') */ - -:: - - ok = PyArg_ParseTuple(args, "(ii)s#", &i, &j, &s, &size); - /* A pair of ints and a string, whose size is also returned */ - /* Possible Python call: f((1, 2), 'three') */ - -:: - - { - const char *file; - const char *mode = "r"; - int bufsize = 0; - ok = PyArg_ParseTuple(args, "s|si", &file, &mode, &bufsize); - /* A string, and optionally another string and an integer */ - /* Possible Python calls: - f('spam') - f('spam', 'w') - f('spam', 'wb', 100000) */ - } - -:: - - { - int left, top, right, bottom, h, v; - ok = PyArg_ParseTuple(args, "((ii)(ii))(ii)", - &left, &top, &right, &bottom, &h, &v); - /* A rectangle and a point */ - /* Possible Python call: - f(((0, 0), (400, 300)), (10, 10)) */ - } - -:: - - { - Py_complex c; - ok = PyArg_ParseTuple(args, "D:myfunction", &c); - /* a complex, also providing a function name for errors */ - /* Possible Python call: myfunction(1+2j) */ - } - - -.. _parsetupleandkeywords: - -Keyword Parameters for Extension Functions -========================================== - -.. index:: single: PyArg_ParseTupleAndKeywords() - -The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: - - int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); - -The *arg* and *format* parameters are identical to those of the -:c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of -keywords received as the third parameter from the Python runtime. The *kwlist* -parameter is a *NULL*-terminated list of strings which identify the parameters; -the names are matched with the type information from *format* from left to -right. On success, :c:func:`PyArg_ParseTupleAndKeywords` returns true, otherwise -it returns false and raises an appropriate exception. - -.. note:: - - Nested tuples cannot be parsed when using keyword arguments! Keyword parameters - passed in which are not present in the *kwlist* will cause :exc:`TypeError` to - be raised. - -.. index:: single: Philbrick, Geoff - -Here is an example module which uses keywords, based on an example by Geoff -Philbrick (philbrick@hks.com):: - - #include "Python.h" - - static PyObject * - keywdarg_parrot(PyObject *self, PyObject *args, PyObject *keywds) - { - int voltage; - char *state = "a stiff"; - char *action = "voom"; - char *type = "Norwegian Blue"; - - static char *kwlist[] = {"voltage", "state", "action", "type", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, keywds, "i|sss", kwlist, - &voltage, &state, &action, &type)) - return NULL; - - printf("-- This parrot wouldn't %s if you put %i Volts through it.\n", - action, voltage); - printf("-- Lovely plumage, the %s -- It's %s!\n", type, state); - - Py_RETURN_NONE; - } - - static PyMethodDef keywdarg_methods[] = { - /* The cast of the function is necessary since PyCFunction values - * only take two PyObject* parameters, and keywdarg_parrot() takes - * three. - */ - {"parrot", (PyCFunction)keywdarg_parrot, METH_VARARGS | METH_KEYWORDS, - "Print a lovely skit to standard output."}, - {NULL, NULL, 0, NULL} /* sentinel */ - }; - - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModule_Create(&keywdargmodule); - } - - -.. _buildvalue: - -Building Arbitrary Values -========================= - -This function is the counterpart to :c:func:`PyArg_ParseTuple`. It is declared -as follows:: - - PyObject *Py_BuildValue(const char *format, ...); - -It recognizes a set of format units similar to the ones recognized by -:c:func:`PyArg_ParseTuple`, but the arguments (which are input to the function, -not output) must not be pointers, just values. It returns a new Python object, -suitable for returning from a C function called from Python. - -One difference with :c:func:`PyArg_ParseTuple`: while the latter requires its -first argument to be a tuple (since Python argument lists are always represented -as tuples internally), :c:func:`Py_BuildValue` does not always build a tuple. It -builds a tuple only if its format string contains two or more format units. If -the format string is empty, it returns ``None``; if it contains exactly one -format unit, it returns whatever object is described by that format unit. To -force it to return a tuple of size 0 or one, parenthesize the format string. - -Examples (to the left the call, to the right the resulting Python value): - -.. code-block:: none - - Py_BuildValue("") None - Py_BuildValue("i", 123) 123 - Py_BuildValue("iii", 123, 456, 789) (123, 456, 789) - Py_BuildValue("s", "hello") 'hello' - Py_BuildValue("y", "hello") b'hello' - Py_BuildValue("ss", "hello", "world") ('hello', 'world') - Py_BuildValue("s#", "hello", 4) 'hell' - Py_BuildValue("y#", "hello", 4) b'hell' - Py_BuildValue("()") () - Py_BuildValue("(i)", 123) (123,) - Py_BuildValue("(ii)", 123, 456) (123, 456) - Py_BuildValue("(i,i)", 123, 456) (123, 456) - Py_BuildValue("[i,i]", 123, 456) [123, 456] - Py_BuildValue("{s:i,s:i}", - "abc", 123, "def", 456) {'abc': 123, 'def': 456} - Py_BuildValue("((ii)(ii)) (ii)", - 1, 2, 3, 4, 5, 6) (((1, 2), (3, 4)), (5, 6)) - - -.. _refcounts: - -Reference Counts -================ - -In languages like C or C++, the programmer is responsible for dynamic allocation -and deallocation of memory on the heap. In C, this is done using the functions -:c:func:`malloc` and :c:func:`free`. In C++, the operators ``new`` and -``delete`` are used with essentially the same meaning and we'll restrict -the following discussion to the C case. - -Every block of memory allocated with :c:func:`malloc` should eventually be -returned to the pool of available memory by exactly one call to :c:func:`free`. -It is important to call :c:func:`free` at the right time. If a block's address -is forgotten but :c:func:`free` is not called for it, the memory it occupies -cannot be reused until the program terminates. This is called a :dfn:`memory -leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block -through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. -It has the same bad consequences as referencing uninitialized data --- core -dumps, wrong results, mysterious crashes. - -Common causes of memory leaks are unusual paths through the code. For instance, -a function may allocate a block of memory, do some calculation, and then free -the block again. Now a change in the requirements for the function may add a -test to the calculation that detects an error condition and can return -prematurely from the function. It's easy to forget to free the allocated memory -block when taking this premature exit, especially when it is added later to the -code. Such leaks, once introduced, often go undetected for a long time: the -error exit is taken only in a small fraction of all calls, and most modern -machines have plenty of virtual memory, so the leak only becomes apparent in a -long-running process that uses the leaking function frequently. Therefore, it's -important to prevent leaks from happening by having a coding convention or -strategy that minimizes this kind of errors. - -Since Python makes heavy use of :c:func:`malloc` and :c:func:`free`, it needs a -strategy to avoid memory leaks as well as the use of freed memory. The chosen -method is called :dfn:`reference counting`. The principle is simple: every -object contains a counter, which is incremented when a reference to the object -is stored somewhere, and which is decremented when a reference to it is deleted. -When the counter reaches zero, the last reference to the object has been deleted -and the object is freed. - -An alternative strategy is called :dfn:`automatic garbage collection`. -(Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big -advantage of automatic garbage collection is that the user doesn't need to call -:c:func:`free` explicitly. (Another claimed advantage is an improvement in speed -or memory usage --- this is no hard fact however.) The disadvantage is that for -C, there is no truly portable automatic garbage collector, while reference -counting can be implemented portably (as long as the functions :c:func:`malloc` -and :c:func:`free` are available --- which the C Standard guarantees). Maybe some -day a sufficiently portable automatic garbage collector will be available for C. -Until then, we'll have to live with reference counts. - -While Python uses the traditional reference counting implementation, it also -offers a cycle detector that works to detect reference cycles. This allows -applications to not worry about creating direct or indirect circular references; -these are the weakness of garbage collection implemented using only reference -counting. Reference cycles consist of objects which contain (possibly indirect) -references to themselves, so that each object in the cycle has a reference count -which is non-zero. Typical reference counting implementations are not able to -reclaim the memory belonging to any objects in a reference cycle, or referenced -from the objects in the cycle, even though there are no further references to -the cycle itself. - -The cycle detector is able to detect garbage cycles and can reclaim them. -The :mod:`gc` module exposes a way to run the detector (the -:func:`~gc.collect` function), as well as configuration -interfaces and the ability to disable the detector at runtime. The cycle -detector is considered an optional component; though it is included by default, -it can be disabled at build time using the :option:`!--without-cycle-gc` option -to the :program:`configure` script on Unix platforms (including Mac OS X). If -the cycle detector is disabled in this way, the :mod:`gc` module will not be -available. - - -.. _refcountsinpython: - -Reference Counting in Python ----------------------------- - -There are two macros, ``Py_INCREF(x)`` and ``Py_DECREF(x)``, which handle the -incrementing and decrementing of the reference count. :c:func:`Py_DECREF` also -frees the object when the count reaches zero. For flexibility, it doesn't call -:c:func:`free` directly --- rather, it makes a call through a function pointer in -the object's :dfn:`type object`. For this purpose (and others), every object -also contains a pointer to its type object. - -The big question now remains: when to use ``Py_INCREF(x)`` and ``Py_DECREF(x)``? -Let's first introduce some terms. Nobody "owns" an object; however, you can -:dfn:`own a reference` to an object. An object's reference count is now defined -as the number of owned references to it. The owner of a reference is -responsible for calling :c:func:`Py_DECREF` when the reference is no longer -needed. Ownership of a reference can be transferred. There are three ways to -dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. -Forgetting to dispose of an owned reference creates a memory leak. - -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The -borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must -not hold on to the object longer than the owner from which it was borrowed. -Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. - -The advantage of borrowing over owning a reference is that you don't need to -take care of disposing of the reference on all possible paths through the code ---- in other words, with a borrowed reference you don't run the risk of leaking -when a premature exit is taken. The disadvantage of borrowing over owning is -that there are some subtle situations where in seemingly correct code a borrowed -reference can be used after the owner from which it was borrowed has in fact -disposed of it. - -A borrowed reference can be changed into an owned reference by calling -:c:func:`Py_INCREF`. This does not affect the status of the owner from which the -reference was borrowed --- it creates a new owned reference, and gives full -owner responsibilities (the new owner must dispose of the reference properly, as -well as the previous owner). - - -.. _ownershiprules: - -Ownership Rules ---------------- - -Whenever an object reference is passed into or out of a function, it is part of -the function's interface specification whether ownership is transferred with the -reference or not. - -Most functions that return a reference to an object pass on ownership with the -reference. In particular, all functions whose function it is to create a new -object, such as :c:func:`PyLong_FromLong` and :c:func:`Py_BuildValue`, pass -ownership to the receiver. Even if the object is not actually new, you still -receive ownership of a new reference to that object. For instance, -:c:func:`PyLong_FromLong` maintains a cache of popular values and can return a -reference to a cached item. - -Many functions that extract objects from other objects also transfer ownership -with the reference, for instance :c:func:`PyObject_GetAttrString`. The picture -is less clear, here, however, since a few common routines are exceptions: -:c:func:`PyTuple_GetItem`, :c:func:`PyList_GetItem`, :c:func:`PyDict_GetItem`, and -:c:func:`PyDict_GetItemString` all return references that you borrow from the -tuple, list or dictionary. - -The function :c:func:`PyImport_AddModule` also returns a borrowed reference, even -though it may actually create the object it returns: this is possible because an -owned reference to the object is stored in ``sys.modules``. - -When you pass an object reference into another function, in general, the -function borrows the reference from you --- if it needs to store it, it will use -:c:func:`Py_INCREF` to become an independent owner. There are exactly two -important exceptions to this rule: :c:func:`PyTuple_SetItem` and -:c:func:`PyList_SetItem`. These functions take over ownership of the item passed -to them --- even if they fail! (Note that :c:func:`PyDict_SetItem` and friends -don't take over ownership --- they are "normal.") - -When a C function is called from Python, it borrows references to its arguments -from the caller. The caller owns a reference to the object, so the borrowed -reference's lifetime is guaranteed until the function returns. Only when such a -borrowed reference must be stored or passed on, it must be turned into an owned -reference by calling :c:func:`Py_INCREF`. - -The object reference returned from a C function that is called from Python must -be an owned reference --- ownership is transferred from the function to its -caller. - - -.. _thinice: - -Thin Ice --------- - -There are a few situations where seemingly harmless use of a borrowed reference -can lead to problems. These all have to do with implicit invocations of the -interpreter, which can cause the owner of a reference to dispose of it. - -The first and most important case to know about is using :c:func:`Py_DECREF` on -an unrelated object while borrowing a reference to a list item. For instance:: - - void - bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - - PyList_SetItem(list, 1, PyLong_FromLong(0L)); - PyObject_Print(item, stdout, 0); /* BUG! */ - } - -This function first borrows a reference to ``list[0]``, then replaces -``list[1]`` with the value ``0``, and finally prints the borrowed reference. -Looks harmless, right? But it's not! - -Let's follow the control flow into :c:func:`PyList_SetItem`. The list owns -references to all its items, so when item 1 is replaced, it has to dispose of -the original item 1. Now let's suppose the original item 1 was an instance of a -user-defined class, and let's further suppose that the class defined a -:meth:`__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`__del__` method. - -Since it is written in Python, the :meth:`__del__` method can execute arbitrary -Python code. Could it perhaps do something to invalidate the reference to -``item`` in :c:func:`bug`? You bet! Assuming that the list passed into -:c:func:`bug` is accessible to the :meth:`__del__` method, it could execute a -statement to the effect of ``del list[0]``, and assuming this was the last -reference to that object, it would free the memory associated with it, thereby -invalidating ``item``. - -The solution, once you know the source of the problem, is easy: temporarily -increment the reference count. The correct version of the function reads:: - - void - no_bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - - Py_INCREF(item); - PyList_SetItem(list, 1, PyLong_FromLong(0L)); - PyObject_Print(item, stdout, 0); - Py_DECREF(item); - } - -This is a true story. An older version of Python contained variants of this bug -and someone spent a considerable amount of time in a C debugger to figure out -why his :meth:`__del__` methods would fail... - -The second case of problems with a borrowed reference is a variant involving -threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro -:c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using -:c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to -let other threads use the processor while waiting for the I/O to complete. -Obviously, the following function has the same problem as the previous one:: - - void - bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - Py_BEGIN_ALLOW_THREADS - ...some blocking I/O call... - Py_END_ALLOW_THREADS - PyObject_Print(item, stdout, 0); /* BUG! */ - } - - -.. _nullpointers: - -NULL Pointers -------------- - -In general, functions that take object references as arguments do not expect you -to pass them *NULL* pointers, and will dump core (or cause later core dumps) if -you do so. Functions that return object references generally return *NULL* only -to indicate that an exception occurred. The reason for not testing for *NULL* -arguments is that functions often pass the objects they receive on to other -function --- if each function were to test for *NULL*, there would be a lot of -redundant tests and the code would run more slowly. - -It is better to test for *NULL* only at the "source:" when a pointer that may be -*NULL* is received, for example, from :c:func:`malloc` or from a function that -may raise an exception. - -The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for *NULL* -pointers --- however, their variants :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` -do. - -The macros for checking for a particular object type (``Pytype_Check()``) don't -check for *NULL* pointers --- again, there is much code that calls several of -these in a row to test an object against various different expected types, and -this would generate redundant tests. There are no variants with *NULL* -checking. - -The C function calling mechanism guarantees that the argument list passed to C -functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple [#]_. - -It is a severe error to ever let a *NULL* pointer "escape" to the Python user. - -.. Frank Stajano: - A pedagogically buggy example, along the lines of the previous listing, would - be helpful here -- showing in more concrete terms what sort of actions could - cause the problem. I can't very well imagine it from the description. - - -.. _cplusplus: - -Writing Extensions in C++ -========================= - -It is possible to write extension modules in C++. Some restrictions apply. If -the main program (the Python interpreter) is compiled and linked by the C -compiler, global or static objects with constructors cannot be used. This is -not a problem if the main program is linked by the C++ compiler. Functions that -will be called by the Python interpreter (in particular, module initialization -functions) have to be declared using ``extern "C"``. It is unnecessary to -enclose the Python header files in ``extern "C" {...}`` --- they use this form -already if the symbol ``__cplusplus`` is defined (all recent C++ compilers -define this symbol). - - -.. _using-capsules: - -Providing a C API for an Extension Module -========================================= - -.. sectionauthor:: Konrad Hinsen <hinsen@cnrs-orleans.fr> - - -Many extension modules just provide new functions and types to be used from -Python, but sometimes the code in an extension module can be useful for other -extension modules. For example, an extension module could implement a type -"collection" which works like lists without order. Just like the standard Python -list type has a C API which permits extension modules to create and manipulate -lists, this new collection type should have a set of C functions for direct -manipulation from other extension modules. - -At first sight this seems easy: just write the functions (without declaring them -``static``, of course), provide an appropriate header file, and document -the C API. And in fact this would work if all extension modules were always -linked statically with the Python interpreter. When modules are used as shared -libraries, however, the symbols defined in one module may not be visible to -another module. The details of visibility depend on the operating system; some -systems use one global namespace for the Python interpreter and all extension -modules (Windows, for example), whereas others require an explicit list of -imported symbols at module link time (AIX is one example), or offer a choice of -different strategies (most Unices). And even if symbols are globally visible, -the module whose functions one wishes to call might not have been loaded yet! - -Portability therefore requires not to make any assumptions about symbol -visibility. This means that all symbols in extension modules should be declared -``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from -other extension modules must be exported in a different way. - -Python provides a special mechanism to pass C-level information (pointers) from -one extension module to another one: Capsules. A Capsule is a Python data type -which stores a pointer (:c:type:`void \*`). Capsules can only be created and -accessed via their C API, but they can be passed around like any other Python -object. In particular, they can be assigned to a name in an extension module's -namespace. Other extension modules can then import this module, retrieve the -value of this name, and then retrieve the pointer from the Capsule. - -There are many ways in which Capsules can be used to export the C API of an -extension module. Each function could get its own Capsule, or all C API pointers -could be stored in an array whose address is published in a Capsule. And the -various tasks of storing and retrieving the pointers can be distributed in -different ways between the module providing the code and the client modules. - -Whichever method you choose, it's important to name your Capsules properly. -The function :c:func:`PyCapsule_New` takes a name parameter -(:c:type:`const char \*`); you're permitted to pass in a *NULL* name, but -we strongly encourage you to specify a name. Properly named Capsules provide -a degree of runtime type-safety; there is no feasible way to tell one unnamed -Capsule from another. - -In particular, Capsules used to expose C APIs should be given a name following -this convention:: - - modulename.attributename - -The convenience function :c:func:`PyCapsule_Import` makes it easy to -load a C API provided via a Capsule, but only if the Capsule's name -matches this convention. This behavior gives C API users a high degree -of certainty that the Capsule they load contains the correct C API. - -The following example demonstrates an approach that puts most of the burden on -the writer of the exporting module, which is appropriate for commonly used -library modules. It stores all C API pointers (just one in the example!) in an -array of :c:type:`void` pointers which becomes the value of a Capsule. The header -file corresponding to the module provides a macro that takes care of importing -the module and retrieving its C API pointers; client modules only have to call -this macro before accessing the C API. - -The exporting module is a modification of the :mod:`spam` module from section -:ref:`extending-simpleexample`. The function :func:`spam.system` does not call -the C library function :c:func:`system` directly, but a function -:c:func:`PySpam_System`, which would of course do something more complicated in -reality (such as adding "spam" to every command). This function -:c:func:`PySpam_System` is also exported to other extension modules. - -The function :c:func:`PySpam_System` is a plain C function, declared -``static`` like everything else:: - - static int - PySpam_System(const char *command) - { - return system(command); - } - -The function :c:func:`spam_system` is modified in a trivial way:: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = PySpam_System(command); - return PyLong_FromLong(sts); - } - -In the beginning of the module, right after the line :: - - #include "Python.h" - -two more lines must be added:: - - #define SPAM_MODULE - #include "spammodule.h" - -The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; - static void *PySpam_API[PySpam_API_pointers]; - PyObject *c_api_object; - - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - - /* Initialize the C API pointer array */ - PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; - - /* Create a Capsule containing the API pointer array's address */ - c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); - - if (c_api_object != NULL) - PyModule_AddObject(m, "_C_API", c_api_object); - return m; - } - -Note that ``PySpam_API`` is declared ``static``; otherwise the pointer -array would disappear when :func:`PyInit_spam` terminates! - -The bulk of the work is in the header file :file:`spammodule.h`, which looks -like this:: - - #ifndef Py_SPAMMODULE_H - #define Py_SPAMMODULE_H - #ifdef __cplusplus - extern "C" { - #endif - - /* Header file for spammodule */ - - /* C API functions */ - #define PySpam_System_NUM 0 - #define PySpam_System_RETURN int - #define PySpam_System_PROTO (const char *command) - - /* Total number of C API pointers */ - #define PySpam_API_pointers 1 - - - #ifdef SPAM_MODULE - /* This section is used when compiling spammodule.c */ - - static PySpam_System_RETURN PySpam_System PySpam_System_PROTO; - - #else - /* This section is used in modules that use spammodule's API */ - - static void **PySpam_API; - - #define PySpam_System \ - (*(PySpam_System_RETURN (*)PySpam_System_PROTO) PySpam_API[PySpam_System_NUM]) - - /* Return -1 on error, 0 on success. - * PyCapsule_Import will set an exception if there's an error. - */ - static int - import_spam(void) - { - PySpam_API = (void **)PyCapsule_Import("spam._C_API", 0); - return (PySpam_API != NULL) ? 0 : -1; - } - - #endif - - #ifdef __cplusplus - } - #endif - - #endif /* !defined(Py_SPAMMODULE_H) */ - -All that a client module must do in order to have access to the function -:c:func:`PySpam_System` is to call the function (or rather macro) -:c:func:`import_spam` in its initialization function:: - - PyMODINIT_FUNC - PyInit_client(void) - { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; - /* additional initialization can happen here */ - return m; - } - -The main disadvantage of this approach is that the file :file:`spammodule.h` is -rather complicated. However, the basic structure is the same for each function -that is exported, so it has to be learned only once. - -Finally it should be mentioned that Capsules offer additional functionality, -which is especially useful for memory allocation and deallocation of the pointer -stored in a Capsule. The details are described in the Python/C API Reference -Manual in the section :ref:`capsules` and in the implementation of Capsules (files -:file:`Include/pycapsule.h` and :file:`Objects/pycapsule.c` in the Python source -code distribution). - -.. rubric:: Footnotes - -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. - -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. - -.. [#] Checking that the reference count is at least 1 **does not work** --- the - reference count itself could be in freed memory and may thus be reused for - another object! - -.. [#] These guarantees don't hold when you use the "old" style calling convention --- - this is still found in much existing code. diff --git a/third_party/python/Doc/extending/index.rst b/third_party/python/Doc/extending/index.rst deleted file mode 100644 index 533ed985f..000000000 --- a/third_party/python/Doc/extending/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _extending-index: - -################################################## - Extending and Embedding the Python Interpreter -################################################## - -This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how -to embed the Python interpreter in another application, for use as an extension -language. Finally, it shows how to compile and link extension modules so that -they can be loaded dynamically (at run time) into the interpreter, if the -underlying operating system supports this feature. - -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` -gives a more formal definition of the language. :ref:`library-index` documents -the existing object types, functions and modules (both built-in and written in -Python) that give the language its wide application range. - -For a detailed description of the whole Python/C API, see the separate -:ref:`c-api-index`. - - -Recommended third party tools -============================= - -This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython <http://cython.org/>`_, `cffi <https://cffi.readthedocs.io>`_, -`SWIG <http://www.swig.org>`_ and `Numba <https://numba.pydata.org/>`_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/en/latest/extensions/>`_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. - - -Creating extensions without third party tools -============================================= - -This section of the guide covers creating C and C++ extensions without -assistance from third party tools. It is intended primarily for creators -of those tools, rather than being a recommended way to create your own -C extensions. - -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst - -Embedding the CPython runtime in a larger application -===================================================== - -Sometimes, rather than creating an extension that runs inside the Python -interpreter as the main application, it is desirable to instead embed -the CPython runtime inside a larger application. This section covers -some of the details involved in doing that successfully. - -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst diff --git a/third_party/python/Doc/extending/newtypes.rst b/third_party/python/Doc/extending/newtypes.rst deleted file mode 100644 index b55796421..000000000 --- a/third_party/python/Doc/extending/newtypes.rst +++ /dev/null @@ -1,619 +0,0 @@ -.. highlightlang:: c - -***************************************** -Defining Extension Types: Assorted Topics -***************************************** - -.. _dnt-type-methods: - -This section aims to give a quick fly-by on the various type methods you can -implement and what they do. - -Here is the definition of :c:type:`PyTypeObject`, with some fields only used in -debug builds omitted: - -.. literalinclude:: ../includes/typestruct.h - - -Now that's a *lot* of methods. Don't worry too much though -- if you have -a type you want to define, the chances are very good that you will only -implement a handful of these. - -As you probably expect by now, we're going to go over this and give more -information about the various handlers. We won't go in the order they are -defined in the structure, because there is a lot of historical baggage that -impacts the ordering of the fields. It's often easiest to find an example -that includes the fields you need and then change the values to suit your new -type. :: - - const char *tp_name; /* For printing */ - -The name of the type -- as mentioned in the previous chapter, this will appear in -various places, almost entirely for diagnostic purposes. Try to choose something -that will be helpful in such a situation! :: - - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ - -These fields tell the runtime how much memory to allocate when new objects of -this type are created. Python has some built-in support for variable length -structures (think: strings, tuples) which is where the :c:member:`~PyTypeObject.tp_itemsize` field -comes in. This will be dealt with later. :: - - const char *tp_doc; - -Here you can put a string (or its address) that you want returned when the -Python script references ``obj.__doc__`` to retrieve the doc string. - -Now we come to the basic type methods -- the ones most extension types will -implement. - - -Finalization and De-allocation ------------------------------- - -.. index:: - single: object; deallocation - single: deallocation, object - single: object; finalization - single: finalization, of objects - -:: - - destructor tp_dealloc; - -This function is called when the reference count of the instance of your type is -reduced to zero and the Python interpreter wants to reclaim it. If your type -has memory to free or other clean-up to perform, you can put it here. The -object itself needs to be freed here as well. Here is an example of this -function:: - - static void - newdatatype_dealloc(newdatatypeobject *obj) - { - free(obj->obj_UnderlyingDatatypePtr); - Py_TYPE(obj)->tp_free(obj); - } - -.. index:: - single: PyErr_Fetch() - single: PyErr_Restore() - -One important requirement of the deallocator function is that it leaves any -pending exceptions alone. This is important since deallocators are frequently -called as the interpreter unwinds the Python stack; when the stack is unwound -due to an exception (rather than normal returns), nothing is done to protect the -deallocators from seeing that an exception has already been set. Any actions -which a deallocator performs which may cause additional Python code to be -executed may detect that an exception has been set. This can lead to misleading -errors from the interpreter. The proper way to protect against this is to save -a pending exception before performing the unsafe action, and restoring it when -done. This can be done using the :c:func:`PyErr_Fetch` and -:c:func:`PyErr_Restore` functions:: - - static void - my_dealloc(PyObject *obj) - { - MyObject *self = (MyObject *) obj; - PyObject *cbresult; - - if (self->my_callback != NULL) { - PyObject *err_type, *err_value, *err_traceback; - - /* This saves the current exception state */ - PyErr_Fetch(&err_type, &err_value, &err_traceback); - - cbresult = PyObject_CallObject(self->my_callback, NULL); - if (cbresult == NULL) - PyErr_WriteUnraisable(self->my_callback); - else - Py_DECREF(cbresult); - - /* This restores the saved exception state */ - PyErr_Restore(err_type, err_value, err_traceback); - - Py_DECREF(self->my_callback); - } - Py_TYPE(obj)->tp_free((PyObject*)self); - } - -.. note:: - There are limitations to what you can safely do in a deallocator function. - First, if your type supports garbage collection (using :c:member:`~PyTypeObject.tp_traverse` - and/or :c:member:`~PyTypeObject.tp_clear`), some of the object's members can have been - cleared or finalized by the time :c:member:`~PyTypeObject.tp_dealloc` is called. Second, in - :c:member:`~PyTypeObject.tp_dealloc`, your object is in an unstable state: its reference - count is equal to zero. Any call to a non-trivial object or API (as in the - example above) might end up calling :c:member:`~PyTypeObject.tp_dealloc` again, causing a - double free and a crash. - - Starting with Python 3.4, it is recommended not to put any complex - finalization code in :c:member:`~PyTypeObject.tp_dealloc`, and instead use the new - :c:member:`~PyTypeObject.tp_finalize` type method. - - .. seealso:: - :pep:`442` explains the new finalization scheme. - -.. index:: - single: string; object representation - builtin: repr - -Object Presentation -------------------- - -In Python, there are two ways to generate a textual representation of an object: -the :func:`repr` function, and the :func:`str` function. (The :func:`print` -function just calls :func:`str`.) These handlers are both optional. - -:: - - reprfunc tp_repr; - reprfunc tp_str; - -The :c:member:`~PyTypeObject.tp_repr` handler should return a string object containing a -representation of the instance for which it is called. Here is a simple -example:: - - static PyObject * - newdatatype_repr(newdatatypeobject * obj) - { - return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); - } - -If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a -representation that uses the type's :c:member:`~PyTypeObject.tp_name` and a uniquely-identifying -value for the object. - -The :c:member:`~PyTypeObject.tp_str` handler is to :func:`str` what the :c:member:`~PyTypeObject.tp_repr` handler -described above is to :func:`repr`; that is, it is called when Python code calls -:func:`str` on an instance of your object. Its implementation is very similar -to the :c:member:`~PyTypeObject.tp_repr` function, but the resulting string is intended for human -consumption. If :c:member:`~PyTypeObject.tp_str` is not specified, the :c:member:`~PyTypeObject.tp_repr` handler is -used instead. - -Here is a simple example:: - - static PyObject * - newdatatype_str(newdatatypeobject * obj) - { - return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); - } - - - -Attribute Management --------------------- - -For every object which can support attributes, the corresponding type must -provide the functions that control how the attributes are resolved. There needs -to be a function which can retrieve attributes (if any are defined), and another -to set attributes (if setting attributes is allowed). Removing an attribute is -a special case, for which the new value passed to the handler is *NULL*. - -Python supports two pairs of attribute handlers; a type that supports attributes -only needs to implement the functions for one pair. The difference is that one -pair takes the name of the attribute as a :c:type:`char\*`, while the other -accepts a :c:type:`PyObject\*`. Each type can use whichever pair makes more -sense for the implementation's convenience. :: - - getattrfunc tp_getattr; /* char * version */ - setattrfunc tp_setattr; - /* ... */ - getattrofunc tp_getattro; /* PyObject * version */ - setattrofunc tp_setattro; - -If accessing attributes of an object is always a simple operation (this will be -explained shortly), there are generic implementations which can be used to -provide the :c:type:`PyObject\*` version of the attribute management functions. -The actual need for type-specific attribute handlers almost completely -disappeared starting with Python 2.2, though there are many examples which have -not been updated to use some of the new generic mechanism that is available. - - -.. _generic-attribute-management: - -Generic Attribute Management -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most extension types only use *simple* attributes. So, what makes the -attributes simple? There are only a couple of conditions that must be met: - -#. The name of the attributes must be known when :c:func:`PyType_Ready` is - called. - -#. No special processing is needed to record that an attribute was looked up or - set, nor do actions need to be taken based on the value. - -Note that this list does not place any restrictions on the values of the -attributes, when the values are computed, or how relevant data is stored. - -When :c:func:`PyType_Ready` is called, it uses three tables referenced by the -type object to create :term:`descriptor`\s which are placed in the dictionary of the -type object. Each descriptor controls access to one attribute of the instance -object. Each of the tables is optional; if all three are *NULL*, instances of -the type will only have attributes that are inherited from their base type, and -should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as -well, allowing the base type to handle attributes. - -The tables are declared as three fields of the type object:: - - struct PyMethodDef *tp_methods; - struct PyMemberDef *tp_members; - struct PyGetSetDef *tp_getset; - -If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of -:c:type:`PyMethodDef` structures. Each entry in the table is an instance of this -structure:: - - typedef struct PyMethodDef { - const char *ml_name; /* method name */ - PyCFunction ml_meth; /* implementation function */ - int ml_flags; /* flags */ - const char *ml_doc; /* docstring */ - } PyMethodDef; - -One entry should be defined for each method provided by the type; no entries are -needed for methods inherited from a base type. One additional entry is needed -at the end; it is a sentinel that marks the end of the array. The -:attr:`ml_name` field of the sentinel must be *NULL*. - -The second table is used to define attributes which map directly to data stored -in the instance. A variety of primitive C types are supported, and access may -be read-only or read-write. The structures in the table are defined as:: - - typedef struct PyMemberDef { - char *name; - int type; - int offset; - int flags; - char *doc; - } PyMemberDef; - -For each entry in the table, a :term:`descriptor` will be constructed and added to the -type which will be able to extract a value from the instance structure. The -:attr:`type` field should contain one of the type codes defined in the -:file:`structmember.h` header; the value will be used to determine how to -convert Python values to and from C values. The :attr:`flags` field is used to -store flags which control how the attribute can be accessed. - -The following flag constants are defined in :file:`structmember.h`; they may be -combined using bitwise-OR. - -+---------------------------+----------------------------------------------+ -| Constant | Meaning | -+===========================+==============================================+ -| :const:`READONLY` | Never writable. | -+---------------------------+----------------------------------------------+ -| :const:`READ_RESTRICTED` | Not readable in restricted mode. | -+---------------------------+----------------------------------------------+ -| :const:`WRITE_RESTRICTED` | Not writable in restricted mode. | -+---------------------------+----------------------------------------------+ -| :const:`RESTRICTED` | Not readable or writable in restricted mode. | -+---------------------------+----------------------------------------------+ - -.. index:: - single: READONLY - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED - -An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build -descriptors that are used at runtime is that any attribute defined this way can -have an associated doc string simply by providing the text in the table. An -application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`__doc__` attribute. - -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value -of *NULL* is required. - -.. XXX Descriptors need to be explained in more detail somewhere, but not here. - - Descriptor objects have two handler functions which correspond to the - \member{tp_getattro} and \member{tp_setattro} handlers. The - \method{__get__()} handler is a function which is passed the descriptor, - instance, and type objects, and returns the value of the attribute, or it - returns \NULL{} and sets an exception. The \method{__set__()} handler is - passed the descriptor, instance, type, and new value; - - -Type-specific Attribute Management -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For simplicity, only the :c:type:`char\*` version will be demonstrated here; the -type of the name parameter is the only difference between the :c:type:`char\*` -and :c:type:`PyObject\*` flavors of the interface. This example effectively does -the same thing as the generic example above, but does not use the generic -support added in Python 2.2. It explains how the handler functions are -called, so that if you do need to extend their functionality, you'll understand -what needs to be done. - -The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute -look-up. It is called in the same situations where the :meth:`__getattr__` -method of a class would be called. - -Here is an example:: - - static PyObject * - newdatatype_getattr(newdatatypeobject *obj, char *name) - { - if (strcmp(name, "data") == 0) - { - return PyLong_FromLong(obj->data); - } - - PyErr_Format(PyExc_AttributeError, - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, name); - return NULL; - } - -The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or -:meth:`__delattr__` method of a class instance would be called. When an -attribute should be deleted, the third parameter will be *NULL*. Here is an -example that simply raises an exception; if this were really all you wanted, the -:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: - - static int - newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) - { - PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name); - return -1; - } - -Object Comparison ------------------ - -:: - - richcmpfunc tp_richcompare; - -The :c:member:`~PyTypeObject.tp_richcompare` handler is called when comparisons are needed. It is -analogous to the :ref:`rich comparison methods <richcmpfuncs>`, like -:meth:`__lt__`, and also called by :c:func:`PyObject_RichCompare` and -:c:func:`PyObject_RichCompareBool`. - -This function is called with two Python objects and the operator as arguments, -where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GT``, -``Py_LT`` or ``Py_GT``. It should compare the two objects with respect to the -specified operator and return ``Py_True`` or ``Py_False`` if the comparison is -successful, ``Py_NotImplemented`` to indicate that comparison is not -implemented and the other object's comparison method should be tried, or *NULL* -if an exception was set. - -Here is a sample implementation, for a datatype that is considered equal if the -size of an internal pointer is equal:: - - static PyObject * - newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op) - { - PyObject *result; - int c, size1, size2; - - /* code to make sure that both arguments are of type - newdatatype omitted */ - - size1 = obj1->obj_UnderlyingDatatypePtr->size; - size2 = obj2->obj_UnderlyingDatatypePtr->size; - - switch (op) { - case Py_LT: c = size1 < size2; break; - case Py_LE: c = size1 <= size2; break; - case Py_EQ: c = size1 == size2; break; - case Py_NE: c = size1 != size2; break; - case Py_GT: c = size1 > size2; break; - case Py_GE: c = size1 >= size2; break; - } - result = c ? Py_True : Py_False; - Py_INCREF(result); - return result; - } - - -Abstract Protocol Support -------------------------- - -Python supports a variety of *abstract* 'protocols;' the specific interfaces -provided to use these interfaces are documented in :ref:`abstract`. - - -A number of these abstract interfaces were defined early in the development of -the Python implementation. In particular, the number, mapping, and sequence -protocols have been part of Python since the beginning. Other protocols have -been added over time. For protocols which depend on several handler routines -from the type implementation, the older protocols have been defined as optional -blocks of handlers referenced by the type object. For newer protocols there are -additional slots in the main type object, with a flag bit being set to indicate -that the slots are present and should be checked by the interpreter. (The flag -bit does not indicate that the slot values are non-*NULL*. The flag may be set -to indicate the presence of a slot, but a slot may still be unfilled.) :: - - PyNumberMethods *tp_as_number; - PySequenceMethods *tp_as_sequence; - PyMappingMethods *tp_as_mapping; - -If you wish your object to be able to act like a number, a sequence, or a -mapping object, then you place the address of a structure that implements the C -type :c:type:`PyNumberMethods`, :c:type:`PySequenceMethods`, or -:c:type:`PyMappingMethods`, respectively. It is up to you to fill in this -structure with appropriate values. You can find examples of the use of each of -these in the :file:`Objects` directory of the Python source distribution. :: - - hashfunc tp_hash; - -This function, if you choose to provide it, should return a hash number for an -instance of your data type. Here is a simple example:: - - static Py_hash_t - newdatatype_hash(newdatatypeobject *obj) - { - Py_hash_t result; - result = obj->some_size + 32767 * obj->some_number; - if (result == -1) - result = -2; - return result; - } - -:c:type:`Py_hash_t` is a signed integer type with a platform-varying width. -Returning ``-1`` from :c:member:`~PyTypeObject.tp_hash` indicates an error, -which is why you should be careful to avoid returning it when hash computation -is successful, as seen above. - -:: - - ternaryfunc tp_call; - -This function is called when an instance of your data type is "called", for -example, if ``obj1`` is an instance of your data type and the Python script -contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is invoked. - -This function takes three arguments: - -#. *self* is the instance of the data type which is the subject of the call. - If the call is ``obj1('hello')``, then *self* is ``obj1``. - -#. *args* is a tuple containing the arguments to the call. You can use - :c:func:`PyArg_ParseTuple` to extract the arguments. - -#. *kwds* is a dictionary of keyword arguments that were passed. If this is - non-*NULL* and you support keyword arguments, use - :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you - do not want to support keyword arguments and this is non-*NULL*, raise a - :exc:`TypeError` with a message saying that keyword arguments are not supported. - -Here is a toy ``tp_call`` implementation:: - - static PyObject * - newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds) - { - PyObject *result; - char *arg1; - char *arg2; - char *arg3; - - if (!PyArg_ParseTuple(args, "sss:call", &arg1, &arg2, &arg3)) { - return NULL; - } - result = PyUnicode_FromFormat( - "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n", - obj->obj_UnderlyingDatatypePtr->size, - arg1, arg2, arg3); - return result; - } - -:: - - /* Iterators */ - getiterfunc tp_iter; - iternextfunc tp_iternext; - -These functions provide support for the iterator protocol. Both handlers -take exactly one parameter, the instance for which they are being called, -and return a new reference. In the case of an error, they should set an -exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds -to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` -corresponds to the Python :meth:`~iterator.__next__` method. - -Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter` -handler, which must return an :term:`iterator` object. Here the same guidelines -apply as for Python classes: - -* For collections (such as lists and tuples) which can support multiple - independent iterators, a new iterator should be created and returned by - each call to :c:member:`~PyTypeObject.tp_iter`. -* Objects which can only be iterated over once (usually due to side effects of - iteration, such as file objects) can implement :c:member:`~PyTypeObject.tp_iter` - by returning a new reference to themselves -- and should also therefore - implement the :c:member:`~PyTypeObject.tp_iternext` handler. - -Any :term:`iterator` object should implement both :c:member:`~PyTypeObject.tp_iter` -and :c:member:`~PyTypeObject.tp_iternext`. An iterator's -:c:member:`~PyTypeObject.tp_iter` handler should return a new reference -to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should -return a new reference to the next object in the iteration, if there is one. -If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext` -may return *NULL* without setting an exception, or it may set -:exc:`StopIteration` *in addition* to returning *NULL*; avoiding -the exception can yield slightly better performance. If an actual error -occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception -and return *NULL*. - - -.. _weakref-support: - -Weak Reference Support ----------------------- - -One of the goals of Python's weak reference implementation is to allow any type -to participate in the weak reference mechanism without incurring the overhead on -performance-critical objects (such as numbers). - -.. seealso:: - Documentation for the :mod:`weakref` module. - -For an object to be weakly referencable, the extension type must do two things: - -#. Include a :c:type:`PyObject\*` field in the C object structure dedicated to - the weak reference mechanism. The object's constructor should leave it - *NULL* (which is automatic when using the default - :c:member:`~PyTypeObject.tp_alloc`). - -#. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member - to the offset of the aforementioned field in the C object structure, - so that the interpreter knows how to access and modify that field. - -Concretely, here is how a trivial object structure would be augmented -with the required field:: - - typedef struct { - PyObject_HEAD - PyObject *weakreflist; /* List of weak references */ - } TrivialObject; - -And the corresponding member in the statically-declared type object:: - - static PyTypeObject TrivialType = { - PyVarObject_HEAD_INIT(NULL, 0) - /* ... other members omitted for brevity ... */ - .tp_weaklistoffset = offsetof(TrivialObject, weakreflist), - }; - -The only further addition is that ``tp_dealloc`` needs to clear any weak -references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is -non-*NULL*:: - - static void - Trivial_dealloc(TrivialObject *self) - { - /* Clear weakrefs first before calling any destructors */ - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - /* ... remainder of destruction code omitted for brevity ... */ - Py_TYPE(self)->tp_free((PyObject *) self); - } - - -More Suggestions ----------------- - -In order to learn how to implement any specific method for your new data type, -get the :term:`CPython` source code. Go to the :file:`Objects` directory, -then search the C source files for ``tp_`` plus the function you want -(for example, ``tp_richcompare``). You will find examples of the function -you want to implement. - -When you need to verify that an object is a concrete instance of the type you -are implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of -its use might be something like the following:: - - if (!PyObject_TypeCheck(some_object, &MyType)) { - PyErr_SetString(PyExc_TypeError, "arg #1 not a mything"); - return NULL; - } - -.. seealso:: - Download CPython source releases. - https://www.python.org/downloads/source/ - - The CPython project on GitHub, where the CPython source code is developed. - https://github.com/python/cpython diff --git a/third_party/python/Doc/extending/newtypes_tutorial.rst b/third_party/python/Doc/extending/newtypes_tutorial.rst deleted file mode 100644 index ac48637bb..000000000 --- a/third_party/python/Doc/extending/newtypes_tutorial.rst +++ /dev/null @@ -1,896 +0,0 @@ -.. highlightlang:: c - -.. _defining-new-types: - -********************************** -Defining Extension Types: Tutorial -********************************** - -.. sectionauthor:: Michael Hudson <mwh@python.net> -.. sectionauthor:: Dave Kuhlman <dkuhlman@rexx.com> -.. sectionauthor:: Jim Fulton <jim@zope.com> - - -Python allows the writer of a C extension module to define new types that -can be manipulated from Python code, much like the built-in :class:`str` -and :class:`list` types. The code for all extension types follows a -pattern, but there are some details that you need to understand before you -can get started. This document is a gentle introduction to the topic. - - -.. _dnt-basics: - -The Basics -========== - -The :term:`CPython` runtime sees all Python objects as variables of type -:c:type:`PyObject\*`, which serves as a "base type" for all Python objects. -The :c:type:`PyObject` structure itself only contains the object's -:term:`reference count` and a pointer to the object's "type object". -This is where the action is; the type object determines which (C) functions -get called by the interpreter when, for instance, an attribute gets looked up -on an object, a method called, or it is multiplied by another object. These -C functions are called "type methods". - -So, if you want to define a new extension type, you need to create a new type -object. - -This sort of thing can only be explained by example, so here's a minimal, but -complete, module that defines a new type named :class:`Custom` inside a C -extension module :mod:`custom`: - -.. note:: - What we're showing here is the traditional way of defining *static* - extension types. It should be adequate for most uses. The C API also - allows defining heap-allocated extension types using the - :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. - -.. literalinclude:: ../includes/custom.c - -Now that's quite a bit to take in at once, but hopefully bits will seem familiar -from the previous chapter. This file defines three things: - -#. What a :class:`Custom` **object** contains: this is the ``CustomObject`` - struct, which is allocated once for each :class:`Custom` instance. -#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct, - which defines a set of flags and function pointers that the interpreter - inspects when specific operations are requested. -#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. - -The first bit is:: - - typedef struct { - PyObject_HEAD - } CustomObject; - -This is what a Custom object will contain. ``PyObject_HEAD`` is mandatory -at the start of each object struct and defines a field called ``ob_base`` -of type :c:type:`PyObject`, containing a pointer to a type object and a -reference count (these can be accessed using the macros :c:macro:`Py_REFCNT` -and :c:macro:`Py_TYPE` respectively). The reason for the macro is to -abstract away the layout and to enable additional fields in debug builds. - -.. note:: - There is no semicolon above after the :c:macro:`PyObject_HEAD` macro. - Be wary of adding one by accident: some compilers will complain. - -Of course, objects generally store additional data besides the standard -``PyObject_HEAD`` boilerplate; for example, here is the definition for -standard Python floats:: - - typedef struct { - PyObject_HEAD - double ob_fval; - } PyFloatObject; - -The second bit is the definition of the type object. :: - - static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom.Custom", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_new = PyType_GenericNew, - }; - -.. note:: - We recommend using C99-style designated initializers as above, to - avoid listing all the :c:type:`PyTypeObject` fields that you don't care - about and also to avoid caring about the fields' declaration order. - -The actual definition of :c:type:`PyTypeObject` in :file:`object.h` has -many more :ref:`fields <type-structs>` than the definition above. The -remaining fields will be filled with zeros by the C compiler, and it's -common practice to not specify them explicitly unless you need them. - -We're going to pick it apart, one field at a time:: - - PyVarObject_HEAD_INIT(NULL, 0) - -This line is mandatory boilerplate to initialize the ``ob_base`` -field mentioned above. :: - - .tp_name = "custom.Custom", - -The name of our type. This will appear in the default textual representation of -our objects and in some error messages, for example: - -.. code-block:: pycon - - >>> "" + custom.Custom() - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: can only concatenate str (not "custom.Custom") to str - -Note that the name is a dotted name that includes both the module name and the -name of the type within the module. The module in this case is :mod:`custom` and -the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`. -Using the real dotted import path is important to make your type compatible -with the :mod:`pydoc` and :mod:`pickle` modules. :: - - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - -This is so that Python knows how much memory to allocate when creating -new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is -only used for variable-sized objects and should otherwise be zero. - -.. note:: - - If you want your type to be subclassable from Python, and your type has the same - :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple - inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's - :meth:`__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its - base type does. Most of the time, this will be true anyway, because either your - base type will be :class:`object`, or else you will be adding data members to - your base type, and therefore increasing its size. - -We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. :: - - .tp_flags = Py_TPFLAGS_DEFAULT, - -All types should include this constant in their flags. It enables all of the -members defined until at least Python 3.3. If you need further members, -you will need to OR the corresponding flags. - -We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: - - .tp_doc = "Custom objects", - -To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` -handler. This is the equivalent of the Python method :meth:`__new__`, but -has to be specified explicitly. In this case, we can just use the default -implementation provided by the API function :c:func:`PyType_GenericNew`. :: - - .tp_new = PyType_GenericNew, - -Everything else in the file should be familiar, except for some code in -:c:func:`PyInit_custom`:: - - if (PyType_Ready(&CustomType) < 0) - return; - -This initializes the :class:`Custom` type, filling in a number of members -to the appropriate default values, including :attr:`ob_type` that we initially -set to *NULL*. :: - - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); - -This adds the type to the module dictionary. This allows us to create -:class:`Custom` instances by calling the :class:`Custom` class: - -.. code-block:: pycon - - >>> import custom - >>> mycustom = custom.Custom() - -That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: - -.. code-block:: python - - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) - -in a file called :file:`setup.py`; then typing - -.. code-block:: shell-session - - $ python setup.py build - -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. - -That wasn't so hard, was it? - -Of course, the current Custom type is pretty uninteresting. It has no data and -doesn't do anything. It can't even be subclassed. - -.. note:: - While this documentation showcases the standard :mod:`distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide <https://packaging.python.org/tutorials/distributing-packages/>`_. - - -Adding data and methods to the Basic example -============================================ - -Let's extend the basic example to add some data and methods. Let's also make -the type usable as a base class. We'll create a new module, :mod:`custom2` that -adds these capabilities: - -.. literalinclude:: ../includes/custom2.c - - -This version of the module has a number of changes. - -We've added an extra include:: - - #include <structmember.h> - -This include provides declarations that we use to handle attributes, as -described a bit later. - -The :class:`Custom` type now has three data attributes in its C struct, -*first*, *last*, and *number*. The *first* and *last* variables are Python -strings containing first and last names. The *number* attribute is a C integer. - -The object structure is updated accordingly:: - - typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; - } CustomObject; - -Because we now have data to manage, we have to be more careful about object -allocation and deallocation. At a minimum, we need a deallocation method:: - - static void - Custom_dealloc(CustomObject *self) - { - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); - } - -which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: - - .tp_dealloc = (destructor) Custom_dealloc, - -This method first clears the reference counts of the two Python attributes. -:c:func:`Py_XDECREF` correctly handles the case where its argument is -*NULL* (which might happen here if ``tp_new`` failed midway). It then -calls the :c:member:`~PyTypeObject.tp_free` member of the object's type -(computed by ``Py_TYPE(self)``) to free the object's memory. Note that -the object's type might not be :class:`CustomType`, because the object may -be an instance of a subclass. - -.. note:: - The explicit cast to ``destructor`` above is needed because we defined - ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc`` - function pointer expects to receive a ``PyObject *`` argument. Otherwise, - the compiler will emit a warning. This is object-oriented polymorphism, - in C! - -We want to make sure that the first and last names are initialized to empty -strings, so we provide a ``tp_new`` implementation:: - - static PyObject * - Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) - { - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; - } - -and install it in the :c:member:`~PyTypeObject.tp_new` member:: - - .tp_new = Custom_new, - -The ``tp_new`` handler is responsible for creating (as opposed to initializing) -objects of the type. It is exposed in Python as the :meth:`__new__` method. -It is not required to define a ``tp_new`` member, and indeed many extension -types will simply reuse :c:func:`PyType_GenericNew` as done in the first -version of the ``Custom`` type above. In this case, we use the ``tp_new`` -handler to initialize the ``first`` and ``last`` attributes to non-*NULL* -default values. - -``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``, -if a subclass is instantiated) and any arguments passed when the type was -called, and is expected to return the instance created. ``tp_new`` handlers -always accept positional and keyword arguments, but they often ignore the -arguments, leaving the argument handling to initializer (a.k.a. ``tp_init`` -in C or ``__init__`` in Python) methods. - -.. note:: - ``tp_new`` shouldn't call ``tp_init`` explicitly, as the interpreter - will do it itself. - -The ``tp_new`` implementation calls the :c:member:`~PyTypeObject.tp_alloc` -slot to allocate memory:: - - self = (CustomObject *) type->tp_alloc(type, 0); - -Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc` -result against *NULL* before proceeding. - -.. note:: - We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather - :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, - which is :class:`object` by default. Most types use the default allocation - strategy. - -.. note:: - If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one - that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), - you must *not* try to determine what method to call using method resolution - order at runtime. Always statically determine what type you are going to - call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via - ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your - type that also inherit from other Python-defined classes may not work correctly. - (Specifically, you may not be able to create instances of such subclasses - without getting a :exc:`TypeError`.) - -We also define an initialization function which accepts arguments to provide -initial values for our instance:: - - static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) - { - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - return 0; - } - -by filling the :c:member:`~PyTypeObject.tp_init` slot. :: - - .tp_init = (initproc) Custom_init, - -The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the -:meth:`__init__` method. It is used to initialize an object after it's -created. Initializers always accept positional and keyword arguments, -and they should return either ``0`` on success or ``-1`` on error. - -Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init`` -is called at all (for example, the :mod:`pickle` module by default -doesn't call :meth:`__init__` on unpickled instances). It can also be -called multiple times. Anyone can call the :meth:`__init__` method on -our objects. For this reason, we have to be extra careful when assigning -the new attribute values. We might be tempted, for example to assign the -``first`` member like this:: - - if (first) { - Py_XDECREF(self->first); - Py_INCREF(first); - self->first = first; - } - -But this would be risky. Our type doesn't restrict the type of the -``first`` member, so it could be any kind of object. It could have a -destructor that causes code to be executed that tries to access the -``first`` member; or that destructor could release the -:term:`Global interpreter Lock` and let arbitrary code run in other -threads that accesses and modifies our object. - -To be paranoid and protect ourselves against this possibility, we almost -always reassign members before decrementing their reference counts. When -don't we have to do this? - -* when we absolutely know that the reference count is greater than 1; - -* when we know that deallocation of the object [#]_ will neither release - the :term:`GIL` nor cause any calls back into our type's code; - -* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` - handler on a type which doesn't support cyclic garbage collection [#]_. - -We want to expose our instance variables as attributes. There are a -number of ways to do that. The simplest way is to define member definitions:: - - static PyMemberDef Custom_members[] = { - {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, - "last name"}, - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ - }; - -and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: - - .tp_members = Custom_members, - -Each member definition has a member name, type, offset, access flags and -documentation string. See the :ref:`Generic-Attribute-Management` section -below for details. - -A disadvantage of this approach is that it doesn't provide a way to restrict the -types of objects that can be assigned to the Python attributes. We expect the -first and last names to be strings, but any Python objects can be assigned. -Further, the attributes can be deleted, setting the C pointers to *NULL*. Even -though we can make sure the members are initialized to non-*NULL* values, the -members can be set to *NULL* if the attributes are deleted. - -We define a single method, :meth:`Custom.name()`, that outputs the objects name as the -concatenation of the first and last names. :: - - static PyObject * - Custom_name(CustomObject *self) - { - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - return PyUnicode_FromFormat("%S %S", self->first, self->last); - } - -The method is implemented as a C function that takes a :class:`Custom` (or -:class:`Custom` subclass) instance as the first argument. Methods always take an -instance as the first argument. Methods often take positional and keyword -arguments as well, but in this case we don't take any and don't need to accept -a positional argument tuple or keyword argument dictionary. This method is -equivalent to the Python method: - -.. code-block:: python - - def name(self): - return "%s %s" % (self.first, self.last) - -Note that we have to check for the possibility that our :attr:`first` and -:attr:`last` members are *NULL*. This is because they can be deleted, in which -case they are set to *NULL*. It would be better to prevent deletion of these -attributes and to restrict the attribute values to be strings. We'll see how to -do that in the next section. - -Now that we've defined the method, we need to create an array of method -definitions:: - - static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ - }; - -(note that we used the :const:`METH_NOARGS` flag to indicate that the method -is expecting no arguments other than *self*) - -and assign it to the :c:member:`~PyTypeObject.tp_methods` slot:: - - .tp_methods = Custom_methods, - -Finally, we'll make our type usable as a base class for subclassing. We've -written our methods carefully so far so that they don't make any assumptions -about the type of the object being created or used, so all we need to do is -to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition:: - - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - -We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the -module name in the :c:type:`PyModuleDef` struct, and update the full class -name in the :c:type:`PyTypeObject` struct. - -Finally, we update our :file:`setup.py` file to build the new module: - -.. code-block:: python - - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) - - -Providing finer control over data attributes -============================================ - -In this section, we'll provide finer control over how the :attr:`first` and -:attr:`last` attributes are set in the :class:`Custom` example. In the previous -version of our module, the instance variables :attr:`first` and :attr:`last` -could be set to non-string values or even deleted. We want to make sure that -these attributes always contain strings. - -.. literalinclude:: ../includes/custom3.c - - -To provide greater control, over the :attr:`first` and :attr:`last` attributes, -we'll use custom getter and setter functions. Here are the functions for -getting and setting the :attr:`first` attribute:: - - static PyObject * - Custom_getfirst(CustomObject *self, void *closure) - { - Py_INCREF(self->first); - return self->first; - } - - static int - Custom_setfirst(CustomObject *self, PyObject *value, void *closure) - { - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - tmp = self->first; - Py_INCREF(value); - self->first = value; - Py_DECREF(tmp); - return 0; - } - -The getter function is passed a :class:`Custom` object and a "closure", which is -a void pointer. In this case, the closure is ignored. (The closure supports an -advanced usage in which definition data is passed to the getter and setter. This -could, for example, be used to allow a single set of getter and setter functions -that decide the attribute to get or set based on data in the closure.) - -The setter function is passed the :class:`Custom` object, the new value, and the -closure. The new value may be *NULL*, in which case the attribute is being -deleted. In our setter, we raise an error if the attribute is deleted or if its -new value is not a string. - -We create an array of :c:type:`PyGetSetDef` structures:: - - static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ - }; - -and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: - - .tp_getset = Custom_getsetters, - -The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned -above. In this case, we aren't using a closure, so we just pass *NULL*. - -We also remove the member definitions for these attributes:: - - static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ - }; - -We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only -allow strings [#]_ to be passed:: - - static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) - { - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; - } - -With these changes, we can assure that the ``first`` and ``last`` members are -never *NULL* so we can remove checks for *NULL* values in almost all cases. -This means that most of the :c:func:`Py_XDECREF` calls can be converted to -:c:func:`Py_DECREF` calls. The only place we can't change these calls is in -the ``tp_dealloc`` implementation, where there is the possibility that the -initialization of these members failed in ``tp_new``. - -We also rename the module initialization function and module name in the -initialization function, as we did before, and we add an extra definition to the -:file:`setup.py` file. - - -Supporting cyclic garbage collection -==================================== - -Python has a :term:`cyclic garbage collector (GC) <garbage collection>` that -can identify unneeded objects even when their reference counts are not zero. -This can happen when objects are involved in cycles. For example, consider: - -.. code-block:: pycon - - >>> l = [] - >>> l.append(l) - >>> del l - -In this example, we create a list that contains itself. When we delete it, it -still has a reference from itself. Its reference count doesn't drop to zero. -Fortunately, Python's cyclic garbage collector will eventually figure out that -the list is garbage and free it. - -In the second version of the :class:`Custom` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. -Besides, in the second and third versions, we allowed subclassing -:class:`Custom`, and subclasses may add arbitrary attributes. For any of -those two reasons, :class:`Custom` objects can participate in cycles: - -.. code-block:: pycon - - >>> import custom3 - >>> class Derived(custom3.Custom): pass - ... - >>> n = Derived() - >>> n.some_attribute = n - -To allow a :class:`Custom` instance participating in a reference cycle to -be properly detected and collected by the cyclic GC, our :class:`Custom` type -needs to fill two additional slots and to enable a flag that enables these slots: - -.. literalinclude:: ../includes/custom4.c - - -First, the traversal method lets the cyclic GC know about subobjects that could -participate in cycles:: - - static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) - { - int vret; - if (self->first) { - vret = visit(self->first, arg); - if (vret != 0) - return vret; - } - if (self->last) { - vret = visit(self->last, arg); - if (vret != 0) - return vret; - } - return 0; - } - -For each subobject that can participate in cycles, we need to call the -:c:func:`visit` function, which is passed to the traversal method. The -:c:func:`visit` function takes as arguments the subobject and the extra argument -*arg* passed to the traversal method. It returns an integer value that must be -returned if it is non-zero. - -Python provides a :c:func:`Py_VISIT` macro that automates calling visit -functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate -in ``Custom_traverse``:: - - static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) - { - Py_VISIT(self->first); - Py_VISIT(self->last); - return 0; - } - -.. note:: - The :c:member:`~PyTypeObject.tp_traverse` implementation must name its - arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. - -Second, we need to provide a method for clearing any subobjects that can -participate in cycles:: - - static int - Custom_clear(CustomObject *self) - { - Py_CLEAR(self->first); - Py_CLEAR(self->last); - return 0; - } - -Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe -way to clear data attributes of arbitrary types while decrementing -their reference counts. If you were to call :c:func:`Py_XDECREF` instead -on the attribute before setting it to *NULL*, there is a possibility -that the attribute's destructor would call back into code that reads the -attribute again (*especially* if there is a reference cycle). - -.. note:: - You could emulate :c:func:`Py_CLEAR` by writing:: - - PyObject *tmp; - tmp = self->first; - self->first = NULL; - Py_XDECREF(tmp); - - Nevertheless, it is much easier and less error-prone to always - use :c:func:`Py_CLEAR` when deleting an attribute. Don't - try to micro-optimize at the expense of robustness! - -The deallocator ``Custom_dealloc`` may call arbitrary code when clearing -attributes. It means the circular GC can be triggered inside the function. -Since the GC assumes reference count is not zero, we need to untrack the object -from the GC by calling :c:func:`PyObject_GC_UnTrack` before clearing members. -Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack` -and ``Custom_clear``:: - - static void - Custom_dealloc(CustomObject *self) - { - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); - } - -Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: - - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - -That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or -:c:member:`~PyTypeObject.tp_free` handlers, we'd need to modify them for cyclic -garbage collection. Most extensions will use the versions automatically provided. - - -Subclassing other types -======================= - -It is possible to create new extension types that are derived from existing -types. It is easiest to inherit from the built in types, since an extension can -easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share -these :c:type:`PyTypeObject` structures between extension modules. - -In this example we will create a :class:`SubList` type that inherits from the -built-in :class:`list` type. The new type will be completely compatible with -regular lists, but will have an additional :meth:`increment` method that -increases an internal counter: - -.. code-block:: pycon - - >>> import sublist - >>> s = sublist.SubList(range(3)) - >>> s.extend(s) - >>> print(len(s)) - 6 - >>> print(s.increment()) - 1 - >>> print(s.increment()) - 2 - -.. literalinclude:: ../includes/sublist.c - - -As you can see, the source code closely resembles the :class:`Custom` examples in -previous sections. We will break down the main differences between them. :: - - typedef struct { - PyListObject list; - int state; - } SubListObject; - -The primary difference for derived type objects is that the base type's -object structure must be the first value. The base type will already include -the :c:func:`PyObject_HEAD` at the beginning of its structure. - -When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer -can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: - - static int - SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) - { - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; - } - -We see above how to call through to the :attr:`__init__` method of the base -type. - -This pattern is important when writing a type with custom -:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_dealloc` -members. The :c:member:`~PyTypeObject.tp_new` handler should not actually -create the memory for the object with its :c:member:`~PyTypeObject.tp_alloc`, -but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_new`. - -The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` -specifying the type's concrete base class. Due to cross-platform compiler -issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization -function:: - - PyMODINIT_FUNC - PyInit_sublist(void) - { - PyObject* m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); - return m; - } - -Before calling :c:func:`PyType_Ready`, the type structure must have the -:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving an -existing type, it is not necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` -slot with :c:func:`PyType_GenericNew` -- the allocation function from the base -type will be inherited. - -After that, calling :c:func:`PyType_Ready` and adding the type object to the -module is the same as with the basic :class:`Custom` examples. - - -.. rubric:: Footnotes - -.. [#] This is true when we know that the object is a basic type, like a string or a - float. - -.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler - in this example, because our type doesn't support garbage collection. - -.. [#] We now know that the first and last members are strings, so perhaps we - could be less careful about decrementing their reference counts, however, - we accept instances of string subclasses. Even though deallocating normal - strings won't call back into our objects, we can't guarantee that deallocating - an instance of a string subclass won't call back into our objects. - -.. [#] Also, even with our attributes restricted to strings instances, the user - could pass arbitrary :class:`str` subclasses and therefore still create - reference cycles. diff --git a/third_party/python/Doc/extending/windows.rst b/third_party/python/Doc/extending/windows.rst deleted file mode 100644 index 67bdd475a..000000000 --- a/third_party/python/Doc/extending/windows.rst +++ /dev/null @@ -1,137 +0,0 @@ -.. highlightlang:: c - - -.. _building-on-windows: - -**************************************** -Building C and C++ Extensions on Windows -**************************************** - -This chapter briefly explains how to create a Windows extension module for -Python using Microsoft Visual C++, and follows with more detailed background -information on how it works. The explanatory material is useful for both the -Windows programmer learning to build Python extensions and the Unix programmer -interested in producing software which can be successfully built on both Unix -and Windows. - -Module authors are encouraged to use the distutils approach for building -extension modules, instead of the one described in this section. You will still -need the C compiler that was used to build Python; typically Microsoft Visual -C++. - -.. note:: - - This chapter mentions a number of filenames that include an encoded Python - version number. These filenames are represented with the version number shown - as ``XY``; in practice, ``'X'`` will be the major version number and ``'Y'`` - will be the minor version number of the Python release you're working with. For - example, if you are using Python 2.2.1, ``XY`` will actually be ``22``. - - -.. _win-cookbook: - -A Cookbook Approach -=================== - -There are two approaches to building extension modules on Windows, just as there -are on Unix: use the :mod:`distutils` package to control the build process, or -do things manually. The distutils approach works well for most extensions; -documentation on using :mod:`distutils` to build and package extension modules -is available in :ref:`distutils-index`. If you find you really need to do -things manually, it may be instructive to study the project file for the -:source:`winsound <PCbuild/winsound.vcxproj>` standard library module. - - -.. _dynamic-linking: - -Differences Between Unix and Windows -==================================== - -.. sectionauthor:: Chris Phoenix <cphoenix@best.com> - - -Unix and Windows use completely different paradigms for run-time loading of -code. Before you try to build a module that can be dynamically loaded, be aware -of how your system works. - -In Unix, a shared object (:file:`.so`) file contains code to be used by the -program, and also the names of functions and data that it expects to find in the -program. When the file is joined to the program, all references to those -functions and data in the file's code are changed to point to the actual -locations in the program where the functions and data are placed in memory. -This is basically a link operation. - -In Windows, a dynamic-link library (:file:`.dll`) file has no dangling -references. Instead, an access to functions or data goes through a lookup -table. So the DLL code does not have to be fixed up at runtime to refer to the -program's memory; instead, the code already uses the DLL's lookup table, and the -lookup table is modified at runtime to point to the functions and data. - -In Unix, there is only one type of library file (:file:`.a`) which contains code -from several object files (:file:`.o`). During the link step to create a shared -object file (:file:`.so`), the linker may find that it doesn't know where an -identifier is defined. The linker will look for it in the object files in the -libraries; if it finds it, it will include all the code from that object file. - -In Windows, there are two types of library, a static library and an import -library (both called :file:`.lib`). A static library is like a Unix :file:`.a` -file; it contains code to be included as necessary. An import library is -basically used only to reassure the linker that a certain identifier is legal, -and will be present in the program when the DLL is loaded. So the linker uses -the information from the import library to build the lookup table for using -identifiers that are not included in the DLL. When an application or a DLL is -linked, an import library may be generated, which will need to be used for all -future DLLs that depend on the symbols in the application or DLL. - -Suppose you are building two dynamic-load modules, B and C, which should share -another block of code A. On Unix, you would *not* pass :file:`A.a` to the -linker for :file:`B.so` and :file:`C.so`; that would cause it to be included -twice, so that B and C would each have their own copy. In Windows, building -:file:`A.dll` will also build :file:`A.lib`. You *do* pass :file:`A.lib` to the -linker for B and C. :file:`A.lib` does not contain code; it just contains -information which will be used at runtime to access A's code. - -In Windows, using an import library is sort of like using ``import spam``; it -gives you access to spam's names, but does not create a separate copy. On Unix, -linking with a library is more like ``from spam import *``; it does create a -separate copy. - - -.. _win-dlls: - -Using DLLs in Practice -====================== - -.. sectionauthor:: Chris Phoenix <cphoenix@best.com> - - -Windows Python is built in Microsoft Visual C++; using other compilers may or -may not work (though Borland seems to). The rest of this section is MSVC++ -specific. - -When creating DLLs in Windows, you must pass :file:`pythonXY.lib` to the linker. -To build two DLLs, spam and ni (which uses C functions found in spam), you could -use these commands:: - - cl /LD /I/python/include spam.c ../libs/pythonXY.lib - cl /LD /I/python/include ni.c spam.lib ../libs/pythonXY.lib - -The first command created three files: :file:`spam.obj`, :file:`spam.dll` and -:file:`spam.lib`. :file:`Spam.dll` does not contain any Python functions (such -as :c:func:`PyArg_ParseTuple`), but it does know how to find the Python code -thanks to :file:`pythonXY.lib`. - -The second command created :file:`ni.dll` (and :file:`.obj` and :file:`.lib`), -which knows how to find the necessary functions from spam, and also from the -Python executable. - -Not every identifier is exported to the lookup table. If you want any other -modules (including Python) to be able to see your identifiers, you have to say -``_declspec(dllexport)``, as in ``void _declspec(dllexport) initspam(void)`` or -``PyObject _declspec(dllexport) *NiGetSpamData(void)``. - -Developer Studio will throw in a lot of import libraries that you do not really -need, adding about 100K to your executable. To get rid of them, use the Project -Settings dialog, Link tab, to specify *ignore default libraries*. Add the -correct :file:`msvcrtxx.lib` to the list of libraries. - diff --git a/third_party/python/Doc/faq/design.rst b/third_party/python/Doc/faq/design.rst deleted file mode 100644 index 234dc9c12..000000000 --- a/third_party/python/Doc/faq/design.rst +++ /dev/null @@ -1,812 +0,0 @@ -====================== -Design and History FAQ -====================== - -.. only:: html - - .. contents:: - - -Why does Python use indentation for grouping of statements? ------------------------------------------------------------ - -Guido van Rossum believes that using indentation for grouping is extremely -elegant and contributes a lot to the clarity of the average Python program. -Most people learn to love this feature after a while. - -Since there are no begin/end brackets there cannot be a disagreement between -grouping perceived by the parser and the human reader. Occasionally C -programmers will encounter a fragment of code like this:: - - if (x <= y) - x++; - y--; - z++; - -Only the ``x++`` statement is executed if the condition is true, but the -indentation leads you to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering why ``y`` is being decremented even -for ``x > y``. - -Because there are no begin/end brackets, Python is much less prone to -coding-style conflicts. In C there are many different ways to place the braces. -If you're used to reading and writing code that uses one style, you will feel at -least slightly uneasy when reading (or being required to write) another style. - -Many coding styles place begin/end brackets on a line by themselves. This makes -programs considerably longer and wastes valuable screen space, making it harder -to get a good overview of a program. Ideally, a function should fit on one -screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 -lines of C. This is not solely due to the lack of begin/end brackets -- the -lack of declarations and the high-level data types are also responsible -- but -the indentation-based syntax certainly helps. - - -Why am I getting strange results with simple arithmetic operations? -------------------------------------------------------------------- - -See the next question. - - -Why are floating-point calculations so inaccurate? --------------------------------------------------- - -Users are often surprised by results like this:: - - >>> 1.2 - 1.0 - 0.19999999999999996 - -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. - -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. - -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: - - >>> x = 1.2 - -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: - - 1.0011001100110011001100110011001100110011001100110011 (binary) - -which is exactly:: - - 1.1999999999999999555910790149937383830547332763671875 (decimal) - -The typical precision of 53 bits provides Python floats with 15--16 -decimal digits of accuracy. - -For a fuller explanation, please see the :ref:`floating point arithmetic -<tut-fp-issues>` chapter in the Python tutorial. - - -Why are Python strings immutable? ---------------------------------- - -There are several advantages. - -One is performance: knowing that a string is immutable means we can allocate -space for it at creation time, and the storage requirements are fixed and -unchanging. This is also one of the reasons for the distinction between tuples -and lists. - -Another advantage is that strings in Python are considered as "elemental" as -numbers. No amount of activity will change the value 8 to anything else, and in -Python, no amount of activity will change the string "eight" to anything else. - - -.. _why-self: - -Why must 'self' be used explicitly in method definitions and calls? -------------------------------------------------------------------- - -The idea was borrowed from Modula-3. It turns out to be very useful, for a -variety of reasons. - -First, it's more obvious that you are using a method or instance attribute -instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it -absolutely clear that an instance variable or method is used even if you don't -know the class definition by heart. In C++, you can sort of tell by the lack of -a local variable declaration (assuming globals are rare or easily recognizable) --- but in Python, there are no local variable declarations, so you'd have to -look up the class definition to be sure. Some C++ and Java coding standards -call for instance attributes to have an ``m_`` prefix, so this explicitness is -still useful in those languages, too. - -Second, it means that no special syntax is necessary if you want to explicitly -reference or call the method from a particular class. In C++, if you want to -use a method from a base class which is overridden in a derived class, you have -to use the ``::`` operator -- in Python you can write -``baseclass.methodname(self, <argument list>)``. This is particularly useful -for :meth:`__init__` methods, and in general in cases where a derived class -method wants to extend the base class method of the same name and thus has to -call the base class method somehow. - -Finally, for instance variables it solves a syntactic problem with assignment: -since local variables in Python are (by definition!) those variables to which a -value is assigned in a function body (and that aren't explicitly declared -global), there has to be some way to tell the interpreter that an assignment was -meant to assign to an instance variable instead of to a local variable, and it -should preferably be syntactic (for efficiency reasons). C++ does this through -declarations, but Python doesn't have declarations and it would be a pity having -to introduce them just for this purpose. Using the explicit ``self.var`` solves -this nicely. Similarly, for using instance variables, having to write -``self.var`` means that references to unqualified names inside a method don't -have to search the instance's directories. To put it another way, local -variables and instance variables live in two different namespaces, and you need -to tell Python which namespace to use. - - -Why can't I use an assignment in an expression? ------------------------------------------------ - -Many people used to C or Perl complain that they want to use this C idiom: - -.. code-block:: c - - while (line = readline(f)) { - // do something with line - } - -where in Python you're forced to write this:: - - while True: - line = f.readline() - if not line: - break - ... # do something with line - -The reason for not allowing assignment in Python expressions is a common, -hard-to-find bug in those other languages, caused by this construct: - -.. code-block:: c - - if (x = 0) { - // error handling - } - else { - // code that only works for nonzero x - } - -The error is a simple typo: ``x = 0``, which assigns 0 to the variable ``x``, -was written while the comparison ``x == 0`` is certainly what was intended. - -Many alternatives have been proposed. Most are hacks that save some typing but -use arbitrary or cryptic syntax or keywords, and fail the simple criterion for -language change proposals: it should intuitively suggest the proper meaning to a -human reader who has not yet been introduced to the construct. - -An interesting phenomenon is that most experienced Python programmers recognize -the ``while True`` idiom and don't seem to be missing the assignment in -expression construct much; it's only newcomers who express a strong desire to -add this to the language. - -There's an alternative way of spelling this that seems attractive but is -generally less robust than the "while True" solution:: - - line = f.readline() - while line: - ... # do something with line... - line = f.readline() - -The problem with this is that if you change your mind about exactly how you get -the next line (e.g. you want to change it into ``sys.stdin.readline()``) you -have to remember to change two places in your program -- the second occurrence -is hidden at the bottom of the loop. - -The best approach is to use iterators, making it possible to loop through -objects using the ``for`` statement. For example, :term:`file objects -<file object>` support the iterator protocol, so you can write simply:: - - for line in f: - ... # do something with line... - - - -Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ----------------------------------------------------------------------------------------------------------------- - -As Guido said: - - (a) For some operations, prefix notation just reads better than - postfix -- prefix (and infix!) operations have a long tradition in - mathematics which likes notations where the visuals help the - mathematician thinking about a problem. Compare the easy with which we - rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of - doing the same thing using a raw OO notation. - - (b) When I read code that says len(x) I *know* that it is asking for - the length of something. This tells me two things: the result is an - integer, and the argument is some kind of container. To the contrary, - when I read x.len(), I have to already know that x is some kind of - container implementing an interface or inheriting from a class that - has a standard len(). Witness the confusion we occasionally have when - a class that is not implementing a mapping has a get() or keys() - method, or something that isn't a file has a write() method. - - -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html - - -Why is join() a string method instead of a list or tuple method? ----------------------------------------------------------------- - -Strings became much more like other standard types starting in Python 1.6, when -methods were added which give the same functionality that has always been -available using the functions of the string module. Most of these new methods -have been widely accepted, but the one which appears to make some programmers -feel uncomfortable is:: - - ", ".join(['1', '2', '4', '8', '16']) - -which gives the result:: - - "1, 2, 4, 8, 16" - -There are two common arguments against this usage. - -The first runs along the lines of: "It looks really ugly using a method of a -string literal (string constant)", to which the answer is that it might, but a -string literal is just a fixed value. If the methods are to be allowed on names -bound to strings there is no logical reason to make them unavailable on -literals. - -The second objection is typically cast as: "I am really telling a sequence to -join its members together with a string constant". Sadly, you aren't. For some -reason there seems to be much less difficulty with having :meth:`~str.split` as -a string method, since in that case it is easy to see that :: - - "1, 2, 4, 8, 16".split(", ") - -is an instruction to a string literal to return the substrings delimited by the -given separator (or, by default, arbitrary runs of white space). - -:meth:`~str.join` is a string method because in using it you are telling the -separator string to iterate over a sequence of strings and insert itself between -adjacent elements. This method can be used with any argument which obeys the -rules for sequence objects, including any new classes you might define yourself. -Similar methods exist for bytes and bytearray objects. - - -How fast are exceptions? ------------------------- - -A try/except block is extremely efficient if no exceptions are raised. Actually -catching an exception is expensive. In versions of Python prior to 2.0 it was -common to use this idiom:: - - try: - value = mydict[key] - except KeyError: - mydict[key] = getvalue(key) - value = mydict[key] - -This only made sense when you expected the dict to have the key almost all the -time. If that wasn't the case, you coded it like this:: - - if key in mydict: - value = mydict[key] - else: - value = mydict[key] = getvalue(key) - -For this specific case, you could also use ``value = dict.setdefault(key, -getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it -is evaluated in all cases. - - -Why isn't there a switch or case statement in Python? ------------------------------------------------------ - -You can do this easily enough with a sequence of ``if... elif... elif... else``. -There have been some proposals for switch statement syntax, but there is no -consensus (yet) on whether and how to do range tests. See :pep:`275` for -complete details and the current status. - -For cases where you need to choose from a very large number of possibilities, -you can create a dictionary mapping case values to functions to call. For -example:: - - def function_1(...): - ... - - functions = {'a': function_1, - 'b': function_2, - 'c': self.method_1, ...} - - func = functions[value] - func() - -For calling methods on objects, you can simplify yet further by using the -:func:`getattr` built-in to retrieve methods with a particular name:: - - def visit_a(self, ...): - ... - ... - - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() - -It's suggested that you use a prefix for the method names, such as ``visit_`` in -this example. Without such a prefix, if values are coming from an untrusted -source, an attacker would be able to call any method on your object. - - -Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? --------------------------------------------------------------------------------------------------------- - -Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for -each Python stack frame. Also, extensions can call back into Python at almost -random moments. Therefore, a complete threads implementation requires thread -support for C. - -Answer 2: Fortunately, there is `Stackless Python <http://www.stackless.com>`_, -which has a completely redesigned interpreter loop that avoids the C stack. - - -Why can't lambda expressions contain statements? ------------------------------------------------- - -Python lambda expressions cannot contain statements because Python's syntactic -framework can't handle statements nested inside expressions. However, in -Python, this is not a serious problem. Unlike lambda forms in other languages, -where they add functionality, Python lambdas are only a shorthand notation if -you're too lazy to define a function. - -Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a -locally-defined function is that you don't need to invent a name for the -function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! - - -Can Python be compiled to machine code, C or some other language? ------------------------------------------------------------------ - -`Cython <http://cython.org/>`_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka <http://www.nuitka.net/>`_ is -an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC <https://voc.readthedocs.io>`_. - - -How does Python manage memory? ------------------------------- - -The details of Python memory management depend on the implementation. The -standard implementation of Python, :term:`CPython`, uses reference counting to -detect inaccessible objects, and another mechanism to collect reference cycles, -periodically executing a cycle detection algorithm which looks for inaccessible -cycles and deletes the objects involved. The :mod:`gc` module provides functions -to perform a garbage collection, obtain debugging statistics, and tune the -collector's parameters. - -Other implementations (such as `Jython <http://www.jython.org>`_ or -`PyPy <http://www.pypy.org>`_), however, can rely on a different mechanism -such as a full-blown garbage collector. This difference can cause some -subtle porting problems if your Python code depends on the behavior of the -reference counting implementation. - -In some Python implementations, the following code (which is fine in CPython) -will probably run out of file descriptors:: - - for file in very_long_list_of_files: - f = open(file) - c = f.read(1) - -Indeed, using CPython's reference counting and destructor scheme, each new -assignment to *f* closes the previous file. With a traditional GC, however, -those file objects will only get collected (and closed) at varying and possibly -long intervals. - -If you want to write code that will work with any Python implementation, -you should explicitly close the file or use the :keyword:`with` statement; -this will work regardless of memory management scheme:: - - for file in very_long_list_of_files: - with open(file) as f: - c = f.read(1) - - -Why doesn't CPython use a more traditional garbage collection scheme? ---------------------------------------------------------------------- - -For one thing, this is not a C standard feature and hence it's not portable. -(Yes, we know about the Boehm GC library. It has bits of assembler code for -*most* common platforms, not for all of them, and although it is mostly -transparent, it isn't completely transparent; patches are required to get -Python to work with it.) - -Traditional GC also becomes a problem when Python is embedded into other -applications. While in a standalone Python it's fine to replace the standard -malloc() and free() with versions provided by the GC library, an application -embedding Python may want to have its *own* substitute for malloc() and free(), -and may not want Python's. Right now, CPython works with anything that -implements malloc() and free() properly. - - -Why isn't all memory freed when CPython exits? ----------------------------------------------- - -Objects referenced from the global namespaces of Python modules are not always -deallocated when Python exits. This may happen if there are circular -references. There are also certain bits of memory that are allocated by the C -library that are impossible to free (e.g. a tool like Purify will complain about -these). Python is, however, aggressive about cleaning up memory on exit and -does try to destroy every single object. - -If you want to force Python to delete certain things on deallocation use the -:mod:`atexit` module to run a function that will force those deletions. - - -Why are there separate tuple and list data types? -------------------------------------------------- - -Lists and tuples, while similar in many respects, are generally used in -fundamentally different ways. Tuples can be thought of as being similar to -Pascal records or C structs; they're small collections of related data which may -be of different types which are operated on as a group. For example, a -Cartesian coordinate is appropriately represented as a tuple of two or three -numbers. - -Lists, on the other hand, are more like arrays in other languages. They tend to -hold a varying number of objects all of which have the same type and which are -operated on one-by-one. For example, ``os.listdir('.')`` returns a list of -strings representing the files in the current directory. Functions which -operate on this output would generally not break if you added another file or -two to the directory. - -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. - - -How are lists implemented in CPython? -------------------------------------- - -CPython's lists are really variable-length arrays, not Lisp-style linked lists. -The implementation uses a contiguous array of references to other objects, and -keeps a pointer to this array and the array's length in a list head structure. - -This makes indexing a list ``a[i]`` an operation whose cost is independent of -the size of the list or the value of the index. - -When items are appended or inserted, the array of references is resized. Some -cleverness is applied to improve the performance of appending items repeatedly; -when the array must be grown, some extra space is allocated so the next few -times don't require an actual resize. - - -How are dictionaries implemented in CPython? --------------------------------------------- - -CPython's dictionaries are implemented as resizable hash tables. Compared to -B-trees, this gives better performance for lookup (the most common operation by -far) under most circumstances, and the implementation is simpler. - -Dictionaries work by computing a hash code for each key stored in the dictionary -using the :func:`hash` built-in function. The hash code varies widely depending -on the key and a per-process seed; for example, "Python" could hash to --539294296 while "python", a string that differs by a single bit, could hash -to 1142331976. The hash code is then used to calculate a location in an -internal array where the value will be stored. Assuming that you're storing -keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in computer science notation -- to retrieve a key. It -also means that no sorted order of the keys is maintained, and traversing the -array as the ``.keys()`` and ``.items()`` do will output the dictionary's -content in some arbitrary jumbled order that can change with every invocation of -a program. - - -Why must dictionary keys be immutable? --------------------------------------- - -The hash table implementation of dictionaries uses a hash value calculated from -the key value to find the key. If the key were a mutable object, its value -could change, and thus its hash could also change. But since whoever changes -the key object can't tell that it was being used as a dictionary key, it can't -move the entry around in the dictionary. Then, when you try to look up the same -object in the dictionary it won't be found because its hash value is different. -If you tried to look up the old value it wouldn't be found either, because the -value of the object found in that hash bin would be different. - -If you want a dictionary indexed with a list, simply convert the list to a tuple -first; the function ``tuple(L)`` creates a tuple with the same entries as the -list ``L``. Tuples are immutable and can therefore be used as dictionary keys. - -Some unacceptable solutions that have been proposed: - -- Hash lists by their address (object ID). This doesn't work because if you - construct a new list with the same value it won't be found; e.g.:: - - mydict = {[1, 2]: '12'} - print(mydict[[1, 2]]) - - would raise a KeyError exception because the id of the ``[1, 2]`` used in the - second line differs from that in the first line. In other words, dictionary - keys should be compared using ``==``, not using :keyword:`is`. - -- Make a copy when using a list as a key. This doesn't work because the list, - being a mutable object, could contain a reference to itself, and then the - copying code would run into an infinite loop. - -- Allow lists as keys but tell the user not to modify them. This would allow a - class of hard-to-track bugs in programs when you forgot or modified a list by - accident. It also invalidates an important invariant of dictionaries: every - value in ``d.keys()`` is usable as a key of the dictionary. - -- Mark lists as read-only once they are used as a dictionary key. The problem - is that it's not just the top-level object that could change its value; you - could use a tuple containing a list as a key. Entering anything as a key into - a dictionary would require marking all objects reachable from there as - read-only -- and again, self-referential objects could cause an infinite loop. - -There is a trick to get around this if you need to, but use it at your own risk: -You can wrap a mutable structure inside a class instance which has both a -:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the -hash value for all such wrapper objects that reside in a dictionary (or other -hash based structure), remain fixed while the object is in the dictionary (or -other structure). :: - - class ListWrapper: - def __init__(self, the_list): - self.the_list = the_list - - def __eq__(self, other): - return self.the_list == other.the_list - - def __hash__(self): - l = self.the_list - result = 98767 - len(l)*555 - for i, el in enumerate(l): - try: - result = result + (hash(el) % 9999999) * 1001 + i - except Exception: - result = (result % 7777777) + i * 333 - return result - -Note that the hash computation is complicated by the possibility that some -members of the list may be unhashable and also by the possibility of arithmetic -overflow. - -Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) -is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), -regardless of whether the object is in a dictionary or not. If you fail to meet -these restrictions dictionaries and other hash based structures will misbehave. - -In the case of ListWrapper, whenever the wrapper object is in a dictionary the -wrapped list must not change to avoid anomalies. Don't do this unless you are -prepared to think hard about the requirements and the consequences of not -meeting them correctly. Consider yourself warned. - - -Why doesn't list.sort() return the sorted list? ------------------------------------------------ - -In situations where performance matters, making a copy of the list just to sort -it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In -order to remind you of that fact, it does not return the sorted list. This way, -you won't be fooled into accidentally overwriting a list when you need a sorted -copy but also need to keep the unsorted version around. - -If you want to return a new list, use the built-in :func:`sorted` function -instead. This function creates a new list from a provided iterable, sorts -it and returns it. For example, here's how to iterate over the keys of a -dictionary in sorted order:: - - for key in sorted(mydict): - ... # do whatever with mydict[key]... - - -How do you specify and enforce an interface spec in Python? ------------------------------------------------------------ - -An interface specification for a module as provided by languages such as C++ and -Java describes the prototypes for the methods and functions of the module. Many -feel that compile-time enforcement of interface specifications helps in the -construction of large programs. - -Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes -(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check -whether an instance or a class implements a particular ABC. The -:mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. - -For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. There is also a tool, -PyChecker, which can be used to find problems due to subclassing. - -A good test suite for a module can both provide a regression test and serve as a -module interface specification and a set of examples. Many Python modules can -be run as a script to provide a simple "self test." Even modules which use -complex external interfaces can often be tested in isolation using trivial -"stub" emulations of the external interface. The :mod:`doctest` and -:mod:`unittest` modules or third-party test frameworks can be used to construct -exhaustive test suites that exercise every line of code in a module. - -An appropriate testing discipline can help build large complex applications in -Python as well as having interface specifications would. In fact, it can be -better because an interface specification cannot test certain properties of a -program. For example, the :meth:`append` method is expected to add new elements -to the end of some internal list; an interface specification cannot test that -your :meth:`append` implementation will actually do this correctly, but it's -trivial to check this property in a test suite. - -Writing test suites is very helpful, and you might want to design your code with -an eye to making it easily tested. One increasingly popular technique, -test-directed development, calls for writing parts of the test suite first, -before you write any of the actual code. Of course Python allows you to be -sloppy and not write test cases at all. - - -Why is there no goto? ---------------------- - -You can use exceptions to provide a "structured goto" that even works across -function calls. Many feel that exceptions can conveniently emulate all -reasonable uses of the "go" or "goto" constructs of C, Fortran, and other -languages. For example:: - - class label(Exception): pass # declare a label - - try: - ... - if condition: raise label() # goto label - ... - except label: # where to goto - pass - ... - -This doesn't allow you to jump into the middle of a loop, but that's usually -considered an abuse of goto anyway. Use sparingly. - - -Why can't raw strings (r-strings) end with a backslash? -------------------------------------------------------- - -More precisely, they can't end with an odd number of backslashes: the unpaired -backslash at the end escapes the closing quote character, leaving an -unterminated string. - -Raw strings were designed to ease creating input for processors (chiefly regular -expression engines) that want to do their own backslash escape processing. Such -processors consider an unmatched trailing backslash to be an error anyway, so -raw strings disallow that. In return, they allow you to pass on the string -quote character by escaping it with a backslash. These rules work well when -r-strings are used for their intended purpose. - -If you're trying to build Windows pathnames, note that all Windows system calls -accept forward slashes too:: - - f = open("/mydir/file.txt") # works fine! - -If you're trying to build a pathname for a DOS command, try e.g. one of :: - - dir = r"\this\is\my\dos\dir" "\\" - dir = r"\this\is\my\dos\dir\ "[:-1] - dir = "\\this\\is\\my\\dos\\dir\\" - - -Why doesn't Python have a "with" statement for attribute assignments? ---------------------------------------------------------------------- - -Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some language have a construct that -looks like this:: - - with obj: - a = 1 # equivalent to obj.a = 1 - total = total + 1 # obj.total = obj.total + 1 - -In Python, such a construct would be ambiguous. - -Other languages, such as Object Pascal, Delphi, and C++, use static types, so -it's possible to know, in an unambiguous way, what member is being assigned -to. This is the main point of static typing -- the compiler *always* knows the -scope of every variable at compile time. - -Python uses dynamic types. It is impossible to know in advance which attribute -will be referenced at runtime. Member attributes may be added or removed from -objects on the fly. This makes it impossible to know, from a simple reading, -what attribute is being referenced: a local one, a global one, or a member -attribute? - -For instance, take the following incomplete snippet:: - - def foo(a): - with a: - print(x) - -The snippet assumes that "a" must have a member attribute called "x". However, -there is nothing in Python that tells the interpreter this. What should happen -if "a" is, let us say, an integer? If there is a global variable named "x", -will it be used inside the with block? As you see, the dynamic nature of Python -makes such choices much harder. - -The primary benefit of "with" and similar language features (reduction of code -volume) can, however, easily be achieved in Python by assignment. Instead of:: - - function(args).mydict[index][index].a = 21 - function(args).mydict[index][index].b = 42 - function(args).mydict[index][index].c = 63 - -write this:: - - ref = function(args).mydict[index][index] - ref.a = 21 - ref.b = 42 - ref.c = 63 - -This also has the side-effect of increasing execution speed because name -bindings are resolved at run-time in Python, and the second version only needs -to perform the resolution once. - - -Why are colons required for the if/while/def/class statements? --------------------------------------------------------------- - -The colon is required primarily to enhance readability (one of the results of -the experimental ABC language). Consider this:: - - if a == b - print(a) - -versus :: - - if a == b: - print(a) - -Notice how the second one is slightly easier to read. Notice further how a -colon sets off the example in this FAQ answer; it's a standard usage in English. - -Another minor reason is that the colon makes it easier for editors with syntax -highlighting; they can look for colons to decide when indentation needs to be -increased instead of having to do a more elaborate parsing of the program text. - - -Why does Python allow commas at the end of lists and tuples? ------------------------------------------------------------- - -Python lets you add a trailing comma at the end of lists, tuples, and -dictionaries:: - - [1, 2, 3,] - ('a', 'b', 'c',) - d = { - "A": [1, 5], - "B": [6, 7], # last trailing comma is optional but good style - } - - -There are several reasons to allow this. - -When you have a literal value for a list, tuple, or dictionary spread across -multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. - -Accidentally omitting the comma can lead to errors that are hard to diagnose. -For example:: - - x = [ - "fee", - "fie" - "foo", - "fum" - ] - -This list looks like it has four elements, but it actually contains three: -"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. - -Allowing the trailing comma may also make programmatic code generation easier. diff --git a/third_party/python/Doc/faq/extending.rst b/third_party/python/Doc/faq/extending.rst deleted file mode 100644 index a0828172d..000000000 --- a/third_party/python/Doc/faq/extending.rst +++ /dev/null @@ -1,445 +0,0 @@ -======================= -Extending/Embedding FAQ -======================= - -.. only:: html - - .. contents:: - -.. highlight:: c - - -.. XXX need review for Python 3. - - -Can I create my own functions in C? ------------------------------------ - -Yes, you can create built-in modules containing functions, variables, exceptions -and even new types in C. This is explained in the document -:ref:`extending-index`. - -Most intermediate or advanced Python books will also cover this topic. - - -Can I create my own functions in C++? -------------------------------------- - -Yes, using the C compatibility features found in C++. Place ``extern "C" { -... }`` around the Python include files and put ``extern "C"`` before each -function that is going to be called by the Python interpreter. Global or static -C++ objects with constructors are probably not a good idea. - - -.. _c-wrapper-software: - -Writing C is hard; are there any alternatives? ----------------------------------------------- - -There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython <http://cython.org>`_ and its relative `Pyrex -<https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG <http://www.swig.org>`_. `SIP -<https://riverbankcomputing.com/software/sip/intro>`__, `CXX -<http://cxx.sourceforge.net/>`_ `Boost -<http://www.boost.org/libs/python/doc/index.html>`_, or `Weave -<https://github.com/scipy/weave>`_ are also -alternatives for wrapping C++ libraries. - - -How can I execute arbitrary Python statements from C? ------------------------------------------------------ - -The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes -a single string argument to be executed in the context of the module -``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred -(including ``SyntaxError``). If you want more control, use -:c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in -``Python/pythonrun.c``. - - -How can I evaluate an arbitrary Python expression from C? ---------------------------------------------------------- - -Call the function :c:func:`PyRun_String` from the previous question with the -start symbol :c:data:`Py_eval_input`; it parses an expression, evaluates it and -returns its value. - - -How do I extract C values from a Python object? ------------------------------------------------ - -That depends on the object's type. If it's a tuple, :c:func:`PyTuple_Size` -returns its length and :c:func:`PyTuple_GetItem` returns the item at a specified -index. Lists have similar functions, :c:func:`PyListSize` and -:c:func:`PyList_GetItem`. - -For bytes, :c:func:`PyBytes_Size` returns its length and -:c:func:`PyBytes_AsStringAndSize` provides a pointer to its value and its -length. Note that Python bytes objects may contain null bytes so C's -:c:func:`strlen` should not be used. - -To test the type of an object, first make sure it isn't *NULL*, and then use -:c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. - -There is also a high-level API to Python objects which is provided by the -so-called 'abstract' interface -- read ``Include/abstract.h`` for further -details. It allows interfacing with any kind of Python sequence using calls -like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc. as well -as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et -al.) and mappings in the PyMapping APIs. - - -How do I use Py_BuildValue() to create a tuple of arbitrary length? -------------------------------------------------------------------- - -You can't. Use :c:func:`PyTuple_Pack` instead. - - -How do I call an object's method from C? ----------------------------------------- - -The :c:func:`PyObject_CallMethod` function can be used to call an arbitrary -method of an object. The parameters are the object, the name of the method to -call, a format string like that used with :c:func:`Py_BuildValue`, and the -argument values:: - - PyObject * - PyObject_CallMethod(PyObject *object, const char *method_name, - const char *arg_format, ...); - -This works for any object that has methods -- whether built-in or user-defined. -You are responsible for eventually :c:func:`Py_DECREF`\ 'ing the return value. - -To call, e.g., a file object's "seek" method with arguments 10, 0 (assuming the -file object pointer is "f"):: - - res = PyObject_CallMethod(f, "seek", "(ii)", 10, 0); - if (res == NULL) { - ... an exception occurred ... - } - else { - Py_DECREF(res); - } - -Note that since :c:func:`PyObject_CallObject` *always* wants a tuple for the -argument list, to call a function without arguments, pass "()" for the format, -and to call a function with one argument, surround the argument in parentheses, -e.g. "(i)". - - -How do I catch the output from PyErr_Print() (or anything that prints to stdout/stderr)? ----------------------------------------------------------------------------------------- - -In Python code, define an object that supports the ``write()`` method. Assign -this object to :data:`sys.stdout` and :data:`sys.stderr`. Call print_error, or -just allow the standard traceback mechanism to work. Then, the output will go -wherever your ``write()`` method sends it. - -The easiest way to do this is to use the :class:`io.StringIO` class: - -.. code-block:: pycon - - >>> import io, sys - >>> sys.stdout = io.StringIO() - >>> print('foo') - >>> print('hello world!') - >>> sys.stderr.write(sys.stdout.getvalue()) - foo - hello world! - -A custom object to do the same would look like this: - -.. code-block:: pycon - - >>> import io, sys - >>> class StdoutCatcher(io.TextIOBase): - ... def __init__(self): - ... self.data = [] - ... def write(self, stuff): - ... self.data.append(stuff) - ... - >>> import sys - >>> sys.stdout = StdoutCatcher() - >>> print('foo') - >>> print('hello world!') - >>> sys.stderr.write(''.join(sys.stdout.data)) - foo - hello world! - - -How do I access a module written in Python from C? --------------------------------------------------- - -You can get a pointer to the module object as follows:: - - module = PyImport_ImportModule("<modulename>"); - -If the module hasn't been imported yet (i.e. it is not yet present in -:data:`sys.modules`), this initializes the module; otherwise it simply returns -the value of ``sys.modules["<modulename>"]``. Note that it doesn't enter the -module into any namespace -- it only ensures it has been initialized and is -stored in :data:`sys.modules`. - -You can then access the module's attributes (i.e. any name defined in the -module) as follows:: - - attr = PyObject_GetAttrString(module, "<attrname>"); - -Calling :c:func:`PyObject_SetAttrString` to assign to variables in the module -also works. - - -How do I interface to C++ objects from Python? ----------------------------------------------- - -Depending on your requirements, there are many approaches. To do this manually, -begin by reading :ref:`the "Extending and Embedding" document -<extending-index>`. Realize that for the Python run-time system, there isn't a -whole lot of difference between C and C++ -- so the strategy of building a new -Python type around a C structure (pointer) type will also work for C++ objects. - -For C++ libraries, see :ref:`c-wrapper-software`. - - -I added a module using the Setup file and the make fails; why? --------------------------------------------------------------- - -Setup must end in a newline, if there is no newline there, the build process -fails. (Fixing this requires some ugly shell script hackery, and this bug is so -minor that it doesn't seem worth the effort.) - - -How do I debug an extension? ----------------------------- - -When using GDB with dynamically loaded extensions, you can't set a breakpoint in -your extension until your extension is loaded. - -In your ``.gdbinit`` file (or interactively), add the command: - -.. code-block:: none - - br _PyImport_LoadDynamicModule - -Then, when you run GDB: - -.. code-block:: shell-session - - $ gdb /local/bin/python - gdb) run myscript.py - gdb) continue # repeat until your extension is loaded - gdb) finish # so that your extension is loaded - gdb) br myfunction.c:50 - gdb) continue - -I want to compile a Python module on my Linux system, but some files are missing. Why? --------------------------------------------------------------------------------------- - -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files -required for compiling Python extensions. - -For Red Hat, install the python-devel RPM to get the necessary files. - -For Debian, run ``apt-get install python-dev``. - - -How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------- - -Sometimes you want to emulate the Python interactive interpreter's behavior, -where it gives you a continuation prompt when the input is incomplete (e.g. you -typed the start of an "if" statement or you didn't close your parentheses or -triple string quotes), but it gives you a syntax error message immediately when -the input is invalid. - -In Python you can use the :mod:`codeop` module, which approximates the parser's -behavior sufficiently. IDLE uses this, for example. - -The easiest way to do it in C is to call :c:func:`PyRun_InteractiveLoop` (perhaps -in a separate thread) and let the Python interpreter handle the input for -you. You can also set the :c:func:`PyOS_ReadlineFunctionPointer` to point at your -custom input function. See ``Modules/readline.c`` and ``Parser/myreadline.c`` -for more hints. - -However sometimes you have to run the embedded Python interpreter in the same -thread as your rest application and you can't allow the -:c:func:`PyRun_InteractiveLoop` to stop while waiting for user input. The one -solution then is to call :c:func:`PyParser_ParseString` and test for ``e.error`` -equal to ``E_EOF``, which means the input is incomplete. Here's a sample code -fragment, untested, inspired by code from Alex Farber:: - - #include <Python.h> - #include <node.h> - #include <errcode.h> - #include <grammar.h> - #include <parsetok.h> - #include <compile.h> - - int testcomplete(char *code) - /* code should end in \n */ - /* return -1 for error, 0 for incomplete, 1 for complete */ - { - node *n; - perrdetail e; - - n = PyParser_ParseString(code, &_PyParser_Grammar, - Py_file_input, &e); - if (n == NULL) { - if (e.error == E_EOF) - return 0; - return -1; - } - - PyNode_Free(n); - return 1; - } - -Another solution is trying to compile the received string with -:c:func:`Py_CompileString`. If it compiles without errors, try to execute the -returned code object by calling :c:func:`PyEval_EvalCode`. Otherwise save the -input for later. If the compilation fails, find out if it's an error or just -more input is required - by extracting the message string from the exception -tuple and comparing it to the string "unexpected EOF while parsing". Here is a -complete example using the GNU readline library (you may want to ignore -**SIGINT** while calling readline()):: - - #include <stdio.h> - #include <readline.h> - - #include <Python.h> - #include <object.h> - #include <compile.h> - #include <eval.h> - - int main (int argc, char* argv[]) - { - int i, j, done = 0; /* lengths of line, code */ - char ps1[] = ">>> "; - char ps2[] = "... "; - char *prompt = ps1; - char *msg, *line, *code = NULL; - PyObject *src, *glb, *loc; - PyObject *exc, *val, *trb, *obj, *dum; - - Py_Initialize (); - loc = PyDict_New (); - glb = PyDict_New (); - PyDict_SetItemString (glb, "__builtins__", PyEval_GetBuiltins ()); - - while (!done) - { - line = readline (prompt); - - if (NULL == line) /* Ctrl-D pressed */ - { - done = 1; - } - else - { - i = strlen (line); - - if (i > 0) - add_history (line); /* save non-empty lines */ - - if (NULL == code) /* nothing in code yet */ - j = 0; - else - j = strlen (code); - - code = realloc (code, i + j + 2); - if (NULL == code) /* out of memory */ - exit (1); - - if (0 == j) /* code was empty, so */ - code[0] = '\0'; /* keep strncat happy */ - - strncat (code, line, i); /* append line to code */ - code[i + j] = '\n'; /* append '\n' to code */ - code[i + j + 1] = '\0'; - - src = Py_CompileString (code, "<stdin>", Py_single_input); - - if (NULL != src) /* compiled just fine - */ - { - if (ps1 == prompt || /* ">>> " or */ - '\n' == code[i + j - 1]) /* "... " and double '\n' */ - { /* so execute it */ - dum = PyEval_EvalCode (src, glb, loc); - Py_XDECREF (dum); - Py_XDECREF (src); - free (code); - code = NULL; - if (PyErr_Occurred ()) - PyErr_Print (); - prompt = ps1; - } - } /* syntax error or E_EOF? */ - else if (PyErr_ExceptionMatches (PyExc_SyntaxError)) - { - PyErr_Fetch (&exc, &val, &trb); /* clears exception! */ - - if (PyArg_ParseTuple (val, "sO", &msg, &obj) && - !strcmp (msg, "unexpected EOF while parsing")) /* E_EOF */ - { - Py_XDECREF (exc); - Py_XDECREF (val); - Py_XDECREF (trb); - prompt = ps2; - } - else /* some other syntax error */ - { - PyErr_Restore (exc, val, trb); - PyErr_Print (); - free (code); - code = NULL; - prompt = ps1; - } - } - else /* some non-syntax error */ - { - PyErr_Print (); - free (code); - code = NULL; - prompt = ps1; - } - - free (line); - } - } - - Py_XDECREF(glb); - Py_XDECREF(loc); - Py_Finalize(); - exit(0); - } - - -How do I find undefined g++ symbols __builtin_new or __pure_virtual? --------------------------------------------------------------------- - -To dynamically load g++ extension modules, you must recompile Python, relink it -using g++ (change LINKCC in the Python Modules Makefile), and link your -extension module using g++ (e.g., ``g++ -shared -o mymodule.so mymodule.o``). - - -Can I create an object class with some methods implemented in C and others in Python (e.g. through inheritance)? ----------------------------------------------------------------------------------------------------------------- - -Yes, you can inherit from built-in classes such as :class:`int`, :class:`list`, -:class:`dict`, etc. - -The Boost Python Library (BPL, http://www.boost.org/libs/python/doc/index.html) -provides a way of doing this from C++ (i.e. you can inherit from an extension -class written in C++ using the BPL). diff --git a/third_party/python/Doc/faq/general.rst b/third_party/python/Doc/faq/general.rst deleted file mode 100644 index 9ec90e802..000000000 --- a/third_party/python/Doc/faq/general.rst +++ /dev/null @@ -1,456 +0,0 @@ -:tocdepth: 2 - -================== -General Python FAQ -================== - -.. only:: html - - .. contents:: - - -General Information -=================== - -What is Python? ---------------- - -Python is an interpreted, interactive, object-oriented programming language. It -incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. Python combines remarkable power with very clear syntax. -It has interfaces to many system calls and libraries, as well as to various -window systems, and is extensible in C or C++. It is also usable as an -extension language for applications that need a programmable interface. -Finally, Python is portable: it runs on many Unix variants, on the Mac, and on -Windows 2000 and later. - -To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_ links to other -introductory tutorials and resources for learning Python. - - -What is the Python Software Foundation? ---------------------------------------- - -The Python Software Foundation is an independent non-profit organization that -holds the copyright on Python versions 2.1 and newer. The PSF's mission is to -advance open source technology related to the Python programming language and to -publicize the use of Python. The PSF's home page is at -https://www.python.org/psf/. - -Donations to the PSF are tax-exempt in the US. If you use Python and find it -helpful, please contribute via `the PSF donation page -<https://www.python.org/psf/donations/>`_. - - -Are there copyright restrictions on the use of Python? ------------------------------------------------------- - -You can do anything you want with the source, as long as you leave the -copyrights in and display those copyrights in any documentation about Python -that you produce. If you honor the copyright rules, it's OK to use Python for -commercial use, to sell copies of Python in source or binary form (modified or -unmodified), or to sell products that incorporate Python in some form. We would -still like to know about all commercial use of Python, of course. - -See `the PSF license page <https://www.python.org/psf/license/>`_ to find further -explanations and a link to the full text of the license. - -The Python logo is trademarked, and in certain cases permission is required to -use it. Consult `the Trademark Usage Policy -<https://www.python.org/psf/trademarks/>`__ for more information. - - -Why was Python created in the first place? ------------------------------------------- - -Here's a *very* brief summary of what started it all, written by Guido van -Rossum: - - I had extensive experience with implementing an interpreted language in the - ABC group at CWI, and from working with this group I had learned a lot about - language design. This is the origin of many Python features, including the - use of indentation for statement grouping and the inclusion of - very-high-level data types (although the details are all different in - Python). - - I had a number of gripes about the ABC language, but also liked many of its - features. It was impossible to extend the ABC language (or its - implementation) to remedy my complaints -- in fact its lack of extensibility - was one of its biggest problems. I had some experience with using Modula-2+ - and talked with the designers of Modula-3 and read the Modula-3 report. - Modula-3 is the origin of the syntax and semantics used for exceptions, and - some other Python features. - - I was working in the Amoeba distributed operating system group at CWI. We - needed a better way to do system administration than by writing either C - programs or Bourne shell scripts, since Amoeba had its own system call - interface which wasn't easily accessible from the Bourne shell. My - experience with error handling in Amoeba made me acutely aware of the - importance of exceptions as a programming language feature. - - It occurred to me that a scripting language with a syntax like ABC but with - access to the Amoeba system calls would fill the need. I realized that it - would be foolish to write an Amoeba-specific language, so I decided that I - needed a language that was generally extensible. - - During the 1989 Christmas holidays, I had a lot of time on my hand, so I - decided to give it a try. During the next year, while still mostly working - on it in my own time, Python was used in the Amoeba project with increasing - success, and the feedback from colleagues made me add many early - improvements. - - In February 1991, after just over a year of development, I decided to post to - USENET. The rest is in the ``Misc/HISTORY`` file. - - -What is Python good for? ------------------------- - -Python is a high-level general-purpose programming language that can be applied -to many different classes of problems. - -The language comes with a large standard library that covers areas such as -string processing (regular expressions, Unicode, calculating differences between -files), Internet protocols (HTTP, FTP, SMTP, XML-RPC, POP, IMAP, CGI -programming), software engineering (unit testing, logging, profiling, parsing -Python code), and operating system interfaces (system calls, filesystems, TCP/IP -sockets). Look at the table of contents for :ref:`library-index` to get an idea -of what's available. A wide variety of third-party extensions are also -available. Consult `the Python Package Index <https://pypi.org>`_ to -find packages of interest to you. - - -How does the Python version numbering scheme work? --------------------------------------------------- - -Python versions are numbered A.B.C or A.B. A is the major version number -- it -is only incremented for really major changes in the language. B is the minor -version number, incremented for less earth-shattering changes. C is the -micro-level -- it is incremented for each bugfix release. See :pep:`6` for more -information about bugfix releases. - -Not all releases are bugfix releases. In the run-up to a new major release, a -series of development releases are made, denoted as alpha, beta, or release -candidate. Alphas are early releases in which interfaces aren't yet finalized; -it's not unexpected to see an interface change between two alpha releases. -Betas are more stable, preserving existing interfaces but possibly adding new -modules, and release candidates are frozen, making no changes except as needed -to fix critical bugs. - -Alpha, beta and release candidate versions have an additional suffix. The -suffix for an alpha version is "aN" for some small number N, the suffix for a -beta version is "bN" for some small number N, and the suffix for a release -candidate version is "cN" for some small number N. In other words, all versions -labeled 2.0aN precede the versions labeled 2.0bN, which precede versions labeled -2.0cN, and *those* precede 2.0. - -You may also find version numbers with a "+" suffix, e.g. "2.2+". These are -unreleased versions, built directly from the CPython development repository. In -practice, after a final minor release is made, the version is incremented to the -next minor version, which becomes the "a0" version, e.g. "2.4a0". - -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and -:data:`sys.version_info`. - - -How do I obtain a copy of the Python source? --------------------------------------------- - -The latest Python source distribution is always available from python.org, at -https://www.python.org/downloads/. The latest development sources can be obtained -at https://github.com/python/cpython/. - -The source distribution is a gzipped tar file containing the complete C source, -Sphinx-formatted documentation, Python library modules, example programs, and -several useful pieces of freely distributable software. The source will compile -and run out of the box on most UNIX platforms. - -Consult the `Getting Started section of the Python Developer's Guide -<https://devguide.python.org/setup/>`__ for more -information on getting the source code and compiling it. - - -How do I get documentation on Python? -------------------------------------- - -.. XXX mention py3k - -The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are -also available at https://docs.python.org/3/download.html. - -The documentation is written in reStructuredText and processed by `the Sphinx -documentation tool <http://sphinx-doc.org/>`__. The reStructuredText source for -the documentation is part of the Python source distribution. - - -I've never programmed before. Is there a Python tutorial? ---------------------------------------------------------- - -There are numerous tutorials and books available. The standard documentation -includes :ref:`tutorial-index`. - -Consult `the Beginner's Guide <https://wiki.python.org/moin/BeginnersGuide>`_ to -find information for beginning Python programmers, including lists of tutorials. - - -Is there a newsgroup or mailing list devoted to Python? -------------------------------------------------------- - -There is a newsgroup, :newsgroup:`comp.lang.python`, and a mailing list, -`python-list <https://mail.python.org/mailman/listinfo/python-list>`_. The -newsgroup and mailing list are gatewayed into each other -- if you can read news -it's unnecessary to subscribe to the mailing list. -:newsgroup:`comp.lang.python` is high-traffic, receiving hundreds of postings -every day, and Usenet readers are often more able to cope with this volume. - -Announcements of new software releases and events can be found in -comp.lang.python.announce, a low-traffic moderated list that receives about five -postings per day. It's available as `the python-announce mailing list -<https://mail.python.org/mailman/listinfo/python-announce-list>`_. - -More info about other mailing lists and newsgroups -can be found at https://www.python.org/community/lists/. - - -How do I get a beta test version of Python? -------------------------------------------- - -Alpha and beta releases are available from https://www.python.org/downloads/. All -releases are announced on the comp.lang.python and comp.lang.python.announce -newsgroups and on the Python home page at https://www.python.org/; an RSS feed of -news is available. - -You can also access the development version of Python through Git. See -`The Python Developer's Guide <https://devguide.python.org/>`_ for details. - - -How do I submit bug reports and patches for Python? ---------------------------------------------------- - -To report a bug or submit a patch, please use the Roundup installation at -https://bugs.python.org/. - -You must have a Roundup account to report bugs; this makes it possible for us to -contact you if we have follow-up questions. It will also enable Roundup to send -you updates as we act on your bug. If you had previously used SourceForge to -report bugs to Python, you can obtain your Roundup password through Roundup's -`password reset procedure <https://bugs.python.org/user?@template=forgotten>`_. - -For more information on how Python is developed, consult `the Python Developer's -Guide <https://devguide.python.org/>`_. - - -Are there any published articles about Python that I can reference? -------------------------------------------------------------------- - -It's probably best to cite your favorite book about Python. - -The very first article about Python was written in 1991 and is now quite -outdated. - - Guido van Rossum and Jelke de Boer, "Interactively Testing Remote Servers - Using the Python Programming Language", CWI Quarterly, Volume 4, Issue 4 - (December 1991), Amsterdam, pp 283--303. - - -Are there any books on Python? ------------------------------- - -Yes, there are many, and more are being published. See the python.org wiki at -https://wiki.python.org/moin/PythonBooks for a list. - -You can also search online bookstores for "Python" and filter out the Monty -Python references; or perhaps search for "Python" and "language". - - -Where in the world is www.python.org located? ---------------------------------------------- - -The Python project's infrastructure is located all over the world. -`www.python.org <https://www.python.org>`_ is graciously hosted by `Rackspace -<https://www.rackspace.com>`_, with CDN caching provided by `Fastly -<https://www.fastly.com>`_. `Upfront Systems -<http://www.upfrontsystems.co.za/>`_ hosts `bugs.python.org -<https://bugs.python.org>`_. Many other Python services like `the Wiki -<https://wiki.python.org>`_ are hosted by `Oregon State -University Open Source Lab <https://osuosl.org>`_. - - -Why is it called Python? ------------------------- - -When he began implementing Python, Guido van Rossum was also reading the -published scripts from `"Monty Python's Flying Circus" -<https://en.wikipedia.org/wiki/Monty_Python>`__, a BBC comedy series from the 1970s. Van Rossum -thought he needed a name that was short, unique, and slightly mysterious, so he -decided to call the language Python. - - -Do I have to like "Monty Python's Flying Circus"? -------------------------------------------------- - -No, but it helps. :) - - -Python in the real world -======================== - -How stable is Python? ---------------------- - -Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. Currently there are -usually around 18 months between major releases. - -The developers issue "bugfix" releases of older versions, so the stability of -existing releases gradually improves. Bugfix releases, indicated by a third -component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability; -only fixes for known problems are included in a bugfix release, and it's -guaranteed that interfaces will remain the same throughout a series of bugfix -releases. - -The latest stable releases can always be found on the `Python download page -<https://www.python.org/downloads/>`_. There are two production-ready version -of Python: 2.x and 3.x, but the recommended one at this times is Python 3.x. -Although Python 2.x is still widely used, `it will not be -maintained after January 1, 2020 <https://www.python.org/dev/peps/pep-0373/>`_. -Python 2.x was known for having more third-party libraries available, however, -by the time of this writing, most of the widely used libraries support Python 3.x, -and some are even dropping the Python 2.x support. - - -How many people are using Python? ---------------------------------- - -There are probably tens of thousands of users, though it's difficult to obtain -an exact count. - -Python is available for free download, so there are no sales figures, and it's -available from many different sites and packaged with many Linux distributions, -so download statistics don't tell the whole story either. - -The comp.lang.python newsgroup is very active, but not all Python users post to -the group or even read it. - - -Have any significant projects been done in Python? --------------------------------------------------- - -See https://www.python.org/about/success for a list of projects that use Python. -Consulting the proceedings for `past Python conferences -<https://www.python.org/community/workshops/>`_ will reveal contributions from many -different companies and organizations. - -High-profile Python projects include `the Mailman mailing list manager -<http://www.list.org>`_ and `the Zope application server -<http://www.zope.org>`_. Several Linux distributions, most notably `Red Hat -<https://www.redhat.com>`_, have written part or all of their installer and -system administration software in Python. Companies that use Python internally -include Google, Yahoo, and Lucasfilm Ltd. - - -What new developments are expected for Python in the future? ------------------------------------------------------------- - -See https://www.python.org/dev/peps/ for the Python Enhancement Proposals -(PEPs). PEPs are design documents describing a suggested new feature for Python, -providing a concise technical specification and a rationale. Look for a PEP -titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been -publicly released yet. - -New development is discussed on `the python-dev mailing list -<https://mail.python.org/mailman/listinfo/python-dev/>`_. - - -Is it reasonable to propose incompatible changes to Python? ------------------------------------------------------------ - -In general, no. There are already millions of lines of Python code around the -world, so any change in the language that invalidates more than a very small -fraction of existing programs has to be frowned upon. Even if you can provide a -conversion program, there's still the problem of updating all documentation; -many books have been written about Python, and we don't want to invalidate them -all at a single stroke. - -Providing a gradual upgrade path is necessary if a feature has to be changed. -:pep:`5` describes the procedure followed for introducing backward-incompatible -changes while minimizing disruption for users. - - -Is Python a good language for beginning programmers? ----------------------------------------------------- - -Yes. - -It is still common to start students with a procedural and statically typed -language such as Pascal, C, or a subset of C++ or Java. Students may be better -served by learning Python as their first language. Python has a very simple and -consistent syntax and a large standard library and, most importantly, using -Python in a beginning programming course lets students concentrate on important -programming skills such as problem decomposition and data type design. With -Python, students can be quickly introduced to basic concepts such as loops and -procedures. They can probably even work with user-defined objects in their very -first course. - -For a student who has never programmed before, using a statically typed language -seems unnatural. It presents additional complexity that the student must master -and slows the pace of the course. The students are trying to learn to think -like a computer, decompose problems, design consistent interfaces, and -encapsulate data. While learning to use a statically typed language is -important in the long term, it is not necessarily the best topic to address in -the students' first programming course. - -Many other aspects of Python make it a good first language. Like Java, Python -has a large standard library so that students can be assigned programming -projects very early in the course that *do* something. Assignments aren't -restricted to the standard four-function calculator and check balancing -programs. By using the standard library, students can gain the satisfaction of -working on realistic applications as they learn the fundamentals of programming. -Using the standard library also teaches students about code reuse. Third-party -modules such as PyGame are also helpful in extending the students' reach. - -Python's interactive interpreter enables students to test language features -while they're programming. They can keep a window with the interpreter running -while they enter their program's source in another window. If they can't -remember the methods for a list, they can do something like this:: - - >>> L = [] - >>> dir(L) # doctest: +NORMALIZE_WHITESPACE - ['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', - '__dir__', '__doc__', '__eq__', '__format__', '__ge__', - '__getattribute__', '__getitem__', '__gt__', '__hash__', '__iadd__', - '__imul__', '__init__', '__iter__', '__le__', '__len__', '__lt__', - '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', - '__repr__', '__reversed__', '__rmul__', '__setattr__', '__setitem__', - '__sizeof__', '__str__', '__subclasshook__', 'append', 'clear', - 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort'] - >>> [d for d in dir(L) if '__' not in d] - ['append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort'] - - >>> help(L.append) - Help on built-in function append: - <BLANKLINE> - append(...) - L.append(object) -> None -- append object to end - <BLANKLINE> - >>> L.append(1) - >>> L - [1] - -With the interpreter, documentation is never far from the student as they are -programming. - -There are also good IDEs for Python. IDLE is a cross-platform IDE for Python -that is written in Python using Tkinter. PythonWin is a Windows-specific IDE. -Emacs users will be happy to know that there is a very good Python mode for -Emacs. All of these programming environments provide syntax highlighting, -auto-indenting, and access to the interactive interpreter while coding. Consult -`the Python wiki <https://wiki.python.org/moin/PythonEditors>`_ for a full list -of Python editing environments. - -If you want to discuss Python's use in education, you may be interested in -joining `the edu-sig mailing list -<https://www.python.org/community/sigs/current/edu-sig>`_. diff --git a/third_party/python/Doc/faq/gui.rst b/third_party/python/Doc/faq/gui.rst deleted file mode 100644 index 38e179626..000000000 --- a/third_party/python/Doc/faq/gui.rst +++ /dev/null @@ -1,159 +0,0 @@ -:tocdepth: 2 - -========================== -Graphic User Interface FAQ -========================== - -.. only:: html - - .. contents:: - -.. XXX need review for Python 3. - - -General GUI Questions -===================== - -What platform-independent GUI toolkits exist for Python? -======================================================== - -Depending on what platform(s) you are aiming at, there are several. Some -of them haven't been ported to Python 3 yet. At least `Tkinter`_ and `Qt`_ -are known to be Python 3-compatible. - -.. XXX check links - -Tkinter -------- - -Standard builds of Python include an object-oriented interface to the Tcl/Tk -widget set, called :ref:`tkinter <Tkinter>`. This is probably the easiest to -install (since it comes included with most -`binary distributions <https://www.python.org/downloads/>`_ of Python) and use. -For more info about Tk, including pointers to the source, see the -`Tcl/Tk home page <https://www.tcl.tk>`_. Tcl/Tk is fully portable to the -Mac OS X, Windows, and Unix platforms. - -wxWidgets ---------- - -wxWidgets (https://www.wxwidgets.org) is a free, portable GUI class -library written in C++ that provides a native look and feel on a -number of platforms, with Windows, Mac OS X, GTK, X11, all listed as -current stable targets. Language bindings are available for a number -of languages including Python, Perl, Ruby, etc. - -wxPython (http://www.wxpython.org) is the Python binding for -wxwidgets. While it often lags slightly behind the official wxWidgets -releases, it also offers a number of features via pure Python -extensions that are not available in other language bindings. There -is an active wxPython user and developer community. - -Both wxWidgets and wxPython are free, open source, software with -permissive licences that allow their use in commercial products as -well as in freeware or shareware. - - -Qt ---- - -There are bindings available for the Qt toolkit (using either `PyQt -<https://riverbankcomputing.com/software/pyqt/intro>`_ or `PySide -<https://wiki.qt.io/PySide>`_) and for KDE (`PyKDE4 <https://techbase.kde.org/Languages/Python/Using_PyKDE_4>`__). -PyQt is currently more mature than PySide, but you must buy a PyQt license from -`Riverbank Computing <https://www.riverbankcomputing.com/commercial/license-faq>`_ -if you want to write proprietary applications. PySide is free for all applications. - -Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses -are available from `The Qt Company <https://www.qt.io/licensing/>`_. - -Gtk+ ----- - -The `GObject introspection bindings <https://wiki.gnome.org/Projects/PyGObject>`_ -for Python allow you to write GTK+ 3 applications. There is also a -`Python GTK+ 3 Tutorial <https://python-gtk-3-tutorial.readthedocs.org/en/latest/>`_. - -The older PyGtk bindings for the `Gtk+ 2 toolkit <http://www.gtk.org>`_ have -been implemented by James Henstridge; see <http://www.pygtk.org>. - -Kivy ----- - -`Kivy <https://kivy.org/>`_ is a cross-platform GUI library supporting both -desktop operating systems (Windows, macOS, Linux) and mobile devices (Android, -iOS). It is written in Python and Cython, and can use a range of windowing -backends. - -Kivy is free and open source software distributed under the MIT license. - -FLTK ----- - -Python bindings for `the FLTK toolkit <http://www.fltk.org>`_, a simple yet -powerful and mature cross-platform windowing system, are available from `the -PyFLTK project <http://pyfltk.sourceforge.net>`_. - -OpenGL ------- - -For OpenGL bindings, see `PyOpenGL <http://pyopengl.sourceforge.net>`_. - - -What platform-specific GUI toolkits exist for Python? -======================================================== - -By installing the `PyObjc Objective-C bridge -<https://pythonhosted.org/pyobjc/>`_, Python programs can use Mac OS X's -Cocoa libraries. - -:ref:`Pythonwin <windows-faq>` by Mark Hammond includes an interface to the -Microsoft Foundation Classes and a Python programming environment -that's written mostly in Python using the MFC classes. - - -Tkinter questions -================= - -How do I freeze Tkinter applications? -------------------------------------- - -Freeze is a tool to create stand-alone applications. When freezing Tkinter -applications, the applications will not be truly stand-alone, as the application -will still need the Tcl and Tk libraries. - -One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` -environment variables. - -To get truly stand-alone applications, the Tcl scripts that form the library -have to be integrated into the application as well. One tool supporting that is -SAM (stand-alone modules), which is part of the Tix distribution -(http://tix.sourceforge.net/). - -Build Tix with SAM enabled, perform the appropriate call to -:c:func:`Tclsam_init`, etc. inside Python's -:file:`Modules/tkappinit.c`, and link with libtclsam and libtksam (you -might include the Tix libraries as well). - - -Can I have Tk events handled while waiting for I/O? ---------------------------------------------------- - -On platforms other than Windows, yes, and you don't even -need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you -to register a callback function which will be called from the Tk mainloop when -I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. - - -I can't get key bindings to work in Tkinter: why? -------------------------------------------------- - -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. - -The most common cause is that the widget to which the binding applies doesn't -have "keyboard focus". Check out the Tk documentation for the focus command. -Usually a widget is given the keyboard focus by clicking in it (but not for -labels; see the takefocus option). diff --git a/third_party/python/Doc/faq/index.rst b/third_party/python/Doc/faq/index.rst deleted file mode 100644 index 46ed3db22..000000000 --- a/third_party/python/Doc/faq/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _faq-index: - -################################### - Python Frequently Asked Questions -################################### - -.. toctree:: - :maxdepth: 1 - - general.rst - programming.rst - design.rst - library.rst - extending.rst - windows.rst - gui.rst - installed.rst diff --git a/third_party/python/Doc/faq/installed.rst b/third_party/python/Doc/faq/installed.rst deleted file mode 100644 index 42296533e..000000000 --- a/third_party/python/Doc/faq/installed.rst +++ /dev/null @@ -1,53 +0,0 @@ -============================================= -"Why is Python Installed on my Computer?" FAQ -============================================= - -What is Python? ---------------- - -Python is a programming language. It's used for many different applications. -It's used in some high schools and colleges as an introductory programming -language because Python is easy to learn, but it's also used by professional -software developers at places such as Google, NASA, and Lucasfilm Ltd. - -If you wish to learn more about Python, start with the `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_. - - -Why is Python installed on my machine? --------------------------------------- - -If you find Python installed on your system but don't remember installing it, -there are several possible ways it could have gotten there. - -* Perhaps another user on the computer wanted to learn programming and installed - it; you'll have to figure out who's been using the machine and might have - installed it. -* A third-party application installed on the machine might have been written in - Python and included a Python installation. There are many such applications, - from GUI programs to network servers and administrative scripts. -* Some Windows machines also have Python installed. At this writing we're aware - of computers from Hewlett-Packard and Compaq that include Python. Apparently - some of HP/Compaq's administrative tools are written in Python. -* Many Unix-compatible operating systems, such as Mac OS X and some Linux - distributions, have Python installed by default; it's included in the base - installation. - - -Can I delete Python? --------------------- - -That depends on where Python came from. - -If someone installed it deliberately, you can remove it without hurting -anything. On Windows, use the Add/Remove Programs icon in the Control Panel. - -If Python was installed by a third-party application, you can also remove it, -but that application will no longer work. You should use that application's -uninstaller rather than removing Python directly. - -If Python came with your operating system, removing it is not recommended. If -you remove it, whatever tools were written in Python will no longer run, and -some of them might be important to you. Reinstalling the whole system would -then be required to fix things again. - diff --git a/third_party/python/Doc/faq/library.rst b/third_party/python/Doc/faq/library.rst deleted file mode 100644 index 1acfc2680..000000000 --- a/third_party/python/Doc/faq/library.rst +++ /dev/null @@ -1,839 +0,0 @@ -:tocdepth: 2 - -========================= -Library and Extension FAQ -========================= - -.. only:: html - - .. contents:: - -General Library Questions -========================= - -How do I find a module or application to perform task X? --------------------------------------------------------- - -Check :ref:`the Library Reference <library-index>` to see if there's a relevant -standard library module. (Eventually you'll learn what's in the standard -library and will be able to skip this step.) - -For third-party packages, search the `Python Package Index -<https://pypi.org>`_ or try `Google <https://www.google.com>`_ or -another Web search engine. Searching for "Python" plus a keyword or two for -your topic of interest will usually find something helpful. - - -Where is the math.py (socket.py, regex.py, etc.) source file? -------------------------------------------------------------- - -If you can't find a source file for a module it may be a built-in or -dynamically loaded module implemented in C, C++ or other compiled language. -In this case you may not have the source file or it may be something like -:file:`mathmodule.c`, somewhere in a C source directory (not on the Python Path). - -There are (at least) three kinds of modules in Python: - -1) modules written in Python (.py); -2) modules written in C and dynamically loaded (.dll, .pyd, .so, .sl, etc); -3) modules written in C and linked with the interpreter; to get a list of these, - type:: - - import sys - print(sys.builtin_module_names) - - -How do I make a Python script executable on Unix? -------------------------------------------------- - -You need to do two things: the script file's mode must be executable and the -first line must begin with ``#!`` followed by the path of the Python -interpreter. - -The first is done by executing ``chmod +x scriptfile`` or perhaps ``chmod 755 -scriptfile``. - -The second can be done in a number of ways. The most straightforward way is to -write :: - - #!/usr/local/bin/python - -as the very first line of your file, using the pathname for where the Python -interpreter is installed on your platform. - -If you would like the script to be independent of where the Python interpreter -lives, you can use the :program:`env` program. Almost all Unix variants support -the following, assuming the Python interpreter is in a directory on the user's -:envvar:`PATH`:: - - #!/usr/bin/env python - -*Don't* do this for CGI scripts. The :envvar:`PATH` variable for CGI scripts is -often very minimal, so you need to use the actual absolute pathname of the -interpreter. - -Occasionally, a user's environment is so full that the :program:`/usr/bin/env` -program fails; or there's no env program at all. In that case, you can try the -following hack (due to Alex Rezinsky): - -.. code-block:: sh - - #! /bin/sh - """:" - exec python $0 ${1+"$@"} - """ - -The minor disadvantage is that this defines the script's __doc__ string. -However, you can fix that by adding :: - - __doc__ = """...Whatever...""" - - - -Is there a curses/termcap package for Python? ---------------------------------------------- - -.. XXX curses *is* built by default, isn't it? - -For Unix variants: The standard Python source distribution comes with a curses -module in the :source:`Modules` subdirectory, though it's not compiled by default. -(Note that this is not available in the Windows distribution -- there is no -curses module for Windows.) - -The :mod:`curses` module supports basic curses features as well as many additional -functions from ncurses and SYSV curses such as colour, alternative character set -support, pads, and mouse support. This means the module isn't compatible with -operating systems that only have BSD curses, but there don't seem to be any -currently maintained OSes that fall into this category. - -For Windows: use `the consolelib module -<http://effbot.org/zone/console-index.htm>`_. - - -Is there an equivalent to C's onexit() in Python? -------------------------------------------------- - -The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. - - -Why don't my signal handlers work? ----------------------------------- - -The most common problem is that the signal handler is declared with the wrong -argument list. It is called as :: - - handler(signum, frame) - -so it should be declared with two arguments:: - - def handler(signum, frame): - ... - - -Common tasks -============ - -How do I test a Python program or component? --------------------------------------------- - -Python comes with two testing frameworks. The :mod:`doctest` module finds -examples in the docstrings for a module and runs them, comparing the output with -the expected output given in the docstring. - -The :mod:`unittest` module is a fancier testing framework modelled on Java and -Smalltalk testing frameworks. - -To make testing easier, you should use good modular design in your program. -Your program should have almost all functionality -encapsulated in either functions or class methods -- and this sometimes has the -surprising and delightful effect of making the program run faster (because local -variable accesses are faster than global accesses). Furthermore the program -should avoid depending on mutating global variables, since this makes testing -much more difficult to do. - -The "global main logic" of your program may be as simple as :: - - if __name__ == "__main__": - main_logic() - -at the bottom of the main module of your program. - -Once your program is organized as a tractable collection of functions and class -behaviours you should write test functions that exercise the behaviours. A test -suite that automates a sequence of tests can be associated with each module. -This sounds like a lot of work, but since Python is so terse and flexible it's -surprisingly easy. You can make coding much more pleasant and fun by writing -your test functions in parallel with the "production code", since this makes it -easy to find bugs and even design flaws earlier. - -"Support modules" that are not intended to be the main module of a program may -include a self-test of the module. :: - - if __name__ == "__main__": - self_test() - -Even programs that interact with complex external interfaces may be tested when -the external interfaces are unavailable by using "fake" interfaces implemented -in Python. - - -How do I create documentation from doc strings? ------------------------------------------------ - -The :mod:`pydoc` module can create HTML from the doc strings in your Python -source code. An alternative for creating API documentation purely from -docstrings is `epydoc <http://epydoc.sourceforge.net/>`_. `Sphinx -<http://sphinx-doc.org>`_ can also include docstring content. - - -How do I get a single keypress at a time? ------------------------------------------ - -For Unix variants there are several solutions. It's straightforward to do this -using curses, but curses is a fairly large module to learn. - -.. XXX this doesn't work out of the box, some IO expert needs to check why - - Here's a solution without curses:: - - import termios, fcntl, sys, os - fd = sys.stdin.fileno() - - oldterm = termios.tcgetattr(fd) - newattr = termios.tcgetattr(fd) - newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO - termios.tcsetattr(fd, termios.TCSANOW, newattr) - - oldflags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK) - - try: - while True: - try: - c = sys.stdin.read(1) - print("Got character", repr(c)) - except OSError: - pass - finally: - termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm) - fcntl.fcntl(fd, fcntl.F_SETFL, oldflags) - - You need the :mod:`termios` and the :mod:`fcntl` module for any of this to - work, and I've only tried it on Linux, though it should work elsewhere. In - this code, characters are read and printed one at a time. - - :func:`termios.tcsetattr` turns off stdin's echoing and disables canonical - mode. :func:`fcntl.fnctl` is used to obtain stdin's file descriptor flags - and modify them for non-blocking mode. Since reading stdin when it is empty - results in an :exc:`OSError`, this error is caught and ignored. - - .. versionchanged:: 3.3 - *sys.stdin.read* used to raise :exc:`IOError`. Starting from Python 3.3 - :exc:`IOError` is alias for :exc:`OSError`. - - -Threads -======= - -How do I program using threads? -------------------------------- - -Be sure to use the :mod:`threading` module and not the :mod:`_thread` module. -The :mod:`threading` module builds convenient abstractions on top of the -low-level primitives provided by the :mod:`_thread` module. - -Aahz has a set of slides from his threading tutorial that are helpful; see -http://www.pythoncraft.com/OSCON2001/. - - -None of my threads seem to run: why? ------------------------------------- - -As soon as the main thread exits, all threads are killed. Your main thread is -running too quickly, giving the threads no time to do any work. - -A simple fix is to add a sleep to the end of the program that's long enough for -all the threads to finish:: - - import threading, time - - def thread_task(name, n): - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) # <---------------------------! - -But now (on many platforms) the threads don't run in parallel, but appear to run -sequentially, one at a time! The reason is that the OS thread scheduler doesn't -start a new thread until the previous thread is blocked. - -A simple fix is to add a tiny sleep to the start of the run function:: - - def thread_task(name, n): - time.sleep(0.001) # <--------------------! - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) - -Instead of trying to guess a good delay value for :func:`time.sleep`, -it's better to use some kind of semaphore mechanism. One idea is to use the -:mod:`queue` module to create a queue object, let each thread append a token to -the queue when it finishes, and let the main thread read as many tokens from the -queue as there are threads. - - -How do I parcel out work among a bunch of worker threads? ---------------------------------------------------------- - -The easiest way is to use the new :mod:`concurrent.futures` module, -especially the :mod:`~concurrent.futures.ThreadPoolExecutor` class. - -Or, if you want fine control over the dispatching algorithm, you can write -your own logic manually. Use the :mod:`queue` module to create a queue -containing a list of jobs. The :class:`~queue.Queue` class maintains a -list of objects and has a ``.put(obj)`` method that adds items to the queue and -a ``.get()`` method to return them. The class will take care of the locking -necessary to ensure that each job is handed out exactly once. - -Here's a trivial example:: - - import threading, queue, time - - # The worker thread gets jobs off the queue. When the queue is empty, it - # assumes there will be no more work and exits. - # (Realistically workers will run until terminated.) - def worker(): - print('Running worker') - time.sleep(0.1) - while True: - try: - arg = q.get(block=False) - except queue.Empty: - print('Worker', threading.currentThread(), end=' ') - print('queue empty') - break - else: - print('Worker', threading.currentThread(), end=' ') - print('running with argument', arg) - time.sleep(0.5) - - # Create queue - q = queue.Queue() - - # Start a pool of 5 workers - for i in range(5): - t = threading.Thread(target=worker, name='worker %i' % (i+1)) - t.start() - - # Begin adding work to the queue - for i in range(50): - q.put(i) - - # Give threads time to run - print('Main thread sleeping') - time.sleep(5) - -When run, this will produce the following output: - -.. code-block:: none - - Running worker - Running worker - Running worker - Running worker - Running worker - Main thread sleeping - Worker <Thread(worker 1, started 130283832797456)> running with argument 0 - Worker <Thread(worker 2, started 130283824404752)> running with argument 1 - Worker <Thread(worker 3, started 130283816012048)> running with argument 2 - Worker <Thread(worker 4, started 130283807619344)> running with argument 3 - Worker <Thread(worker 5, started 130283799226640)> running with argument 4 - Worker <Thread(worker 1, started 130283832797456)> running with argument 5 - ... - -Consult the module's documentation for more details; the :class:`~queue.Queue` -class provides a featureful interface. - - -What kinds of global value mutation are thread-safe? ----------------------------------------------------- - -A :term:`global interpreter lock` (GIL) is used internally to ensure that only one -thread runs in the Python VM at a time. In general, Python offers to switch -among threads only between bytecode instructions; how frequently it switches can -be set via :func:`sys.setswitchinterval`. Each bytecode instruction and -therefore all the C implementation code reached from each instruction is -therefore atomic from the point of view of a Python program. - -In theory, this means an exact accounting requires an exact understanding of the -PVM bytecode implementation. In practice, it means that operations on shared -variables of built-in data types (ints, lists, dicts, etc) that "look atomic" -really are. - -For example, the following operations are all atomic (L, L1, L2 are lists, D, -D1, D2 are dicts, x, y are objects, i, j are ints):: - - L.append(x) - L1.extend(L2) - x = L[i] - x = L.pop() - L1[i:j] = L2 - L.sort() - x = y - x.field = y - D[x] = y - D1.update(D2) - D.keys() - -These aren't:: - - i = i+1 - L.append(L[-1]) - L[i] = L[j] - D[x] = D[x] + 1 - -Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can -affect things. This is especially true for the mass updates to dictionaries and -lists. When in doubt, use a mutex! - - -Can't we get rid of the Global Interpreter Lock? ------------------------------------------------- - -.. XXX link to dbeazley's talk about GIL? - -The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's -deployment on high-end multiprocessor server machines, because a multi-threaded -Python program effectively only uses one CPU, due to the insistence that -(almost) all Python code can only run while the GIL is held. - -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive -patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment -in his `python-safethread <http://code.google.com/p/python-safethread/>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread -performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. - -This doesn't mean that you can't make good use of Python on multi-CPU machines! -You just have to be creative with dividing the work up between multiple -*processes* rather than multiple *threads*. The -:class:`~concurrent.futures.ProcessPoolExecutor` class in the new -:mod:`concurrent.futures` module provides an easy way of doing so; the -:mod:`multiprocessing` module provides a lower-level API in case you want -more control over dispatching of tasks. - -Judicious use of C extensions will also help; if you use a C extension to -perform a time-consuming task, the extension can release the GIL while the -thread of execution is in the C code and allow other threads to get some work -done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` -already do this. - -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? - - -Input and Output -================ - -How do I delete a file? (And other file questions...) ------------------------------------------------------ - -Use ``os.remove(filename)`` or ``os.unlink(filename)``; for documentation, see -the :mod:`os` module. The two functions are identical; :func:`~os.unlink` is simply -the name of the Unix system call for this function. - -To remove a directory, use :func:`os.rmdir`; use :func:`os.mkdir` to create one. -``os.makedirs(path)`` will create any intermediate directories in ``path`` that -don't exist. ``os.removedirs(path)`` will remove intermediate directories as -long as they're empty; if you want to delete an entire directory tree and its -contents, use :func:`shutil.rmtree`. - -To rename a file, use ``os.rename(old_path, new_path)``. - -To truncate a file, open it using ``f = open(filename, "rb+")``, and use -``f.truncate(offset)``; offset defaults to the current seek position. There's -also ``os.ftruncate(fd, offset)`` for files opened with :func:`os.open`, where -*fd* is the file descriptor (a small integer). - -The :mod:`shutil` module also contains a number of functions to work on files -including :func:`~shutil.copyfile`, :func:`~shutil.copytree`, and -:func:`~shutil.rmtree`. - - -How do I copy a file? ---------------------- - -The :mod:`shutil` module contains a :func:`~shutil.copyfile` function. Note -that on MacOS 9 it doesn't copy the resource fork and Finder info. - - -How do I read (or write) binary data? -------------------------------------- - -To read or write complex binary data formats, it's best to use the :mod:`struct` -module. It allows you to take a string containing binary data (usually numbers) -and convert it to Python objects; and vice versa. - -For example, the following code reads two 2-byte integers and one 4-byte integer -in big-endian format from a file:: - - import struct - - with open(filename, "rb") as f: - s = f.read(8) - x, y, z = struct.unpack(">hhl", s) - -The '>' in the format string forces big-endian data; the letter 'h' reads one -"short integer" (2 bytes), and 'l' reads one "long integer" (4 bytes) from the -string. - -For data that is more regular (e.g. a homogeneous list of ints or floats), -you can also use the :mod:`array` module. - -.. note:: - - To read and write binary data, it is mandatory to open the file in - binary mode (here, passing ``"rb"`` to :func:`open`). If you use - ``"r"`` instead (the default), the file will be open in text mode - and ``f.read()`` will return :class:`str` objects rather than - :class:`bytes` objects. - - -I can't seem to use os.read() on a pipe created with os.popen(); why? ---------------------------------------------------------------------- - -:func:`os.read` is a low-level function which takes a file descriptor, a small -integer representing the opened file. :func:`os.popen` creates a high-level -file object, the same type returned by the built-in :func:`open` function. -Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need to -use ``p.read(n)``. - - -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :data:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from http://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - <https://pypi.org/project/pexpect/>`_. - - -How do I access the serial (RS232) port? ----------------------------------------- - -For Win32, POSIX (Linux, BSD, etc.), Jython: - - http://pyserial.sourceforge.net - -For Unix, see a Usenet post by Mitch Chapman: - - https://groups.google.com/groups?selm=34A04430.CF9@ohioee.com - - -Why doesn't closing sys.stdout (stdin, stderr) really close it? ---------------------------------------------------------------- - -Python :term:`file objects <file object>` are a high-level layer of -abstraction on low-level C file descriptors. - -For most file objects you create in Python via the built-in :func:`open` -function, ``f.close()`` marks the Python file object as being closed from -Python's point of view, and also arranges to close the underlying C file -descriptor. This also happens automatically in ``f``'s destructor, when -``f`` becomes garbage. - -But stdin, stdout and stderr are treated specially by Python, because of the -special status also given to them by C. Running ``sys.stdout.close()`` marks -the Python-level file object as being closed, but does *not* close the -associated C file descriptor. - -To close the underlying C file descriptor for one of these three, you should -first be sure that's what you really want to do (e.g., you may confuse -extension modules trying to do I/O). If it is, use :func:`os.close`:: - - os.close(stdin.fileno()) - os.close(stdout.fileno()) - os.close(stderr.fileno()) - -Or you can use the numeric constants 0, 1 and 2, respectively. - - -Network/Internet Programming -============================ - -What WWW tools are there for Python? ------------------------------------- - -See the chapters titled :ref:`internet` and :ref:`netdata` in the Library -Reference Manual. Python has many modules that will help you build server-side -and client-side web systems. - -.. XXX check if wiki page is still up to date - -A summary of available frameworks is maintained by Paul Boddie at -https://wiki.python.org/moin/WebProgramming\ . - -Cameron Laird maintains a useful set of pages about Python web technologies at -http://phaseit.net/claird/comp.lang.python/web_python. - - -How can I mimic CGI form submission (METHOD=POST)? --------------------------------------------------- - -I would like to retrieve web pages that are the result of POSTing a form. Is -there existing code that would let me do this easily? - -Yes. Here's a simple example that uses urllib.request:: - - #!/usr/local/bin/python - - import urllib.request - - # build the query string - qs = "First=Josephine&MI=Q&Last=Public" - - # connect and send the server a path - req = urllib.request.urlopen('http://www.some-server.out-there' - '/cgi-bin/some-cgi-script', data=qs) - with req: - msg, hdrs = req.read(), req.info() - -Note that in general for percent-encoded POST operations, query strings must be -quoted using :func:`urllib.parse.urlencode`. For example, to send -``name=Guy Steele, Jr.``:: - - >>> import urllib.parse - >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'}) - 'name=Guy+Steele%2C+Jr.' - -.. seealso:: :ref:`urllib-howto` for extensive examples. - - -What module should I use to help with generating HTML? ------------------------------------------------------- - -.. XXX add modern template languages - -You can find a collection of useful links on the `Web Programming wiki page -<https://wiki.python.org/moin/WebProgramming>`_. - - -How do I send mail from a Python script? ----------------------------------------- - -Use the standard library module :mod:`smtplib`. - -Here's a very simple interactive mail sender that uses it. This method will -work on any host that supports an SMTP listener. :: - - import sys, smtplib - - fromaddr = input("From: ") - toaddrs = input("To: ").split(',') - print("Enter message, end with ^D:") - msg = '' - while True: - line = sys.stdin.readline() - if not line: - break - msg += line - - # The actual mail send - server = smtplib.SMTP('localhost') - server.sendmail(fromaddr, toaddrs, msg) - server.quit() - -A Unix-only alternative uses sendmail. The location of the sendmail program -varies between systems; sometimes it is ``/usr/lib/sendmail``, sometimes -``/usr/sbin/sendmail``. The sendmail manual page will help you out. Here's -some sample code:: - - import os - - SENDMAIL = "/usr/sbin/sendmail" # sendmail location - p = os.popen("%s -t -i" % SENDMAIL, "w") - p.write("To: receiver@example.com\n") - p.write("Subject: test\n") - p.write("\n") # blank line separating headers from body - p.write("Some text\n") - p.write("some more text\n") - sts = p.close() - if sts != 0: - print("Sendmail exit status", sts) - - -How do I avoid blocking in the connect() method of a socket? ------------------------------------------------------------- - -The :mod:`select` module is commonly used to help with asynchronous I/O on -sockets. - -To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the ``connect()``, you will either connect immediately -(unlikely) or get an exception that contains the error number as ``.errno``. -``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't -finished yet. Different OSes will return different values, so you're going to -have to check what's returned on your system. - -You can use the ``connect_ex()`` method to avoid creating an exception. It will -just return the errno value. To poll, you can call ``connect_ex()`` again later --- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this -socket to select to check if it's writable. - -.. note:: - The :mod:`asyncore` module presents a framework-like approach to the problem - of writing non-blocking networking code. - The third-party `Twisted <https://twistedmatrix.com/trac/>`_ library is - a popular and feature-rich alternative. - - -Databases -========= - -Are there any interfaces to database packages in Python? --------------------------------------------------------- - -Yes. - -Interfaces to disk-based hashes such as :mod:`DBM <dbm.ndbm>` and :mod:`GDBM -<dbm.gnu>` are also included with standard Python. There is also the -:mod:`sqlite3` module, which provides a lightweight disk-based relational -database. - -Support for most relational databases is available. See the -`DatabaseProgramming wiki page -<https://wiki.python.org/moin/DatabaseProgramming>`_ for details. - - -How do you implement persistent objects in Python? --------------------------------------------------- - -The :mod:`pickle` library module solves this in a very general way (though you -still can't store things like open files, sockets or windows), and the -:mod:`shelve` library module uses pickle and (g)dbm to create persistent -mappings containing arbitrary Python objects. - - -Mathematics and Numerics -======================== - -How do I generate random numbers in Python? -------------------------------------------- - -The standard module :mod:`random` implements a random number generator. Usage -is simple:: - - import random - random.random() - -This returns a random floating point number in the range [0, 1). - -There are also many other specialized generators in this module, such as: - -* ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). -* ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. - -Some higher-level functions operate on sequences directly, such as: - -* ``choice(S)`` chooses random element from a given sequence -* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly - -There's also a ``Random`` class you can instantiate to create independent -multiple random number generators. diff --git a/third_party/python/Doc/faq/programming.rst b/third_party/python/Doc/faq/programming.rst deleted file mode 100644 index b717ab8f0..000000000 --- a/third_party/python/Doc/faq/programming.rst +++ /dev/null @@ -1,1868 +0,0 @@ -:tocdepth: 2 - -=============== -Programming FAQ -=============== - -.. only:: html - - .. contents:: - -General Questions -================= - -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- - -Yes. - -The pdb module is a simple but adequate console-mode debugger for Python. It is -part of the standard Python library, and is :mod:`documented in the Library -Reference Manual <pdb>`. You can also write your own debugger by using the code -for pdb as an example. - -The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as Tools/scripts/idle), includes a -graphical debugger. - -PythonWin is a Python IDE that includes a GUI debugger based on pdb. The -Pythonwin debugger colors breakpoints and has quite a few cool features such as -debugging non-Pythonwin programs. Pythonwin is available as part of the `Python -for Windows Extensions <https://sourceforge.net/projects/pywin32/>`__ project and -as a part of the ActivePython distribution (see -https://www.activestate.com/activepython\ ). - -`Boa Constructor <http://boa-constructor.sourceforge.net/>`_ is an IDE and GUI -builder that uses wxWidgets. It offers visual frame creation and manipulation, -an object inspector, many views on the source like object browsers, inheritance -hierarchies, doc string generated html documentation, an advanced debugger, -integrated help, and Zope support. - -`Eric <http://eric-ide.python-projects.org/>`_ is an IDE built on PyQt -and the Scintilla editing component. - -Pydb is a version of the standard Python debugger pdb, modified for use with DDD -(Data Display Debugger), a popular graphical debugger front end. Pydb can be -found at http://bashdb.sourceforge.net/pydb/ and DDD can be found at -https://www.gnu.org/software/ddd. - -There are a number of commercial Python IDEs that include graphical debuggers. -They include: - -* Wing IDE (https://wingware.com/) -* Komodo IDE (https://komodoide.com/) -* PyCharm (https://www.jetbrains.com/pycharm/) - - -Is there a tool to help find bugs or perform static analysis? -------------------------------------------------------------- - -Yes. - -PyChecker is a static analysis tool that finds bugs in Python source code and -warns about code complexity and style. You can get PyChecker from -http://pychecker.sourceforge.net/. - -`Pylint <https://www.pylint.org/>`_ is another tool that checks -if a module satisfies a coding standard, and also makes it possible to write -plug-ins to add a custom feature. In addition to the bug checking that -PyChecker performs, Pylint offers some additional features such as checking line -length, whether variable names are well-formed according to your coding -standard, whether declared interfaces are fully implemented, and more. -https://docs.pylint.org/ provides a full list of Pylint's features. - - -How can I create a stand-alone binary from a Python script? ------------------------------------------------------------ - -You don't need the ability to compile Python to C code if all you want is a -stand-alone program that users can download and run without having to install -the Python distribution first. There are a number of tools that determine the -set of modules required by a program and bind these modules together with a -Python binary to produce a single executable. - -One is to use the freeze tool, which is included in the Python source tree as -``Tools/freeze``. It converts Python byte code to C arrays; a C compiler you can -embed all your modules into a new program, which is then linked with the -standard Python modules. - -It works by scanning your source recursively for import statements (in both -forms) and looking for the modules in the standard Python path as well as in the -source directory (for built-in modules). It then turns the bytecode for modules -written in Python into C code (array initializers that can be turned into code -objects using the marshal module) and creates a custom-made config file that -only contains those built-in modules which are actually used in the program. It -then compiles the generated C code and links it with the rest of the Python -interpreter to form a self-contained binary which acts exactly like your script. - -Obviously, freeze requires a C compiler. There are several other utilities -which don't. One is Thomas Heller's py2exe (Windows only) at - - http://www.py2exe.org/ - -Another tool is Anthony Tuininga's `cx_Freeze <http://cx-freeze.sourceforge.net/>`_. - - -Are there coding standards or a style guide for Python programs? ----------------------------------------------------------------- - -Yes. The coding style required for standard library modules is documented as -:pep:`8`. - - -Core Language -============= - -Why am I getting an UnboundLocalError when the variable has a value? --------------------------------------------------------------------- - -It can be a surprise to get the UnboundLocalError in previously working -code when it is modified by adding an assignment statement somewhere in -the body of a function. - -This code: - - >>> x = 10 - >>> def bar(): - ... print(x) - >>> bar() - 10 - -works, but this code: - - >>> x = 10 - >>> def foo(): - ... print(x) - ... x += 1 - -results in an UnboundLocalError: - - >>> foo() - Traceback (most recent call last): - ... - UnboundLocalError: local variable 'x' referenced before assignment - -This is because when you make an assignment to a variable in a scope, that -variable becomes local to that scope and shadows any similarly named variable -in the outer scope. Since the last statement in foo assigns a new value to -``x``, the compiler recognizes it as a local variable. Consequently when the -earlier ``print(x)`` attempts to print the uninitialized local variable and -an error results. - -In the example above you can access the outer scope variable by declaring it -global: - - >>> x = 10 - >>> def foobar(): - ... global x - ... print(x) - ... x += 1 - >>> foobar() - 10 - -This explicit declaration is required in order to remind you that (unlike the -superficially analogous situation with class and instance variables) you are -actually modifying the value of the variable in the outer scope: - - >>> print(x) - 11 - -You can do a similar thing in a nested scope using the :keyword:`nonlocal` -keyword: - - >>> def foo(): - ... x = 10 - ... def bar(): - ... nonlocal x - ... print(x) - ... x += 1 - ... bar() - ... print(x) - >>> foo() - 10 - 11 - - -What are the rules for local and global variables in Python? ------------------------------------------------------------- - -In Python, variables that are only referenced inside a function are implicitly -global. If a variable is assigned a value anywhere within the function's body, -it's assumed to be a local unless explicitly declared as global. - -Though a bit surprising at first, a moment's consideration explains this. On -one hand, requiring :keyword:`global` for assigned variables provides a bar -against unintended side-effects. On the other hand, if ``global`` was required -for all global references, you'd be using ``global`` all the time. You'd have -to declare as global every reference to a built-in function or to a component of -an imported module. This clutter would defeat the usefulness of the ``global`` -declaration for identifying side-effects. - - -Why do lambdas defined in a loop with different values all return the same result? ----------------------------------------------------------------------------------- - -Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda: x**2) - -This gives you a list that contains 5 lambdas that calculate ``x**2``. You -might expect that, when called, they would return, respectively, ``0``, ``1``, -``4``, ``9``, and ``16``. However, when you actually try you will see that -they all return ``16``:: - - >>> squares[2]() - 16 - >>> squares[4]() - 16 - -This happens because ``x`` is not local to the lambdas, but is defined in -the outer scope, and it is accessed when the lambda is called --- not when it -is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by -changing the value of ``x`` and see how the results of the lambdas change:: - - >>> x = 8 - >>> squares[2]() - 64 - -In order to avoid this, you need to save the values in variables local to the -lambdas, so that they don't rely on the value of the global ``x``:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda n=x: n**2) - -Here, ``n=x`` creates a new variable ``n`` local to the lambda and computed -when the lambda is defined so that it has the same value that ``x`` had at -that point in the loop. This means that the value of ``n`` will be ``0`` -in the first lambda, ``1`` in the second, ``2`` in the third, and so on. -Therefore each lambda will now return the correct result:: - - >>> squares[2]() - 4 - >>> squares[4]() - 16 - -Note that this behaviour is not peculiar to lambdas, but applies to regular -functions too. - - -How do I share global variables across modules? ------------------------------------------------- - -The canonical way to share information across modules within a single program is -to create a special module (often called config or cfg). Just import the config -module in all modules of your application; the module then becomes available as -a global name. Because there is only one instance of each module, any changes -made to the module object get reflected everywhere. For example: - -config.py:: - - x = 0 # Default value of the 'x' configuration setting - -mod.py:: - - import config - config.x = 1 - -main.py:: - - import config - import mod - print(config.x) - -Note that using a module is also the basis for implementing the Singleton design -pattern, for the same reason. - - -What are the "best practices" for using import in a module? ------------------------------------------------------------ - -In general, don't use ``from modulename import *``. Doing so clutters the -importer's namespace, and makes it much harder for linters to detect undefined -names. - -Import modules at the top of a file. Doing so makes it clear what other modules -your code requires and avoids questions of whether the module name is in scope. -Using one import per line makes it easy to add and delete module imports, but -using multiple imports per line uses less screen space. - -It's good practice if you import modules in the following order: - -1. standard library modules -- e.g. ``sys``, ``os``, ``getopt``, ``re`` -2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. mx.DateTime, ZODB, PIL.Image, etc. -3. locally-developed modules - -It is sometimes necessary to move imports to a function or class to avoid -problems with circular imports. Gordon McMillan says: - - Circular imports are fine where both modules use the "import <module>" form - of import. They fail when the 2nd module wants to grab a name out of the - first ("from module import name") and the import is at the top level. That's - because names in the 1st are not yet available, because the first module is - busy importing the 2nd. - -In this case, if the second module is only used in one function, then the import -can easily be moved into that function. By the time the import is called, the -first module will have finished initializing, and the second module can do its -import. - -It may also be necessary to move imports out of the top level of code if some of -the modules are platform-specific. In that case, it may not even be possible to -import all of the modules at the top of the file. In this case, importing the -correct modules in the corresponding platform-specific code is a good option. - -Only move imports into a local scope, such as inside a function definition, if -it's necessary to solve a problem such as avoiding a circular import or are -trying to reduce the initialization time of a module. This technique is -especially helpful if many of the imports are unnecessary depending on how the -program executes. You may also want to move imports into a function if the -modules are only ever used in that function. Note that loading a module the -first time may be expensive because of the one time initialization of the -module, but loading a module multiple times is virtually free, costing only a -couple of dictionary lookups. Even if the module name has gone out of scope, -the module is probably available in :data:`sys.modules`. - - -Why are default values shared between objects? ----------------------------------------------- - -This type of bug commonly bites neophyte programmers. Consider this function:: - - def foo(mydict={}): # Danger: shared reference to one dict for all calls - ... compute something ... - mydict[key] = value - return mydict - -The first time you call this function, ``mydict`` contains a single item. The -second time, ``mydict`` contains two items because when ``foo()`` begins -executing, ``mydict`` starts out with an item already in it. - -It is often expected that a function call creates new objects for default -values. This is not what happens. Default values are created exactly once, when -the function is defined. If that object is changed, like the dictionary in this -example, subsequent calls to the function will refer to this changed object. - -By definition, immutable objects such as numbers, strings, tuples, and ``None``, -are safe from change. Changes to mutable objects such as dictionaries, lists, -and class instances can lead to confusion. - -Because of this feature, it is good programming practice to not use mutable -objects as default values. Instead, use ``None`` as the default value and -inside the function, check if the parameter is ``None`` and create a new -list/dictionary/whatever if it is. For example, don't write:: - - def foo(mydict={}): - ... - -but:: - - def foo(mydict=None): - if mydict is None: - mydict = {} # create a new dict for local namespace - -This feature can be useful. When you have a function that's time-consuming to -compute, a common technique is to cache the parameters and the resulting value -of each call to the function, and return the cached value if the same value is -requested again. This is called "memoizing", and can be implemented like this:: - - # Callers can only provide two parameters and optionally pass _cache by keyword - def expensive(arg1, arg2, *, _cache={}): - if (arg1, arg2) in _cache: - return _cache[(arg1, arg2)] - - # Calculate the value - result = ... expensive computation ... - _cache[(arg1, arg2)] = result # Store result in the cache - return result - -You could use a global variable containing a dictionary instead of the default -value; it's a matter of taste. - - -How can I pass optional or keyword parameters from one function to another? ---------------------------------------------------------------------------- - -Collect the arguments using the ``*`` and ``**`` specifiers in the function's -parameter list; this gives you the positional arguments as a tuple and the -keyword arguments as a dictionary. You can then pass these arguments when -calling another function by using ``*`` and ``**``:: - - def f(x, *args, **kwargs): - ... - kwargs['width'] = '14.3c' - ... - g(x, *args, **kwargs) - - -.. index:: - single: argument; difference from parameter - single: parameter; difference from argument - -.. _faq-argument-vs-parameter: - -What is the difference between arguments and parameters? --------------------------------------------------------- - -:term:`Parameters <parameter>` are defined by the names that appear in a -function definition, whereas :term:`arguments <argument>` are the values -actually passed to a function when calling it. Parameters define what types of -arguments a function can accept. For example, given the function definition:: - - def func(foo, bar=None, **kwargs): - pass - -*foo*, *bar* and *kwargs* are parameters of ``func``. However, when calling -``func``, for example:: - - func(42, bar=314, extra=somevar) - -the values ``42``, ``314``, and ``somevar`` are arguments. - - -Why did changing list 'y' also change list 'x'? ------------------------------------------------- - -If you wrote code like:: - - >>> x = [] - >>> y = x - >>> y.append(10) - >>> y - [10] - >>> x - [10] - -you might be wondering why appending an element to ``y`` changed ``x`` too. - -There are two factors that produce this result: - -1) Variables are simply names that refer to objects. Doing ``y = x`` doesn't - create a copy of the list -- it creates a new variable ``y`` that refers to - the same object ``x`` refers to. This means that there is only one object - (the list), and both ``x`` and ``y`` refer to it. -2) Lists are :term:`mutable`, which means that you can change their content. - -After the call to :meth:`~list.append`, the content of the mutable object has -changed from ``[]`` to ``[10]``. Since both the variables refer to the same -object, using either name accesses the modified value ``[10]``. - -If we instead assign an immutable object to ``x``:: - - >>> x = 5 # ints are immutable - >>> y = x - >>> x = x + 1 # 5 can't be mutated, we are creating a new object here - >>> x - 6 - >>> y - 5 - -we can see that in this case ``x`` and ``y`` are not equal anymore. This is -because integers are :term:`immutable`, and when we do ``x = x + 1`` we are not -mutating the int ``5`` by incrementing its value; instead, we are creating a -new object (the int ``6``) and assigning it to ``x`` (that is, changing which -object ``x`` refers to). After this assignment we have two objects (the ints -``6`` and ``5``) and two variables that refer to them (``x`` now refers to -``6`` but ``y`` still refers to ``5``). - -Some operations (for example ``y.append(10)`` and ``y.sort()``) mutate the -object, whereas superficially similar operations (for example ``y = y + [10]`` -and ``sorted(y)``) create a new object. In general in Python (and in all cases -in the standard library) a method that mutates an object will return ``None`` -to help avoid getting the two types of operations confused. So if you -mistakenly write ``y.sort()`` thinking it will give you a sorted copy of ``y``, -you'll instead end up with ``None``, which will likely cause your program to -generate an easily diagnosed error. - -However, there is one class of operations where the same operation sometimes -has different behaviors with different types: the augmented assignment -operators. For example, ``+=`` mutates lists but not tuples or ints (``a_list -+= [1, 2, 3]`` is equivalent to ``a_list.extend([1, 2, 3])`` and mutates -``a_list``, whereas ``some_tuple += (1, 2, 3)`` and ``some_int += 1`` create -new objects). - -In other words: - -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables - that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, - but operations that transform that value into a new value always return a new - object. - -If you want to know if two variables refer to the same object or not, you can -use the :keyword:`is` operator, or the built-in function :func:`id`. - - -How do I write a function with output parameters (call by reference)? ---------------------------------------------------------------------- - -Remember that arguments are passed by assignment in Python. Since assignment -just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the -desired effect in a number of ways. - -1) By returning a tuple of the results:: - - def func2(a, b): - a = 'new-value' # a and b are local names - b = b + 1 # assigned to new objects - return a, b # return new values - - x, y = 'old-value', 99 - x, y = func2(x, y) - print(x, y) # output: new-value 100 - - This is almost always the clearest solution. - -2) By using global variables. This isn't thread-safe, and is not recommended. - -3) By passing a mutable (changeable in-place) object:: - - def func1(a): - a[0] = 'new-value' # 'a' references a mutable list - a[1] = a[1] + 1 # changes a shared object - - args = ['old-value', 99] - func1(args) - print(args[0], args[1]) # output: new-value 100 - -4) By passing in a dictionary that gets mutated:: - - def func3(args): - args['a'] = 'new-value' # args is a mutable dictionary - args['b'] = args['b'] + 1 # change it in-place - - args = {'a': 'old-value', 'b': 99} - func3(args) - print(args['a'], args['b']) - -5) Or bundle up values in a class instance:: - - class callByRef: - def __init__(self, **args): - for (key, value) in args.items(): - setattr(self, key, value) - - def func4(args): - args.a = 'new-value' # args is a mutable callByRef - args.b = args.b + 1 # change object in-place - - args = callByRef(a='old-value', b=99) - func4(args) - print(args.a, args.b) - - - There's almost never a good reason to get this complicated. - -Your best choice is to return a tuple containing the multiple results. - - -How do you make a higher order function in Python? --------------------------------------------------- - -You have two choices: you can use nested scopes or you can use callable objects. -For example, suppose you wanted to define ``linear(a,b)`` which returns a -function ``f(x)`` that computes the value ``a*x+b``. Using nested scopes:: - - def linear(a, b): - def result(x): - return a * x + b - return result - -Or using a callable object:: - - class linear: - - def __init__(self, a, b): - self.a, self.b = a, b - - def __call__(self, x): - return self.a * x + self.b - -In both cases, :: - - taxes = linear(0.3, 2) - -gives a callable object where ``taxes(10e6) == 0.3 * 10e6 + 2``. - -The callable object approach has the disadvantage that it is a bit slower and -results in slightly longer code. However, note that a collection of callables -can share their signature via inheritance:: - - class exponential(linear): - # __init__ inherited - def __call__(self, x): - return self.a * (x ** self.b) - -Object can encapsulate state for several methods:: - - class counter: - - value = 0 - - def set(self, x): - self.value = x - - def up(self): - self.value = self.value + 1 - - def down(self): - self.value = self.value - 1 - - count = counter() - inc, dec, reset = count.up, count.down, count.set - -Here ``inc()``, ``dec()`` and ``reset()`` act like functions which share the -same counting variable. - - -How do I copy an object in Python? ----------------------------------- - -In general, try :func:`copy.copy` or :func:`copy.deepcopy` for the general case. -Not all objects can be copied, but most can. - -Some objects can be copied more easily. Dictionaries have a :meth:`~dict.copy` -method:: - - newdict = olddict.copy() - -Sequences can be copied by slicing:: - - new_l = l[:] - - -How can I find the methods or attributes of an object? ------------------------------------------------------- - -For an instance x of a user-defined class, ``dir(x)`` returns an alphabetized -list of the names containing the instance attributes and methods and attributes -defined by its class. - - -How can my code discover the name of an object? ------------------------------------------------ - -Generally speaking, it can't, because objects don't really have names. -Essentially, assignment always binds a name to a value; The same is true of -``def`` and ``class`` statements, but in that case the value is a -callable. Consider the following code:: - - >>> class A: - ... pass - ... - >>> B = A - >>> a = B() - >>> b = a - >>> print(b) - <__main__.A object at 0x16D07CC> - >>> print(a) - <__main__.A object at 0x16D07CC> - -Arguably the class has a name: even though it is bound to two names and invoked -through the name B the created instance is still reported as an instance of -class A. However, it is impossible to say whether the instance's name is a or -b, since both names are bound to the same value. - -Generally speaking it should not be necessary for your code to "know the names" -of particular values. Unless you are deliberately writing introspective -programs, this is usually an indication that a change of approach might be -beneficial. - -In comp.lang.python, Fredrik Lundh once gave an excellent analogy in answer to -this question: - - The same way as you get the name of that cat you found on your porch: the cat - (object) itself cannot tell you its name, and it doesn't really care -- so - the only way to find out what it's called is to ask all your neighbours - (namespaces) if it's their cat (object)... - - ....and don't be surprised if you'll find that it's known by many names, or - no name at all! - - -What's up with the comma operator's precedence? ------------------------------------------------ - -Comma is not an operator in Python. Consider this session:: - - >>> "a" in "b", "a" - (False, 'a') - -Since the comma is not an operator, but a separator between expressions the -above is evaluated as if you had entered:: - - ("a" in "b"), "a" - -not:: - - "a" in ("b", "a") - -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. - - -Is there an equivalent of C's "?:" ternary operator? ----------------------------------------------------- - -Yes, there is. The syntax is as follows:: - - [on_true] if [expression] else [on_false] - - x, y = 50, 25 - small = x if x < y else y - -Before this syntax was introduced in Python 2.5, a common idiom was to use -logical operators:: - - [expression] and [on_true] or [on_false] - -However, this idiom is unsafe, as it can give wrong results when *on_true* -has a false boolean value. Therefore, it is always better to use -the ``... if ... else ...`` form. - - -Is it possible to write obfuscated one-liners in Python? --------------------------------------------------------- - -Yes. Usually this is done by nesting :keyword:`lambda` within -:keyword:`lambda`. See the following three examples, due to Ulf Bartelt:: - - from functools import reduce - - # Primes < 1000 - print(list(filter(None,map(lambda y:y*reduce(lambda x,y:x*y!=0, - map(lambda x,y=y:y%x,range(2,int(pow(y,0.5)+1))),1),range(2,1000))))) - - # First 10 Fibonacci numbers - print(list(map(lambda x,f=lambda x,f:(f(x-1,f)+f(x-2,f)) if x>1 else 1: - f(x,f), range(10)))) - - # Mandelbrot set - print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y, - Iu=Iu,Io=Io,Ru=Ru,Ro=Ro,Sy=Sy,L=lambda yc,Iu=Iu,Io=Io,Ru=Ru,Ro=Ro,i=IM, - Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro, - i=i,Sx=Sx,F=lambda xc,yc,x,y,k,f=lambda xc,yc,x,y,k,f:(k<=0)or (x*x+y*y - >=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr( - 64+F(Ru+x*(Ro-Ru)/Sx,yc,0,0,i)),range(Sx))):L(Iu+y*(Io-Iu)/Sy),range(Sy - ))))(-2.1, 0.7, -1.2, 1.2, 30, 80, 24)) - # \___ ___/ \___ ___/ | | |__ lines on screen - # V V | |______ columns on screen - # | | |__________ maximum of "iterations" - # | |_________________ range on y axis - # |____________________________ range on x axis - -Don't try this at home, kids! - - -Numbers and strings -=================== - -How do I specify hexadecimal and octal integers? ------------------------------------------------- - -To specify an octal digit, precede the octal value with a zero, and then a lower -or uppercase "o". For example, to set the variable "a" to the octal value "10" -(8 in decimal), type:: - - >>> a = 0o10 - >>> a - 8 - -Hexadecimal is just as easy. Simply precede the hexadecimal number with a zero, -and then a lower or uppercase "x". Hexadecimal digits can be specified in lower -or uppercase. For example, in the Python interpreter:: - - >>> a = 0xa5 - >>> a - 165 - >>> b = 0XB2 - >>> b - 178 - - -Why does -22 // 10 return -3? ------------------------------ - -It's primarily driven by the desire that ``i % j`` have the same sign as ``j``. -If you want that, and also want:: - - i == (i // j) * j + (i % j) - -then integer division has to return the floor. C also requires that identity to -hold, and then compilers that truncate ``i // j`` need to make ``i % j`` have -the same sign as ``i``. - -There are few real use cases for ``i % j`` when ``j`` is negative. When ``j`` -is positive, there are many, and in virtually all of them it's more useful for -``i % j`` to be ``>= 0``. If the clock says 10 now, what did it say 200 hours -ago? ``-190 % 12 == 2`` is useful; ``-190 % 12 == -10`` is a bug waiting to -bite. - - -How do I convert a string to a number? --------------------------------------- - -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, -e.g. ``float('144') == 144.0``. - -By default, these interpret the number as decimal, so that ``int('0144') == -144`` and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, base)`` takes -the base to convert from as a second optional argument, so ``int('0x144', 16) == -324``. If the base is specified as 0, the number is interpreted using Python's -rules: a leading '0o' indicates octal, and '0x' indicates a hex number. - -Do not use the built-in function :func:`eval` if all you need is to convert -strings to numbers. :func:`eval` will be significantly slower and it presents a -security risk: someone could pass you a Python expression that might have -unwanted side effects. For example, someone could pass -``__import__('os').system("rm -rf $HOME")`` which would erase your home -directory. - -:func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow -leading '0' in a decimal number (except '0'). - - -How do I convert a number to a string? --------------------------------------- - -To convert, e.g., the number 144 to the string '144', use the built-in type -constructor :func:`str`. If you want a hexadecimal or octal representation, use -the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields -``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. - - -How do I modify a string in place? ----------------------------------- - -You can't, because strings are immutable. In most situations, you should -simply construct a new string from the various parts you want to assemble -it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` -module:: - - >>> import io - >>> s = "Hello, world" - >>> sio = io.StringIO(s) - >>> sio.getvalue() - 'Hello, world' - >>> sio.seek(7) - 7 - >>> sio.write("there!") - 6 - >>> sio.getvalue() - 'Hello, there!' - - >>> import array - >>> a = array.array('u', s) - >>> print(a) - array('u', 'Hello, world') - >>> a[0] = 'y' - >>> print(a) - array('u', 'yello, world') - >>> a.tounicode() - 'yello, world' - - -How do I use strings to call functions/methods? ------------------------------------------------ - -There are various techniques. - -* The best is to use a dictionary that maps strings to functions. The primary - advantage of this technique is that the strings do not need to match the names - of the functions. This is also the primary technique used to emulate a case - construct:: - - def a(): - pass - - def b(): - pass - - dispatch = {'go': a, 'stop': b} # Note lack of parens for funcs - - dispatch[get_input()]() # Note trailing parens to call function - -* Use the built-in function :func:`getattr`:: - - import foo - getattr(foo, 'bar')() - - Note that :func:`getattr` works on any object, including classes, class - instances, modules, and so on. - - This is used in several places in the standard library, like this:: - - class Foo: - def do_foo(self): - ... - - def do_bar(self): - ... - - f = getattr(foo_instance, 'do_' + opname) - f() - - -* Use :func:`locals` or :func:`eval` to resolve the function name:: - - def myFunc(): - print("hello") - - fname = "myFunc" - - f = locals()[fname] - f() - - f = eval(fname) - f() - - Note: Using :func:`eval` is slow and dangerous. If you don't have absolute - control over the contents of the string, someone could pass a string that - resulted in an arbitrary function being executed. - -Is there an equivalent to Perl's chomp() for removing trailing newlines from strings? -------------------------------------------------------------------------------------- - -You can use ``S.rstrip("\r\n")`` to remove all occurrences of any line -terminator from the end of the string ``S`` without removing other trailing -whitespace. If the string ``S`` represents more than one line, with several -empty lines at the end, the line terminators for all the blank lines will -be removed:: - - >>> lines = ("line 1 \r\n" - ... "\r\n" - ... "\r\n") - >>> lines.rstrip("\n\r") - 'line 1 ' - -Since this is typically only desired when reading text one line at a time, using -``S.rstrip()`` this way works well. - - -Is there a scanf() or sscanf() equivalent? ------------------------------------------- - -Not as such. - -For simple input parsing, the easiest approach is usually to split the line into -whitespace-delimited words using the :meth:`~str.split` method of string objects -and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. ``split()`` supports an optional "sep" parameter which is useful -if the line uses something other than whitespace as a separator. - -For more complicated input parsing, regular expressions are more powerful -than C's :c:func:`sscanf` and better suited for the task. - - -What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? -------------------------------------------------------------------- - -See the :ref:`unicode-howto`. - - -Performance -=========== - -My program is too slow. How do I speed it up? ---------------------------------------------- - -That's a tough one, in general. First, here are a list of things to -remember before diving further: - -* Performance characteristics vary across Python implementations. This FAQ - focusses on :term:`CPython`. -* Behaviour can vary across operating systems, especially when talking about - I/O or multi-threading. -* You should always find the hot spots in your program *before* attempting to - optimize any code (see the :mod:`profile` module). -* Writing benchmark scripts will allow you to iterate quickly when searching - for improvements (see the :mod:`timeit` module). -* It is highly recommended to have good code coverage (through unit testing - or any other technique) before potentially introducing regressions hidden - in sophisticated optimizations. - -That being said, there are many tricks to speed up Python code. Here are -some general principles which go a long way towards reaching acceptable -performance levels: - -* Making your algorithms faster (or changing to faster ones) can yield - much larger benefits than trying to sprinkle micro-optimization tricks - all over your code. - -* Use the right data structures. Study documentation for the :ref:`bltin-types` - and the :mod:`collections` module. - -* When the standard library provides a primitive for doing something, it is - likely (although not guaranteed) to be faster than any alternative you - may come up with. This is doubly true for primitives written in C, such - as builtins and some extension types. For example, be sure to use - either the :meth:`list.sort` built-in method or the related :func:`sorted` - function to do sorting (and see the :ref:`sortinghowto` for examples - of moderately advanced usage). - -* Abstractions tend to create indirections and force the interpreter to work - more. If the levels of indirection outweigh the amount of useful work - done, your program will be slower. You should avoid excessive abstraction, - especially under the form of tiny functions or methods (which are also often - detrimental to readability). - -If you have reached the limit of what pure Python can allow, there are tools -to take you further away. For example, `Cython <http://cython.org>`_ can -compile a slightly modified version of Python code into a C extension, and -can be used on many different platforms. Cython can take advantage of -compilation (and optional type annotations) to make your code significantly -faster than when interpreted. If you are confident in your C programming -skills, you can also :ref:`write a C extension module <extending-index>` -yourself. - -.. seealso:: - The wiki page devoted to `performance tips - <https://wiki.python.org/moin/PythonSpeed/PerformanceTips>`_. - -.. _efficient_string_concatenation: - -What is the most efficient way to concatenate many strings together? --------------------------------------------------------------------- - -:class:`str` and :class:`bytes` objects are immutable, therefore concatenating -many strings together is inefficient as each concatenation creates a new -object. In the general case, the total runtime cost is quadratic in the -total string length. - -To accumulate many :class:`str` objects, the recommended idiom is to place -them into a list and call :meth:`str.join` at the end:: - - chunks = [] - for s in my_strings: - chunks.append(s) - result = ''.join(chunks) - -(another reasonably efficient idiom is to use :class:`io.StringIO`) - -To accumulate many :class:`bytes` objects, the recommended idiom is to extend -a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: - - result = bytearray() - for b in my_bytes_objects: - result += b - - -Sequences (Tuples/Lists) -======================== - -How do I convert between tuples and lists? ------------------------------------------- - -The type constructor ``tuple(seq)`` converts any sequence (actually, any -iterable) into a tuple with the same items in the same order. - -For example, ``tuple([1, 2, 3])`` yields ``(1, 2, 3)`` and ``tuple('abc')`` -yields ``('a', 'b', 'c')``. If the argument is a tuple, it does not make a copy -but returns the same object, so it is cheap to call :func:`tuple` when you -aren't sure that an object is already a tuple. - -The type constructor ``list(seq)`` converts any sequence or iterable into a list -with the same items in the same order. For example, ``list((1, 2, 3))`` yields -``[1, 2, 3]`` and ``list('abc')`` yields ``['a', 'b', 'c']``. If the argument -is a list, it makes a copy just like ``seq[:]`` would. - - -What's a negative index? ------------------------- - -Python sequences are indexed with positive numbers and negative numbers. For -positive numbers 0 is the first index 1 is the second index and so forth. For -negative indices -1 is the last index and -2 is the penultimate (next to last) -index and so forth. Think of ``seq[-n]`` as the same as ``seq[len(seq)-n]``. - -Using negative indices can be very convenient. For example ``S[:-1]`` is all of -the string except for its last character, which is useful for removing the -trailing newline from a string. - - -How do I iterate over a sequence in reverse order? --------------------------------------------------- - -Use the :func:`reversed` built-in function, which is new in Python 2.4:: - - for x in reversed(sequence): - ... # do something with x ... - -This won't touch your original sequence, but build a new copy with reversed -order to iterate over. - -With Python 2.3, you can use an extended slice syntax:: - - for x in sequence[::-1]: - ... # do something with x ... - - -How do you remove duplicates from a list? ------------------------------------------ - -See the Python Cookbook for a long discussion of many ways to do this: - - https://code.activestate.com/recipes/52560/ - -If you don't mind reordering the list, sort it and then scan from the end of the -list, deleting duplicates as you go:: - - if mylist: - mylist.sort() - last = mylist[-1] - for i in range(len(mylist)-2, -1, -1): - if last == mylist[i]: - del mylist[i] - else: - last = mylist[i] - -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: - - mylist = list(set(mylist)) - -This converts the list into a set, thereby removing duplicates, and then back -into a list. - - -How do you make an array in Python? ------------------------------------ - -Use a list:: - - ["this", 1, "is", "an", "array"] - -Lists are equivalent to C or Pascal arrays in their time complexity; the primary -difference is that a Python list can contain objects of many different types. - -The ``array`` module also provides methods for creating arrays of fixed types -with compact representations, but they are slower to index than lists. Also -note that the Numeric extensions and others define array-like structures with -various characteristics as well. - -To get Lisp-style linked lists, you can emulate cons cells using tuples:: - - lisp_list = ("like", ("this", ("example", None) ) ) - -If mutability is desired, you could use lists instead of tuples. Here the -analogue of lisp car is ``lisp_list[0]`` and the analogue of cdr is -``lisp_list[1]``. Only do this if you're sure you really need to, because it's -usually a lot slower than using Python lists. - - -.. _faq-multidimensional-list: - -How do I create a multidimensional list? ----------------------------------------- - -You probably tried to make a multidimensional array like this:: - - >>> A = [[None] * 2] * 3 - -This looks correct if you print it: - -.. testsetup:: - - A = [[None] * 2] * 3 - -.. doctest:: - - >>> A - [[None, None], [None, None], [None, None]] - -But when you assign a value, it shows up in multiple places: - -.. testsetup:: - - A = [[None] * 2] * 3 - -.. doctest:: - - >>> A[0][0] = 5 - >>> A - [[5, None], [5, None], [5, None]] - -The reason is that replicating a list with ``*`` doesn't create copies, it only -creates references to the existing objects. The ``*3`` creates a list -containing 3 references to the same list of length two. Changes to one row will -show in all rows, which is almost certainly not what you want. - -The suggested approach is to create a list of the desired length first and then -fill in each element with a newly created list:: - - A = [None] * 3 - for i in range(3): - A[i] = [None] * 2 - -This generates a list containing 3 different lists of length two. You can also -use a list comprehension:: - - w, h = 2, 3 - A = [[None] * w for i in range(h)] - -Or, you can use an extension that provides a matrix datatype; `NumPy -<http://www.numpy.org/>`_ is the best known. - - -How do I apply a method to a sequence of objects? -------------------------------------------------- - -Use a list comprehension:: - - result = [obj.method() for obj in mylist] - -.. _faq-augmented-assignment-tuple-error: - -Why does a_tuple[i] += ['item'] raise an exception when the addition works? ---------------------------------------------------------------------------- - -This is because of a combination of the fact that augmented assignment -operators are *assignment* operators, and the difference between mutable and -immutable objects in Python. - -This discussion applies in general when augmented assignment operators are -applied to elements of a tuple that point to mutable objects, but we'll use -a ``list`` and ``+=`` as our exemplar. - -If you wrote:: - - >>> a_tuple = (1, 2) - >>> a_tuple[0] += 1 - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The reason for the exception should be immediately clear: ``1`` is added to the -object ``a_tuple[0]`` points to (``1``), producing the result object, ``2``, -but when we attempt to assign the result of the computation, ``2``, to element -``0`` of the tuple, we get an error because we can't change what an element of -a tuple points to. - -Under the covers, what this augmented assignment statement is doing is -approximately this:: - - >>> result = a_tuple[0] + 1 - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -It is the assignment part of the operation that produces the error, since a -tuple is immutable. - -When you write something like:: - - >>> a_tuple = (['foo'], 'bar') - >>> a_tuple[0] += ['item'] - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The exception is a bit more surprising, and even more surprising is the fact -that even though there was an error, the append worked:: - - >>> a_tuple[0] - ['foo', 'item'] - -To see why this happens, you need to know that (a) if an object implements an -``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment -is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for ``list.extend``:: - - >>> a_list = [] - >>> a_list += [1] - >>> a_list - [1] - -This is equivalent to:: - - >>> result = a_list.__iadd__([1]) - >>> a_list = result - -The object pointed to by a_list has been mutated, and the pointer to the -mutated object is assigned back to ``a_list``. The end result of the -assignment is a no-op, since it is a pointer to the same object that ``a_list`` -was previously pointing to, but the assignment still happens. - -Thus, in our tuple example what is happening is equivalent to:: - - >>> result = a_tuple[0].__iadd__(['item']) - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The ``__iadd__`` succeeds, and thus the list is extended, but even though -``result`` points to the same object that ``a_tuple[0]`` already points to, -that final assignment still results in an error, because tuples are immutable. - - -Dictionaries -============ - -How can I get a dictionary to store and display its keys in a consistent order? -------------------------------------------------------------------------------- - -Use :class:`collections.OrderedDict`. - -I want to do a complicated sort: can you do a Schwartzian Transform in Python? ------------------------------------------------------------------------------- - -The technique, attributed to Randal Schwartz of the Perl community, sorts the -elements of a list by a metric which maps each element to its "sort value". In -Python, use the ``key`` argument for the :meth:`list.sort` method:: - - Isorted = L[:] - Isorted.sort(key=lambda s: int(s[10:15])) - - -How can I sort one list by values from another list? ----------------------------------------------------- - -Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: - - >>> list1 = ["what", "I'm", "sorting", "by"] - >>> list2 = ["something", "else", "to", "sort"] - >>> pairs = zip(list1, list2) - >>> pairs = sorted(pairs) - >>> pairs - [("I'm", 'else'), ('by', 'sort'), ('sorting', 'to'), ('what', 'something')] - >>> result = [x[1] for x in pairs] - >>> result - ['else', 'sort', 'to', 'something'] - - -An alternative for the last step is:: - - >>> result = [] - >>> for p in pairs: result.append(p[1]) - -If you find this more legible, you might prefer to use this instead of the final -list comprehension. However, it is almost twice as slow for long lists. Why? -First, the ``append()`` operation has to reallocate memory, and while it uses -some tricks to avoid doing that each time, it still has to do it occasionally, -and that costs quite a bit. Second, the expression "result.append" requires an -extra attribute lookup, and third, there's a speed reduction from having to make -all those function calls. - - -Objects -======= - -What is a class? ----------------- - -A class is the particular object type created by executing a class statement. -Class objects are used as templates to create instance objects, which embody -both the data (attributes) and code (methods) specific to a datatype. - -A class can be based on one or more other classes, called its base class(es). It -then inherits the attributes and methods of its base classes. This allows an -object model to be successively refined by inheritance. You might have a -generic ``Mailbox`` class that provides basic accessor methods for a mailbox, -and subclasses such as ``MboxMailbox``, ``MaildirMailbox``, ``OutlookMailbox`` -that handle various specific mailbox formats. - - -What is a method? ------------------ - -A method is a function on some object ``x`` that you normally call as -``x.name(arguments...)``. Methods are defined as functions inside the class -definition:: - - class C: - def meth(self, arg): - return arg * 2 + self.attribute - - -What is self? -------------- - -Self is merely a conventional name for the first argument of a method. A method -defined as ``meth(self, a, b, c)`` should be called as ``x.meth(a, b, c)`` for -some instance ``x`` of the class in which the definition occurs; the called -method will think it is called as ``meth(x, a, b, c)``. - -See also :ref:`why-self`. - - -How do I check if an object is an instance of a given class or of a subclass of it? ------------------------------------------------------------------------------------ - -Use the built-in function ``isinstance(obj, cls)``. You can check if an object -is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. -``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. - -Note that most programs do not use :func:`isinstance` on user-defined classes -very often. If you are developing the classes yourself, a more proper -object-oriented style is to define methods on the classes that encapsulate a -particular behaviour, instead of checking the object's class and doing a -different thing based on what class it is. For example, if you have a function -that does something:: - - def search(obj): - if isinstance(obj, Mailbox): - ... # code to search a mailbox - elif isinstance(obj, Document): - ... # code to search a document - elif ... - -A better approach is to define a ``search()`` method on all the classes and just -call it:: - - class Mailbox: - def search(self): - ... # code to search a mailbox - - class Document: - def search(self): - ... # code to search a document - - obj.search() - - -What is delegation? -------------------- - -Delegation is an object oriented technique (also called a design pattern). -Let's say you have an object ``x`` and want to change the behaviour of just one -of its methods. You can create a new class that provides a new implementation -of the method you're interested in changing and delegates all other methods to -the corresponding method of ``x``. - -Python programmers can easily implement delegation. For example, the following -class implements a class that behaves like a file but converts all written data -to uppercase:: - - class UpperOut: - - def __init__(self, outfile): - self._outfile = outfile - - def write(self, s): - self._outfile.write(s.upper()) - - def __getattr__(self, name): - return getattr(self._outfile, name) - -Here the ``UpperOut`` class redefines the ``write()`` method to convert the -argument string to uppercase before calling the underlying -``self.__outfile.write()`` method. All other methods are delegated to the -underlying ``self.__outfile`` object. The delegation is accomplished via the -``__getattr__`` method; consult :ref:`the language reference <attribute-access>` -for more information about controlling attribute access. - -Note that for more general cases delegation can get trickier. When attributes -must be set as well as retrieved, the class must define a :meth:`__setattr__` -method too, and it must do so carefully. The basic implementation of -:meth:`__setattr__` is roughly equivalent to the following:: - - class X: - ... - def __setattr__(self, name, value): - self.__dict__[name] = value - ... - -Most :meth:`__setattr__` implementations must modify ``self.__dict__`` to store -local state for self without causing an infinite recursion. - - -How do I call a method defined in a base class from a derived class that overrides it? --------------------------------------------------------------------------------------- - -Use the built-in :func:`super` function:: - - class Derived(Base): - def meth(self): - super(Derived, self).meth() - -For version prior to 3.0, you may be using classic classes: For a class -definition such as ``class Derived(Base): ...`` you can call method ``meth()`` -defined in ``Base`` (or one of ``Base``'s base classes) as ``Base.meth(self, -arguments...)``. Here, ``Base.meth`` is an unbound method, so you need to -provide the ``self`` argument. - - -How can I organize my code to make it easier to change the base class? ----------------------------------------------------------------------- - -You could define an alias for the base class, assign the real base class to it -before your class definition, and use the alias throughout your class. Then all -you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability -of resources) which base class to use. Example:: - - BaseAlias = <real base class> - - class Derived(BaseAlias): - def meth(self): - BaseAlias.meth(self) - ... - - -How do I create static class data and static class methods? ------------------------------------------------------------ - -Both static data and static methods (in the sense of C++ or Java) are supported -in Python. - -For static data, simply define a class attribute. To assign a new value to the -attribute, you have to explicitly use the class name in the assignment:: - - class C: - count = 0 # number of times C.__init__ called - - def __init__(self): - C.count = C.count + 1 - - def getcount(self): - return C.count # or return self.count - -``c.count`` also refers to ``C.count`` for any ``c`` such that ``isinstance(c, -C)`` holds, unless overridden by ``c`` itself or by some class on the base-class -search path from ``c.__class__`` back to ``C``. - -Caution: within a method of C, an assignment like ``self.count = 42`` creates a -new and unrelated instance named "count" in ``self``'s own dict. Rebinding of a -class-static data name must always specify the class whether inside a method or -not:: - - C.count = 314 - -Static methods are possible:: - - class C: - @staticmethod - def static(arg1, arg2, arg3): - # No 'self' parameter! - ... - -However, a far more straightforward way to get the effect of a static method is -via a simple module-level function:: - - def getcount(): - return C.count - -If your code is structured so as to define one class (or tightly related class -hierarchy) per module, this supplies the desired encapsulation. - - -How can I overload constructors (or methods) in Python? -------------------------------------------------------- - -This answer actually applies to all methods, but the question usually comes up -first in the context of constructors. - -In C++ you'd write - -.. code-block:: c - - class C { - C() { cout << "No arguments\n"; } - C(int i) { cout << "Argument is " << i << "\n"; } - } - -In Python you have to write a single constructor that catches all cases using -default arguments. For example:: - - class C: - def __init__(self, i=None): - if i is None: - print("No arguments") - else: - print("Argument is", i) - -This is not entirely equivalent, but close enough in practice. - -You could also try a variable-length argument list, e.g. :: - - def __init__(self, *args): - ... - -The same approach works for all method definitions. - - -I try to use __spam and I get an error about _SomeClassName__spam. ------------------------------------------------------------------- - -Variable names with double leading underscores are "mangled" to provide a simple -but effective way to define class private variables. Any identifier of the form -``__spam`` (at least two leading underscores, at most one trailing underscore) -is textually replaced with ``_classname__spam``, where ``classname`` is the -current class name with any leading underscores stripped. - -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. - - -My class defines __del__ but it is not called when I delete the object. ------------------------------------------------------------------------ - -There are several possible reasons for this. - -The del statement does not necessarily call :meth:`__del__` -- it simply -decrements the object's reference count, and if this reaches zero -:meth:`__del__` is called. - -If your data structures contain circular links (e.g. a tree where each child has -a parent reference and each parent has a list of children) the reference counts -will never go back to zero. Once in a while Python runs an algorithm to detect -such cycles, but the garbage collector might run some time after the last -reference to your data structure vanishes, so your :meth:`__del__` method may be -called at an inconvenient and random time. This is inconvenient if you're trying -to reproduce a problem. Worse, the order in which object's :meth:`__del__` -methods are executed is arbitrary. You can run :func:`gc.collect` to force a -collection, but there *are* pathological cases where objects will never be -collected. - -Despite the cycle collector, it's still a good idea to define an explicit -``close()`` method on objects to be called whenever you're done with them. The -``close()`` method can then remove attributes that refer to subobjects. Don't -call :meth:`__del__` directly -- :meth:`__del__` should call ``close()`` and -``close()`` should make sure that it can be called more than once for the same -object. - -Another way to avoid cyclical references is to use the :mod:`weakref` module, -which allows you to point to objects without incrementing their reference count. -Tree data structures, for instance, should use weak references for their parent -and sibling references (if they need them!). - -.. XXX relevant for Python 3? - - If the object has ever been a local variable in a function that caught an - expression in an except clause, chances are that a reference to the object - still exists in that function's stack frame as contained in the stack trace. - Normally, calling :func:`sys.exc_clear` will take care of this by clearing - the last recorded exception. - -Finally, if your :meth:`__del__` method raises an exception, a warning message -is printed to :data:`sys.stderr`. - - -How do I get a list of all instances of a given class? ------------------------------------------------------- - -Python does not keep track of all instances of a class (or of a built-in type). -You can program the class's constructor to keep track of all instances by -keeping a list of weak references to each instance. - - -Why does the result of ``id()`` appear to be not unique? --------------------------------------------------------- - -The :func:`id` builtin returns an integer that is guaranteed to be unique during -the lifetime of the object. Since in CPython, this is the object's memory -address, it happens frequently that after an object is deleted from memory, the -next freshly created object is allocated at the same position in memory. This -is illustrated by this example: - ->>> id(1000) # doctest: +SKIP -13901272 ->>> id(2000) # doctest: +SKIP -13901272 - -The two ids belong to different integer objects that are created before, and -deleted immediately after execution of the ``id()`` call. To be sure that -objects whose id you want to examine are still alive, create another reference -to the object: - ->>> a = 1000; b = 2000 ->>> id(a) # doctest: +SKIP -13901272 ->>> id(b) # doctest: +SKIP -13891296 - - -Modules -======= - -How do I create a .pyc file? ----------------------------- - -When a module is imported for the first time (or when the source file has -changed since the current compiled file was created) a ``.pyc`` file containing -the compiled code should be created in a ``__pycache__`` subdirectory of the -directory containing the ``.py`` file. The ``.pyc`` file will have a -filename that starts with the same name as the ``.py`` file, and ends with -``.pyc``, with a middle component that depends on the particular ``python`` -binary that created it. (See :pep:`3147` for details.) - -One reason that a ``.pyc`` file may not be created is a permissions problem -with the directory containing the source file, meaning that the ``__pycache__`` -subdirectory cannot be created. This can happen, for example, if you develop as -one user but run as another, such as if you are testing with a web server. - -Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, -creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` -subdirectory and write the compiled module to that subdirectory. - -Running Python on a top level script is not considered an import and no -``.pyc`` will be created. For example, if you have a top-level module -``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by -typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for -``xyz`` because ``xyz`` is imported, but no ``.pyc`` file will be created for -``foo`` since ``foo.py`` isn't being imported. - -If you need to create a ``.pyc`` file for ``foo`` -- that is, to create a -``.pyc`` file for a module that is not imported -- you can, using the -:mod:`py_compile` and :mod:`compileall` modules. - -The :mod:`py_compile` module can manually compile any module. One way is to use -the ``compile()`` function in that module interactively:: - - >>> import py_compile - >>> py_compile.compile('foo.py') # doctest: +SKIP - -This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same -location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). - -You can also automatically compile all files in a directory or directories using -the :mod:`compileall` module. You can do it from the shell prompt by running -``compileall.py`` and providing the path of a directory containing Python files -to compile:: - - python -m compileall . - - -How do I find the current module name? --------------------------------------- - -A module can find out its own module name by looking at the predefined global -variable ``__name__``. If this has the value ``'__main__'``, the program is -running as a script. Many modules that are usually used by importing them also -provide a command-line interface or a self-test, and only execute this code -after checking ``__name__``:: - - def main(): - print('Running test...') - ... - - if __name__ == '__main__': - main() - - -How can I have modules that mutually import each other? -------------------------------------------------------- - -Suppose you have the following modules: - -foo.py:: - - from bar import bar_var - foo_var = 1 - -bar.py:: - - from foo import foo_var - bar_var = 2 - -The problem is that the interpreter will perform the following steps: - -* main imports foo -* Empty globals for foo are created -* foo is compiled and starts executing -* foo imports bar -* Empty globals for bar are created -* bar is compiled and starts executing -* bar imports foo (which is a no-op since there already is a module named foo) -* bar.foo_var = foo.foo_var - -The last step fails, because Python isn't done with interpreting ``foo`` yet and -the global symbol dictionary for ``foo`` is still empty. - -The same thing happens when you use ``import foo``, and then try to access -``foo.foo_var`` in global code. - -There are (at least) three possible workarounds for this problem. - -Guido van Rossum recommends avoiding all uses of ``from <module> import ...``, -and placing all code inside functions. Initializations of global variables and -class variables should use constants or built-in functions only. This means -everything from an imported module is referenced as ``<module>.<name>``. - -Jim Roskind suggests performing steps in the following order in each module: - -* exports (globals, functions, and classes that don't need imported base - classes) -* ``import`` statements -* active code (including globals that are initialized from imported values). - -van Rossum doesn't like this approach much because the imports appear in a -strange place, but it does work. - -Matthias Urlichs recommends restructuring your code so that the recursive import -is not necessary in the first place. - -These solutions are not mutually exclusive. - - -__import__('x.y.z') returns <module 'x'>; how do I get z? ---------------------------------------------------------- - -Consider using the convenience function :func:`~importlib.import_module` from -:mod:`importlib` instead:: - - z = importlib.import_module('x.y.z') - - -When I edit an imported module and reimport it, the changes don't show up. Why does this happen? -------------------------------------------------------------------------------------------------- - -For reasons of efficiency as well as consistency, Python only reads the module -file on the first time a module is imported. If it didn't, in a program -consisting of many modules where each one imports the same basic module, the -basic module would be parsed and re-parsed many times. To force re-reading of a -changed module, do this:: - - import importlib - import modname - importlib.reload(modname) - -Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: - - from modname import some_objects - -will continue to work with the old version of the imported objects. If the -module contains class definitions, existing class instances will *not* be -updated to use the new class definition. This can result in the following -paradoxical behaviour: - - >>> import importlib - >>> import cls - >>> c = cls.C() # Create an instance of C - >>> importlib.reload(cls) - <module 'cls' from 'cls.py'> - >>> isinstance(c, cls.C) # isinstance is false?!? - False - -The nature of the problem is made clear if you print out the "identity" of the -class objects: - - >>> hex(id(c.__class__)) - '0x7352a0' - >>> hex(id(cls.C)) - '0x4198d0' diff --git a/third_party/python/Doc/faq/python-video-icon.png b/third_party/python/Doc/faq/python-video-icon.png deleted file mode 100644 index 4de54b403..000000000 Binary files a/third_party/python/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/third_party/python/Doc/faq/windows.rst b/third_party/python/Doc/faq/windows.rst deleted file mode 100644 index 2b44f6569..000000000 --- a/third_party/python/Doc/faq/windows.rst +++ /dev/null @@ -1,282 +0,0 @@ -:tocdepth: 2 - -.. highlightlang:: none - -.. _windows-faq: - -===================== -Python on Windows FAQ -===================== - -.. only:: html - - .. contents:: - -.. XXX need review for Python 3. - XXX need review for Windows Vista/Seven? - - -How do I run a Python program under Windows? --------------------------------------------- - -This is not necessarily a straightforward question. If you are already familiar -with running programs from the Windows command line then everything will seem -obvious; otherwise, you might need a little more guidance. - -Unless you use some sort of integrated development environment, you will end up -*typing* Windows commands into what is variously referred to as a "DOS window" -or "Command prompt window". Usually you can create such a window from your -search bar by searching for ``cmd``. You should be able to recognize -when you have started such a window because you will see a Windows "command -prompt", which usually looks like this: - -.. code-block:: doscon - - C:\> - -The letter may be different, and there might be other things after it, so you -might just as easily see something like: - -.. code-block:: doscon - - D:\YourName\Projects\Python> - -depending on how your computer has been set up and what else you have recently -done with it. Once you have started such a window, you are well on the way to -running Python programs. - -You need to realize that your Python scripts have to be processed by another -program called the Python *interpreter*. The interpreter reads your script, -compiles it into bytecodes, and then executes the bytecodes to run your -program. So, how do you arrange for the interpreter to handle your Python? - -First, you need to make sure that your command window recognises the word -"py" as an instruction to start the interpreter. If you have opened a -command window, you should try entering the command ``py`` and hitting -return: - -.. code-block:: doscon - - C:\Users\YourName> py - -You should then see something like: - -.. code-block:: pycon - - Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32 - Type "help", "copyright", "credits" or "license" for more information. - >>> - -You have started the interpreter in "interactive mode". That means you can enter -Python statements or expressions interactively and have them executed or -evaluated while you wait. This is one of Python's strongest features. Check it -by entering a few expressions of your choice and seeing the results: - -.. code-block:: pycon - - >>> print("Hello") - Hello - >>> "Hello" * 3 - 'HelloHelloHello' - -Many people use the interactive mode as a convenient yet highly programmable -calculator. When you want to end your interactive Python session, -call the :func:`exit` function or hold the :kbd:`Ctrl` key down -while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get -back to your Windows command prompt. - -You may also find that you have a Start-menu entry such as :menuselection:`Start ---> Programs --> Python 3.x --> Python (command line)` that results in you -seeing the ``>>>`` prompt in a new window. If so, the window will disappear -after you call the :func:`exit` function or enter the :kbd:`Ctrl-Z` -character; Windows is running a single "python" -command in the window, and closes it when you terminate the interpreter. - -Now that we know the ``py`` command is recognized, you can give your -Python script to it. You'll have to give either an absolute or a -relative path to the Python script. Let's say your Python script is -located in your desktop and is named ``hello.py``, and your command -prompt is nicely opened in your home directory so you're seeing something -similar to:: - - C:\Users\YourName> - -So now you'll ask the ``py`` command to give your script to Python by -typing ``py`` followed by your script path:: - - - C:\Users\YourName> py Desktop\hello.py - hello - -How do I make Python scripts executable? ----------------------------------------- - -On Windows, the standard Python installer already associates the .py -extension with a file type (Python.File) and gives that file type an open -command that runs the interpreter (``D:\Program Files\Python\python.exe "%1" -%*``). This is enough to make scripts executable from the command prompt as -'foo.py'. If you'd rather be able to execute the script by simple typing 'foo' -with no extension you need to add .py to the PATHEXT environment variable. - -Why does Python sometimes take so long to start? ------------------------------------------------- - -Usually Python starts very quickly on Windows, but occasionally there are bug -reports that Python suddenly begins to take a long time to start up. This is -made even more puzzling because Python will work fine on other Windows systems -which appear to be configured identically. - -The problem may be caused by a misconfiguration of virus checking software on -the problem machine. Some virus scanners have been known to introduce startup -overhead of two orders of magnitude when the scanner is configured to monitor -all reads from the filesystem. Try checking the configuration of virus scanning -software on your systems to ensure that they are indeed configured identically. -McAfee, when configured to scan all file system read activity, is a particular -offender. - - -How do I make an executable from a Python script? -------------------------------------------------- - -See http://cx-freeze.sourceforge.net/ for a distutils extension that allows you -to create console and GUI executables from Python code. -`py2exe <http://www.py2exe.org/>`_, the most popular extension for building -Python 2.x-based executables, does not yet support Python 3 but a version that -does is in development. - - -Is a ``*.pyd`` file the same as a DLL? --------------------------------------- - -Yes, .pyd files are dll's, but there are a few differences. If you have a DLL -named ``foo.pyd``, then it must have a function ``PyInit_foo()``. You can then -write Python "import foo", and Python will search for foo.pyd (as well as -foo.py, foo.pyc) and if it finds it, will attempt to call ``PyInit_foo()`` to -initialize it. You do not link your .exe with foo.lib, as that would cause -Windows to require the DLL to be present. - -Note that the search path for foo.pyd is PYTHONPATH, not the same as the path -that Windows uses to search for foo.dll. Also, foo.pyd need not be present to -run your program, whereas if you linked your program with a dll, the dll is -required. Of course, foo.pyd is required if you want to say ``import foo``. In -a DLL, linkage is declared in the source code with ``__declspec(dllexport)``. -In a .pyd, linkage is defined in a list of available functions. - - -How can I embed Python into a Windows application? --------------------------------------------------- - -Embedding the Python interpreter in a Windows app can be summarized as follows: - -1. Do _not_ build Python into your .exe file directly. On Windows, Python must - be a DLL to handle importing modules that are themselves DLL's. (This is the - first key undocumented fact.) Instead, link to :file:`python{NN}.dll`; it is - typically installed in ``C:\Windows\System``. *NN* is the Python version, a - number such as "33" for Python 3.3. - - You can link to Python in two different ways. Load-time linking means - linking against :file:`python{NN}.lib`, while run-time linking means linking - against :file:`python{NN}.dll`. (General note: :file:`python{NN}.lib` is the - so-called "import lib" corresponding to :file:`python{NN}.dll`. It merely - defines symbols for the linker.) - - Run-time linking greatly simplifies link options; everything happens at run - time. Your code must load :file:`python{NN}.dll` using the Windows - ``LoadLibraryEx()`` routine. The code must also use access routines and data - in :file:`python{NN}.dll` (that is, Python's C API's) using pointers obtained - by the Windows ``GetProcAddress()`` routine. Macros can make using these - pointers transparent to any C code that calls routines in Python's C API. - - Borland note: convert :file:`python{NN}.lib` to OMF format using Coff2Omf.exe - first. - - .. XXX what about static linking? - -2. If you use SWIG, it is easy to create a Python "extension module" that will - make the app's data and methods available to Python. SWIG will handle just - about all the grungy details for you. The result is C code that you link - *into* your .exe file (!) You do _not_ have to create a DLL file, and this - also simplifies linking. - -3. SWIG will create an init function (a C function) whose name depends on the - name of the extension module. For example, if the name of the module is leo, - the init function will be called initleo(). If you use SWIG shadow classes, - as you should, the init function will be called initleoc(). This initializes - a mostly hidden helper class used by the shadow class. - - The reason you can link the C code in step 2 into your .exe file is that - calling the initialization function is equivalent to importing the module - into Python! (This is the second key undocumented fact.) - -4. In short, you can use the following code to initialize the Python interpreter - with your extension module. - - .. code-block:: c - - #include "python.h" - ... - Py_Initialize(); // Initialize Python. - initmyAppc(); // Initialize (import) the helper class. - PyRun_SimpleString("import myApp"); // Import the shadow class. - -5. There are two problems with Python's C API which will become apparent if you - use a compiler other than MSVC, the compiler used to build pythonNN.dll. - - Problem 1: The so-called "Very High Level" functions that take FILE * - arguments will not work in a multi-compiler environment because each - compiler's notion of a struct FILE will be different. From an implementation - standpoint these are very _low_ level functions. - - Problem 2: SWIG generates the following code when generating wrappers to void - functions: - - .. code-block:: c - - Py_INCREF(Py_None); - _resultobj = Py_None; - return _resultobj; - - Alas, Py_None is a macro that expands to a reference to a complex data - structure called _Py_NoneStruct inside pythonNN.dll. Again, this code will - fail in a mult-compiler environment. Replace such code by: - - .. code-block:: c - - return Py_BuildValue(""); - - It may be possible to use SWIG's ``%typemap`` command to make the change - automatically, though I have not been able to get this to work (I'm a - complete SWIG newbie). - -6. Using a Python shell script to put up a Python interpreter window from inside - your Windows app is not a good idea; the resulting window will be independent - of your app's windowing system. Rather, you (or the wxPythonWindow class) - should create a "native" interpreter window. It is easy to connect that - window to the Python interpreter. You can redirect Python's i/o to _any_ - object that supports read and write, so all you need is a Python object - (defined in your extension module) that contains read() and write() methods. - -How do I keep editors from inserting tabs into my Python source? ----------------------------------------------------------------- - -The FAQ does not recommend using tabs, and the Python style guide, :pep:`8`, -recommends 4 spaces for distributed Python code; this is also the Emacs -python-mode default. - -Under any editor, mixing tabs and spaces is a bad idea. MSVC is no different in -this respect, and is easily configured to use spaces: Take :menuselection:`Tools ---> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent -size" to 4, and select the "Insert spaces" radio button. - -Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs -and spaces are causing problems in leading whitespace. -You may also run the :mod:`tabnanny` module to check a directory tree -in batch mode. - - -How do I check for a keypress without blocking? ------------------------------------------------ - -Use the msvcrt module. This is a standard Windows-specific extension module. -It defines a function ``kbhit()`` which checks whether a keyboard hit is -present, and ``getch()`` which gets one character without echoing it. diff --git a/third_party/python/Doc/glossary.rst b/third_party/python/Doc/glossary.rst deleted file mode 100644 index 2aab8bf6c..000000000 --- a/third_party/python/Doc/glossary.rst +++ /dev/null @@ -1,1122 +0,0 @@ -.. _glossary: - -******** -Glossary -******** - -.. if you add new entries, keep the alphabetical sorting! - -.. glossary:: - - ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. - - ``...`` - The default Python prompt of the interactive shell when entering code for - an indented code block, when within a pair of matching left and right - delimiters (parentheses, square brackets, curly braces or triple quotes), - or after specifying a decorator. - - 2to3 - A tool that tries to convert Python 2.x code to Python 3.x code by - handling most of the incompatibilities which can be detected by parsing the - source and traversing the parse tree. - - 2to3 is available in the standard library as :mod:`lib2to3`; a standalone - entry point is provided as :file:`Tools/scripts/2to3`. See - :ref:`2to3-reference`. - - abstract base class - Abstract base classes complement :term:`duck-typing` by - providing a way to define interfaces when other techniques like - :func:`hasattr` would be clumsy or subtly wrong (for example with - :ref:`magic methods <special-lookup>`). ABCs introduce virtual - subclasses, which are classes that don't inherit from a class but are - still recognized by :func:`isinstance` and :func:`issubclass`; see the - :mod:`abc` module documentation. Python comes with many built-in ABCs for - data structures (in the :mod:`collections.abc` module), numbers (in the - :mod:`numbers` module), streams (in the :mod:`io` module), import finders - and loaders (in the :mod:`importlib.abc` module). You can create your own - ABCs with the :mod:`abc` module. - - annotation - A label associated with a variable, a class - attribute or a function parameter or return value, - used by convention as a :term:`type hint`. - - Annotations of local variables cannot be accessed at runtime, but - annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. - - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. - - argument - A value passed to a :term:`function` (or :term:`method`) when calling the - function. There are two kinds of argument: - - * :dfn:`keyword argument`: an argument preceded by an identifier (e.g. - ``name=``) in a function call or passed as a value in a dictionary - preceded by ``**``. For example, ``3`` and ``5`` are both keyword - arguments in the following calls to :func:`complex`:: - - complex(real=3, imag=5) - complex(**{'real': 3, 'imag': 5}) - - * :dfn:`positional argument`: an argument that is not a keyword argument. - Positional arguments can appear at the beginning of an argument list - and/or be passed as elements of an :term:`iterable` preceded by ``*``. - For example, ``3`` and ``5`` are both positional arguments in the - following calls:: - - complex(3, 5) - complex(*(3, 5)) - - Arguments are assigned to the named local variables in a function body. - See the :ref:`calls` section for the rules governing this assignment. - Syntactically, any expression can be used to represent an argument; the - evaluated value is assigned to the local variable. - - See also the :term:`parameter` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, and :pep:`362`. - - asynchronous context manager - An object which controls the environment seen in an - :keyword:`async with` statement by defining :meth:`__aenter__` and - :meth:`__aexit__` methods. Introduced by :pep:`492`. - - asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. - - An asynchronous generator function may contain :keyword:`await` - expressions as well as :keyword:`async for`, and :keyword:`async with` - statements. - - asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. - - This is an :term:`asynchronous iterator` which when called using the - :meth:`__anext__` method returns an awaitable object which will execute - the body of the asynchronous generator function until the next - :keyword:`yield` expression. - - Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending - try-statements). When the *asynchronous generator iterator* effectively - resumes with another awaitable returned by :meth:`__anext__`, it - picks up where it left off. See :pep:`492` and :pep:`525`. - - asynchronous iterable - An object, that can be used in an :keyword:`async for` statement. - Must return an :term:`asynchronous iterator` from its - :meth:`__aiter__` method. Introduced by :pep:`492`. - - asynchronous iterator - An object that implements the :meth:`__aiter__` and :meth:`__anext__` - methods. ``__anext__`` must return an :term:`awaitable` object. - :keyword:`async for` resolves the awaitables returned by an asynchronous - iterator's :meth:`__anext__` method until it raises a - :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. - - attribute - A value associated with an object which is referenced by name using - dotted expressions. For example, if an object *o* has an attribute - *a* it would be referenced as *o.a*. - - awaitable - An object that can be used in an :keyword:`await` expression. Can be - a :term:`coroutine` or an object with an :meth:`__await__` method. - See also :pep:`492`. - - BDFL - Benevolent Dictator For Life, a.k.a. `Guido van Rossum - <https://www.python.org/~guido/>`_, Python's creator. - - binary file - A :term:`file object` able to read and write - :term:`bytes-like objects <bytes-like object>`. - Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. - - See also :term:`text file` for a file object able to read and write - :class:`str` objects. - - bytes-like object - An object that supports the :ref:`bufferobjects` and can - export a C-:term:`contiguous` buffer. This includes all :class:`bytes`, - :class:`bytearray`, and :class:`array.array` objects, as well as many - common :class:`memoryview` objects. Bytes-like objects can - be used for various operations that work with binary data; these include - compression, saving to a binary file, and sending over a socket. - - Some operations need the binary data to be mutable. The documentation - often refers to these as "read-write bytes-like objects". Example - mutable buffer objects include :class:`bytearray` and a - :class:`memoryview` of a :class:`bytearray`. - Other operations require the binary data to be stored in - immutable objects ("read-only bytes-like objects"); examples - of these include :class:`bytes` and a :class:`memoryview` - of a :class:`bytes` object. - - bytecode - Python source code is compiled into bytecode, the internal representation - of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` files so that executing the same file is - faster the second time (recompilation from source to bytecode can be - avoided). This "intermediate language" is said to run on a - :term:`virtual machine` that executes the machine code corresponding to - each bytecode. Do note that bytecodes are not expected to work between - different Python virtual machines, nor to be stable between Python - releases. - - A list of bytecode instructions can be found in the documentation for - :ref:`the dis module <bytecodes>`. - - class - A template for creating user-defined objects. Class definitions - normally contain method definitions which operate on instances of the - class. - - class variable - A variable defined in a class and intended to be modified only at - class level (i.e., not in an instance of the class). - - coercion - The implicit conversion of an instance of one type to another during an - operation which involves two arguments of the same type. For example, - ``int(3.15)`` converts the floating point number to the integer ``3``, but - in ``3+4.5``, each argument is of a different type (one int, one float), - and both must be converted to the same type before they can be added or it - will raise a ``TypeError``. Without coercion, all arguments of even - compatible types would have to be normalized to the same value by the - programmer, e.g., ``float(3)+4.5`` rather than just ``3+4.5``. - - complex number - An extension of the familiar real number system in which all numbers are - expressed as a sum of a real part and an imaginary part. Imaginary - numbers are real multiples of the imaginary unit (the square root of - ``-1``), often written ``i`` in mathematics or ``j`` in - engineering. Python has built-in support for complex numbers, which are - written with this latter notation; the imaginary part is written with a - ``j`` suffix, e.g., ``3+1j``. To get access to complex equivalents of the - :mod:`math` module, use :mod:`cmath`. Use of complex numbers is a fairly - advanced mathematical feature. If you're not aware of a need for them, - it's almost certain you can safely ignore them. - - context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`__enter__` and :meth:`__exit__` methods. - See :pep:`343`. - - contiguous - .. index:: C-contiguous, Fortran contiguous - - A buffer is considered contiguous exactly if it is either - *C-contiguous* or *Fortran contiguous*. Zero-dimensional buffers are - C and Fortran contiguous. In one-dimensional arrays, the items - must be laid out in memory next to each other, in order of - increasing indexes starting from zero. In multidimensional - C-contiguous arrays, the last index varies the fastest when - visiting items in order of memory address. However, in - Fortran contiguous arrays, the first index varies the fastest. - - coroutine - Coroutines is a more generalized form of subroutines. Subroutines are - entered at one point and exited at another point. Coroutines can be - entered, exited, and resumed at many different points. They can be - implemented with the :keyword:`async def` statement. See also - :pep:`492`. - - coroutine function - A function which returns a :term:`coroutine` object. A coroutine - function may be defined with the :keyword:`async def` statement, - and may contain :keyword:`await`, :keyword:`async for`, and - :keyword:`async with` keywords. These were introduced - by :pep:`492`. - - CPython - The canonical implementation of the Python programming language, as - distributed on `python.org <https://www.python.org>`_. The term "CPython" - is used when necessary to distinguish this implementation from others - such as Jython or IronPython. - - decorator - A function returning another function, usually applied as a function - transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. - - The decorator syntax is merely syntactic sugar, the following two - function definitions are semantically equivalent:: - - def f(...): - ... - f = staticmethod(f) - - @staticmethod - def f(...): - ... - - The same concept exists for classes, but is less commonly used there. See - the documentation for :ref:`function definitions <function>` and - :ref:`class definitions <class>` for more about decorators. - - descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special - binding behavior is triggered upon attribute lookup. Normally, using - *a.b* to get, set or delete an attribute looks up the object named *b* in - the class dictionary for *a*, but if *b* is a descriptor, the respective - descriptor method gets called. Understanding descriptors is a key to a - deep understanding of Python because they are the basis for many features - including functions, methods, properties, class methods, static methods, - and reference to super classes. - - For more information about descriptors' methods, see :ref:`descriptors`. - - dictionary - An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. - Called a hash in Perl. - - dictionary view - The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and - :meth:`dict.items` are called dictionary views. They provide a dynamic - view on the dictionary’s entries, which means that when the dictionary - changes, the view reflects these changes. To force the - dictionary view to become a full list use ``list(dictview)``. See - :ref:`dict-views`. - - docstring - A string literal which appears as the first expression in a class, - function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute - of the enclosing class, function or module. Since it is available via - introspection, it is the canonical place for documentation of the - object. - - duck-typing - A programming style which does not look at an object's type to determine - if it has the right interface; instead, the method or attribute is simply - called or used ("If it looks like a duck and quacks like a duck, it - must be a duck.") By emphasizing interfaces rather than specific types, - well-designed code improves its flexibility by allowing polymorphic - substitution. Duck-typing avoids tests using :func:`type` or - :func:`isinstance`. (Note, however, that duck-typing can be complemented - with :term:`abstract base classes <abstract base class>`.) Instead, it - typically employs :func:`hasattr` tests or :term:`EAFP` programming. - - EAFP - Easier to ask for forgiveness than permission. This common Python coding - style assumes the existence of valid keys or attributes and catches - exceptions if the assumption proves false. This clean and fast style is - characterized by the presence of many :keyword:`try` and :keyword:`except` - statements. The technique contrasts with the :term:`LBYL` style - common to many other languages such as C. - - expression - A piece of syntax which can be evaluated to some value. In other words, - an expression is an accumulation of expression elements like literals, - names, attribute access, operators or function calls which all return a - value. In contrast to many other languages, not all language constructs - are expressions. There are also :term:`statement`\s which cannot be used - as expressions, such as :keyword:`if`. Assignments are also statements, - not expressions. - - extension module - A module written in C or C++, using Python's C API to interact with the - core and with user code. - - f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called - "f-strings" which is short for - :ref:`formatted string literals <f-strings>`. See also :pep:`498`. - - file object - An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending - on the way it was created, a file object can mediate access to a real - on-disk file or to another type of storage or communication device - (for example standard input/output, in-memory buffers, sockets, pipes, - etc.). File objects are also called :dfn:`file-like objects` or - :dfn:`streams`. - - There are actually three categories of file objects: raw - :term:`binary files <binary file>`, buffered - :term:`binary files <binary file>` and :term:`text files <text file>`. - Their interfaces are defined in the :mod:`io` module. The canonical - way to create a file object is by using the :func:`open` function. - - file-like object - A synonym for :term:`file object`. - - finder - An object that tries to find the :term:`loader` for a module that is - being imported. - - Since Python 3.3, there are two types of finder: :term:`meta path finders - <meta path finder>` for use with :data:`sys.meta_path`, and :term:`path - entry finders <path entry finder>` for use with :data:`sys.path_hooks`. - - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. - - floor division - Mathematical division that rounds down to nearest integer. The floor - division operator is ``//``. For example, the expression ``11 // 4`` - evaluates to ``2`` in contrast to the ``2.75`` returned by float true - division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` - rounded *downward*. See :pep:`238`. - - function - A series of statements which returns some value to a caller. It can also - be passed zero or more :term:`arguments <argument>` which may be used in - the execution of the body. See also :term:`parameter`, :term:`method`, - and the :ref:`function` section. - - function annotation - An :term:`annotation` of a function parameter or return value. - - Function annotations are usually used for - :term:`type hints <type hint>`: for example, this function is expected to take two - :class:`int` arguments and is also expected to have an :class:`int` - return value:: - - def sum_two_numbers(a: int, b: int) -> int: - return a + b - - Function annotation syntax is explained in section :ref:`function`. - - See :term:`variable annotation` and :pep:`484`, - which describe this functionality. - - __future__ - A pseudo-module which programmers can use to enable new language features - which are not compatible with the current interpreter. - - By importing the :mod:`__future__` module and evaluating its variables, - you can see when a new feature was first added to the language and when it - becomes the default:: - - >>> import __future__ - >>> __future__.division - _Feature((2, 2, 0, 'alpha', 2), (3, 0, 0, 'alpha', 0), 8192) - - garbage collection - The process of freeing memory when it is not used anymore. Python - performs garbage collection via reference counting and a cyclic garbage - collector that is able to detect and break reference cycles. The - garbage collector can be controlled using the :mod:`gc` module. - - .. index:: single: generator - - generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. - - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. - - generator iterator - An object created by a :term:`generator` function. - - Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). - - .. index:: single: generator expression - - generator expression - An expression that returns an iterator. It looks like a normal expression - followed by a :keyword:`for` expression defining a loop variable, range, - and an optional :keyword:`if` expression. The combined expression - generates values for an enclosing function:: - - >>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81 - 285 - - generic function - A function composed of multiple functions implementing the same operation - for different types. Which implementation should be used during a call is - determined by the dispatch algorithm. - - See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. - - - GIL - See :term:`global interpreter lock`. - - global interpreter lock - The mechanism used by the :term:`CPython` interpreter to assure that - only one thread executes Python :term:`bytecode` at a time. - This simplifies the CPython implementation by making the object model - (including critical built-in types such as :class:`dict`) implicitly - safe against concurrent access. Locking the entire interpreter - makes it easier for the interpreter to be multi-threaded, at the - expense of much of the parallelism afforded by multi-processor - machines. - - However, some extension modules, either standard or third-party, - are designed so as to release the GIL when doing computationally-intensive - tasks such as compression or hashing. Also, the GIL is always released - when doing I/O. - - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. - - hashable - An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which - compare equal must have the same hash value. - - Hashability makes an object usable as a dictionary key and a set member, - because these data structures use the hash value internally. - - All of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not. Objects which are - instances of user-defined classes are hashable by default. They all - compare unequal (except with themselves), and their hash value is derived - from their :func:`id`. - - IDLE - An Integrated Development Environment for Python. IDLE is a basic editor - and interpreter environment which ships with the standard distribution of - Python. - - immutable - An object with a fixed value. Immutable objects include numbers, strings and - tuples. Such an object cannot be altered. A new object has to - be created if a different value has to be stored. They play an important - role in places where a constant hash value is needed, for example as a key - in a dictionary. - - import path - A list of locations (or :term:`path entries <path entry>`) that are - searched by the :term:`path based finder` for modules to import. During - import, this list of locations usually comes from :data:`sys.path`, but - for subpackages it may also come from the parent package's ``__path__`` - attribute. - - importing - The process by which Python code in one module is made available to - Python code in another module. - - importer - An object that both finds and loads a module; both a - :term:`finder` and :term:`loader` object. - - interactive - Python has an interactive interpreter which means you can enter - statements and expressions at the interpreter prompt, immediately - execute them and see their results. Just launch ``python`` with no - arguments (possibly by selecting it from your computer's main - menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). - - interpreted - Python is an interpreted language, as opposed to a compiled one, - though the distinction can be blurry because of the presence of the - bytecode compiler. This means that source files can be run directly - without explicitly creating an executable which is then run. - Interpreted languages typically have a shorter development/debug cycle - than compiled ones, though their programs generally also run more - slowly. See also :term:`interactive`. - - interpreter shutdown - When asked to shut down, the Python interpreter enters a special phase - where it gradually releases all allocated resources, such as modules - and various critical internal structures. It also makes several calls - to the :term:`garbage collector <garbage collection>`. This can trigger - the execution of code in user-defined destructors or weakref callbacks. - Code executed during the shutdown phase can encounter various - exceptions as the resources it relies on may not function anymore - (common examples are library modules or the warnings machinery). - - The main reason for interpreter shutdown is that the ``__main__`` module - or the script being run has finished executing. - - iterable - An object capable of returning its members one at a time. Examples of - iterables include all sequence types (such as :class:`list`, :class:`str`, - and :class:`tuple`) and some non-sequence types like :class:`dict`, - :term:`file objects <file object>`, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`__getitem__` method - that implements :term:`Sequence` semantics. - - Iterables can be - used in a :keyword:`for` loop and in many other places where a sequence is - needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed - as an argument to the built-in function :func:`iter`, it returns an - iterator for the object. This iterator is good for one pass over the set - of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` - statement does that automatically for you, creating a temporary unnamed - variable to hold the iterator for the duration of the loop. See also - :term:`iterator`, :term:`sequence`, and :term:`generator`. - - iterator - An object representing a stream of data. Repeated calls to the iterator's - :meth:`~iterator.__next__` method (or passing it to the built-in function - :func:`next`) return successive items in the stream. When no more data - are available a :exc:`StopIteration` exception is raised instead. At this - point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator - object itself so every iterator is also iterable and may be used in most - places where other iterables are accepted. One notable exception is code - which attempts multiple iteration passes. A container object (such as a - :class:`list`) produces a fresh new iterator each time you pass it to the - :func:`iter` function or use it in a :keyword:`for` loop. Attempting this - with an iterator will just return the same exhausted iterator object used - in the previous iteration pass, making it appear like an empty container. - - More information can be found in :ref:`typeiter`. - - key function - A key function or collation function is a callable that returns a value - used for sorting or ordering. For example, :func:`locale.strxfrm` is - used to produce a sort key that is aware of locale specific sort - conventions. - - A number of tools in Python accept key functions to control how elements - are ordered or grouped. They include :func:`min`, :func:`max`, - :func:`sorted`, :meth:`list.sort`, :func:`heapq.merge`, - :func:`heapq.nsmallest`, :func:`heapq.nlargest`, and - :func:`itertools.groupby`. - - There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive - sorts. Alternatively, a key function can be built from a - :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, - the :mod:`operator` module provides three key function constructors: - :func:`~operator.attrgetter`, :func:`~operator.itemgetter`, and - :func:`~operator.methodcaller`. See the :ref:`Sorting HOW TO - <sortinghowto>` for examples of how to create and use key functions. - - keyword argument - See :term:`argument`. - - lambda - An anonymous inline function consisting of a single :term:`expression` - which is evaluated when the function is called. The syntax to create - a lambda function is ``lambda [parameters]: expression`` - - LBYL - Look before you leap. This coding style explicitly tests for - pre-conditions before making calls or lookups. This style contrasts with - the :term:`EAFP` approach and is characterized by the presence of many - :keyword:`if` statements. - - In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another - thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. - - list - A built-in Python :term:`sequence`. Despite its name it is more akin - to an array in other languages than to a linked list since access to - elements is O(1). - - list comprehension - A compact way to process all or part of the elements in a sequence and - return a list with the results. ``result = ['{:#04x}'.format(x) for x in - range(256) if x % 2 == 0]`` generates a list of strings containing - even hex numbers (0x..) in the range from 0 to 255. The :keyword:`if` - clause is optional. If omitted, all elements in ``range(256)`` are - processed. - - loader - An object that loads a module. It must define a method named - :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. - - mapping - A container object that supports arbitrary key lookups and implements the - methods specified in the :class:`~collections.abc.Mapping` or - :class:`~collections.abc.MutableMapping` - :ref:`abstract base classes <collections-abstract-base-classes>`. Examples - include :class:`dict`, :class:`collections.defaultdict`, - :class:`collections.OrderedDict` and :class:`collections.Counter`. - - meta path finder - A :term:`finder` returned by a search of :data:`sys.meta_path`. Meta path - finders are related to, but different from :term:`path entry finders - <path entry finder>`. - - See :class:`importlib.abc.MetaPathFinder` for the methods that meta path - finders implement. - - metaclass - The class of a class. Class definitions create a class name, a class - dictionary, and a list of base classes. The metaclass is responsible for - taking those three arguments and creating the class. Most object oriented - programming languages provide a default implementation. What makes Python - special is that it is possible to create custom metaclasses. Most users - never need this tool, but when the need arises, metaclasses can provide - powerful, elegant solutions. They have been used for logging attribute - access, adding thread-safety, tracking object creation, implementing - singletons, and many other tasks. - - More information can be found in :ref:`metaclasses`. - - method - A function which is defined inside a class body. If called as an attribute - of an instance of that class, the method will get the instance object as - its first :term:`argument` (which is usually called ``self``). - See :term:`function` and :term:`nested scope`. - - method resolution order - Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - <https://www.python.org/download/releases/2.3/mro/>`_ for details of the - algorithm used by the Python interpreter since the 2.3 release. - - module - An object that serves as an organizational unit of Python code. Modules - have a namespace containing arbitrary Python objects. Modules are loaded - into Python by the process of :term:`importing`. - - See also :term:`package`. - - module spec - A namespace containing the import-related information used to load a - module. An instance of :class:`importlib.machinery.ModuleSpec`. - - MRO - See :term:`method resolution order`. - - mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. - - named tuple - Any tuple-like class whose indexable elements are also accessible using - named attributes (for example, :func:`time.localtime` returns a - tuple-like object where the *year* is accessible either with an - index such as ``t[0]`` or with a named attribute like ``t.tm_year``). - - A named tuple can be a built-in type such as :class:`time.struct_time`, - or it can be created with a regular class definition. A full featured - named tuple can also be created with the factory function - :func:`collections.namedtuple`. The latter approach automatically - provides extra features such as a self-documenting representation like - ``Employee(name='jones', title='programmer')``. - - namespace - The place where a variable is stored. Namespaces are implemented as - dictionaries. There are the local, global and built-in namespaces as well - as nested namespaces in objects (in methods). Namespaces support - modularity by preventing naming conflicts. For instance, the functions - :func:`builtins.open <.open>` and :func:`os.open` are distinguished by - their namespaces. Namespaces also aid readability and maintainability by - making it clear which module implements a function. For instance, writing - :func:`random.seed` or :func:`itertools.islice` makes it clear that those - functions are implemented by the :mod:`random` and :mod:`itertools` - modules, respectively. - - namespace package - A :pep:`420` :term:`package` which serves only as a container for - subpackages. Namespace packages may have no physical representation, - and specifically are not like a :term:`regular package` because they - have no ``__init__.py`` file. - - See also :term:`module`. - - nested scope - The ability to refer to a variable in an enclosing definition. For - instance, a function defined inside another function can refer to - variables in the outer function. Note that nested scopes by default work - only for reference and not for assignment. Local variables both read and - write in the innermost scope. Likewise, global variables read and write - to the global namespace. The :keyword:`nonlocal` allows writing to outer - scopes. - - new-style class - Old name for the flavor of classes now used for all class objects. In - earlier Python versions, only new-style classes could use Python's newer, - versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. - - object - Any data with state (attributes or value) and defined behavior - (methods). Also the ultimate base class of any :term:`new-style - class`. - - package - A Python :term:`module` which can contain submodules or recursively, - subpackages. Technically, a package is a Python module with an - ``__path__`` attribute. - - See also :term:`regular package` and :term:`namespace package`. - - parameter - A named entity in a :term:`function` (or method) definition that - specifies an :term:`argument` (or in some cases, arguments) that the - function can accept. There are five kinds of parameter: - - * :dfn:`positional-or-keyword`: specifies an argument that can be passed - either :term:`positionally <argument>` or as a :term:`keyword argument - <argument>`. This is the default kind of parameter, for example *foo* - and *bar* in the following:: - - def func(foo, bar=None): ... - - .. _positional-only_parameter: - - * :dfn:`positional-only`: specifies an argument that can be supplied only - by position. Python has no syntax for defining positional-only - parameters. However, some built-in functions have positional-only - parameters (e.g. :func:`abs`). - - .. _keyword-only_parameter: - - * :dfn:`keyword-only`: specifies an argument that can be supplied only - by keyword. Keyword-only parameters can be defined by including a - single var-positional parameter or bare ``*`` in the parameter list - of the function definition before them, for example *kw_only1* and - *kw_only2* in the following:: - - def func(arg, *, kw_only1, kw_only2): ... - - * :dfn:`var-positional`: specifies that an arbitrary sequence of - positional arguments can be provided (in addition to any positional - arguments already accepted by other parameters). Such a parameter can - be defined by prepending the parameter name with ``*``, for example - *args* in the following:: - - def func(*args, **kwargs): ... - - * :dfn:`var-keyword`: specifies that arbitrarily many keyword arguments - can be provided (in addition to any keyword arguments already accepted - by other parameters). Such a parameter can be defined by prepending - the parameter name with ``**``, for example *kwargs* in the example - above. - - Parameters can specify both optional and required arguments, as well as - default values for some optional arguments. - - See also the :term:`argument` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, the :class:`inspect.Parameter` class, the - :ref:`function` section, and :pep:`362`. - - path entry - A single location on the :term:`import path` which the :term:`path - based finder` consults to find modules for importing. - - path entry finder - A :term:`finder` returned by a callable on :data:`sys.path_hooks` - (i.e. a :term:`path entry hook`) which knows how to locate modules given - a :term:`path entry`. - - See :class:`importlib.abc.PathEntryFinder` for the methods that path entry - finders implement. - - path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path - entry finder` if it knows how to find modules on a specific :term:`path - entry`. - - path based finder - One of the default :term:`meta path finders <meta path finder>` which - searches an :term:`import path` for modules. - - path-like object - An object representing a file system path. A path-like object is either - a :class:`str` or :class:`bytes` object representing a path, or an object - implementing the :class:`os.PathLike` protocol. An object that supports - the :class:`os.PathLike` protocol can be converted to a :class:`str` or - :class:`bytes` file system path by calling the :func:`os.fspath` function; - :func:`os.fsdecode` and :func:`os.fsencode` can be used to guarantee a - :class:`str` or :class:`bytes` result instead, respectively. Introduced - by :pep:`519`. - - PEP - Python Enhancement Proposal. A PEP is a design document - providing information to the Python community, or describing a new - feature for Python or its processes or environment. PEPs should - provide a concise technical specification and a rationale for proposed - features. - - PEPs are intended to be the primary mechanisms for proposing major new - features, for collecting community input on an issue, and for documenting - the design decisions that have gone into Python. The PEP author is - responsible for building consensus within the community and documenting - dissenting opinions. - - See :pep:`1`. - - portion - A set of files in a single directory (possibly stored in a zip file) - that contribute to a namespace package, as defined in :pep:`420`. - - positional argument - See :term:`argument`. - - provisional API - A provisional API is one which has been deliberately excluded from - the standard library's backwards compatibility guarantees. While major - changes to such interfaces are not expected, as long as they are marked - provisional, backwards incompatible changes (up to and including removal - of the interface) may occur if deemed necessary by core developers. Such - changes will not be made gratuitously -- they will occur only if serious - fundamental flaws are uncovered that were missed prior to the inclusion - of the API. - - Even for provisional APIs, backwards incompatible changes are seen as - a "solution of last resort" - every attempt will still be made to find - a backwards compatible resolution to any identified problems. - - This process allows the standard library to continue to evolve over - time, without locking in problematic design errors for extended periods - of time. See :pep:`411` for more details. - - provisional package - See :term:`provisional API`. - - Python 3000 - Nickname for the Python 3.x release line (coined long ago when the - release of version 3 was something in the distant future.) This is also - abbreviated "Py3k". - - Pythonic - An idea or piece of code which closely follows the most common idioms - of the Python language, rather than implementing code using concepts - common to other languages. For example, a common idiom in Python is - to loop over all elements of an iterable using a :keyword:`for` - statement. Many other languages don't have this type of construct, so - people unfamiliar with Python sometimes use a numerical counter instead:: - - for i in range(len(food)): - print(food[i]) - - As opposed to the cleaner, Pythonic method:: - - for piece in food: - print(piece) - - qualified name - A dotted name showing the "path" from a module's global scope to a - class, function or method defined in that module, as defined in - :pep:`3155`. For top-level functions and classes, the qualified name - is the same as the object's name:: - - >>> class C: - ... class D: - ... def meth(self): - ... pass - ... - >>> C.__qualname__ - 'C' - >>> C.D.__qualname__ - 'C.D' - >>> C.D.meth.__qualname__ - 'C.D.meth' - - When used to refer to modules, the *fully qualified name* means the - entire dotted path to the module, including any parent packages, - e.g. ``email.mime.text``:: - - >>> import email.mime.text - >>> email.mime.text.__name__ - 'email.mime.text' - - reference count - The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is - generally not visible to Python code, but it is a key element of the - :term:`CPython` implementation. The :mod:`sys` module defines a - :func:`~sys.getrefcount` function that programmers can call to return the - reference count for a particular object. - - regular package - A traditional :term:`package`, such as a directory containing an - ``__init__.py`` file. - - See also :term:`namespace package`. - - __slots__ - A declaration inside a class that saves memory by pre-declaring space for - instance attributes and eliminating instance dictionaries. Though - popular, the technique is somewhat tricky to get right and is best - reserved for rare cases where there are large numbers of instances in a - memory-critical application. - - sequence - An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. - Some built-in sequence types are :class:`list`, :class:`str`, - :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`__getitem__` and :meth:`__len__`, but is considered a - mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. - - The :class:`collections.abc.Sequence` abstract base class - defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded - interface can be registered explicitly using - :func:`~abc.register`. - - single dispatch - A form of :term:`generic function` dispatch where the implementation is - chosen based on the type of a single argument. - - slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. - - special method - A method that is called implicitly by Python to execute a certain - operation on a type, such as addition. Such methods have names starting - and ending with double underscores. Special methods are documented in - :ref:`specialnames`. - - statement - A statement is part of a suite (a "block" of code). A statement is either - an :term:`expression` or one of several constructs with a keyword, such - as :keyword:`if`, :keyword:`while` or :keyword:`for`. - - struct sequence - A tuple with named elements. Struct sequences expose an interface similar - to :term:`named tuple` in that elements can be accessed either by - index or as an attribute. However, they do not have any of the named tuple - methods like :meth:`~collections.somenamedtuple._make` or - :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences - include :data:`sys.float_info` and the return value of :func:`os.stat`. - - text encoding - A codec which encodes Unicode strings to bytes. - - text file - A :term:`file object` able to read and write :class:`str` objects. - Often, a text file actually accesses a byte-oriented datastream - and handles the :term:`text encoding` automatically. - Examples of text files are files opened in text mode (``'r'`` or ``'w'``), - :data:`sys.stdin`, :data:`sys.stdout`, and instances of - :class:`io.StringIO`. - - See also :term:`binary file` for a file object able to read and write - :term:`bytes-like objects <bytes-like object>`. - - triple-quoted string - A string which is bound by three instances of either a quotation mark - (") or an apostrophe ('). While they don't provide any functionality - not available with single-quoted strings, they are useful for a number - of reasons. They allow you to include unescaped single and double - quotes within a string and they can span multiple lines without the - use of the continuation character, making them especially useful when - writing docstrings. - - type - The type of a Python object determines what kind of object it is; every - object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with - ``type(obj)``. - - type alias - A synonym for a type, created by assigning the type to an identifier. - - Type aliases are useful for simplifying :term:`type hints <type hint>`. - For example:: - - from typing import List, Tuple - - def remove_gray_shades( - colors: List[Tuple[int, int, int]]) -> List[Tuple[int, int, int]]: - pass - - could be made more readable like this:: - - from typing import List, Tuple - - Color = Tuple[int, int, int] - - def remove_gray_shades(colors: List[Color]) -> List[Color]: - pass - - See :mod:`typing` and :pep:`484`, which describe this functionality. - - type hint - An :term:`annotation` that specifies the expected type for a variable, a class - attribute, or a function parameter or return value. - - Type hints are optional and are not enforced by Python but - they are useful to static type analysis tools, and aid IDEs with code - completion and refactoring. - - Type hints of global variables, class attributes, and functions, - but not local variables, can be accessed using - :func:`typing.get_type_hints`. - - See :mod:`typing` and :pep:`484`, which describe this functionality. - - universal newlines - A manner of interpreting text streams in which all of the following are - recognized as ending a line: the Unix end-of-line convention ``'\n'``, - the Windows convention ``'\r\n'``, and the old Macintosh convention - ``'\r'``. See :pep:`278` and :pep:`3116`, as well as - :func:`bytes.splitlines` for an additional use. - - variable annotation - An :term:`annotation` of a variable or a class attribute. - - When annotating a variable or a class attribute, assignment is optional:: - - class C: - field: 'annotation' - - Variable annotations are usually used for - :term:`type hints <type hint>`: for example this variable is expected to take - :class:`int` values:: - - count: int = 0 - - Variable annotation syntax is explained in section :ref:`annassign`. - - See :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. - - virtual environment - A cooperatively isolated runtime environment that allows Python users - and applications to install and upgrade Python distribution packages - without interfering with the behaviour of other Python applications - running on the same system. - - See also :mod:`venv`. - - virtual machine - A computer defined entirely in software. Python's virtual machine - executes the :term:`bytecode` emitted by the bytecode compiler. - - Zen of Python - Listing of Python design principles and philosophies that are helpful in - understanding and using the language. The listing can be found by typing - "``import this``" at the interactive prompt. diff --git a/third_party/python/Doc/howto/argparse.rst b/third_party/python/Doc/howto/argparse.rst deleted file mode 100644 index e78a022b3..000000000 --- a/third_party/python/Doc/howto/argparse.rst +++ /dev/null @@ -1,765 +0,0 @@ -***************** -Argparse Tutorial -***************** - -:author: Tshepang Lekhonkhobe - -.. _argparse-tutorial: - -This tutorial is intended to be a gentle introduction to :mod:`argparse`, the -recommended command-line parsing module in the Python standard library. - -.. note:: - - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for :c:func:`getopt` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. - - -Concepts -======== - -Let's show the sort of functionality that we are going to explore in this -introductory tutorial by making use of the :command:`ls` command: - -.. code-block:: shell-session - - $ ls - cpython devguide prog.py pypy rm-unused-function.patch - $ ls pypy - ctypes_configure demo dotviewer include lib_pypy lib-python ... - $ ls -l - total 20 - drwxr-xr-x 19 wena wena 4096 Feb 18 18:51 cpython - drwxr-xr-x 4 wena wena 4096 Feb 8 12:04 devguide - -rwxr-xr-x 1 wena wena 535 Feb 19 00:05 prog.py - drwxr-xr-x 14 wena wena 4096 Feb 7 00:59 pypy - -rw-r--r-- 1 wena wena 741 Feb 18 01:01 rm-unused-function.patch - $ ls --help - Usage: ls [OPTION]... [FILE]... - List information about the FILEs (the current directory by default). - Sort entries alphabetically if none of -cftuvSUX nor --sort is specified. - ... - -A few concepts we can learn from the four commands: - -* The :command:`ls` command is useful when run without any options at all. It defaults - to displaying the contents of the current directory. - -* If we want beyond what it provides by default, we tell it a bit more. In - this case, we want it to display a different directory, ``pypy``. - What we did is specify what is known as a positional argument. It's named so - because the program should know what to do with the value, solely based on - where it appears on the command line. This concept is more relevant - to a command like :command:`cp`, whose most basic usage is ``cp SRC DEST``. - The first position is *what you want copied,* and the second - position is *where you want it copied to*. - -* Now, say we want to change behaviour of the program. In our example, - we display more info for each file instead of just showing the file names. - The ``-l`` in that case is known as an optional argument. - -* That's a snippet of the help text. It's very useful in that you can - come across a program you have never used before, and can figure out - how it works simply by reading its help text. - - -The basics -========== - -Let us start with a very simple example which does (almost) nothing:: - - import argparse - parser = argparse.ArgumentParser() - parser.parse_args() - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] - - optional arguments: - -h, --help show this help message and exit - $ python3 prog.py --verbose - usage: prog.py [-h] - prog.py: error: unrecognized arguments: --verbose - $ python3 prog.py foo - usage: prog.py [-h] - prog.py: error: unrecognized arguments: foo - -Here is what is happening: - -* Running the script without any options results in nothing displayed to - stdout. Not so useful. - -* The second one starts to display the usefulness of the :mod:`argparse` - module. We have done almost nothing, but already we get a nice help message. - -* The ``--help`` option, which can also be shortened to ``-h``, is the only - option we get for free (i.e. no need to specify it). Specifying anything - else results in an error. But even then, we do get a useful usage message, - also for free. - - -Introducing Positional arguments -================================ - -An example:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo") - args = parser.parse_args() - print(args.echo) - -And running the code: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] echo - prog.py: error: the following arguments are required: echo - $ python3 prog.py --help - usage: prog.py [-h] echo - - positional arguments: - echo - - optional arguments: - -h, --help show this help message and exit - $ python3 prog.py foo - foo - -Here is what's happening: - -* We've added the :meth:`add_argument` method, which is what we use to specify - which command-line options the program is willing to accept. In this case, - I've named it ``echo`` so that it's in line with its function. - -* Calling our program now requires us to specify an option. - -* The :meth:`parse_args` method actually returns some data from the - options specified, in this case, ``echo``. - -* The variable is some form of 'magic' that :mod:`argparse` performs for free - (i.e. no need to specify which variable that value is stored in). - You will also notice that its name matches the string argument given - to the method, ``echo``. - -Note however that, although the help display looks nice and all, it currently -is not as helpful as it can be. For example we see that we got ``echo`` as a -positional argument, but we don't know what it does, other than by guessing or -by reading the source code. So, let's make it a bit more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo", help="echo the string you use here") - args = parser.parse_args() - print(args.echo) - -And we get: - -.. code-block:: shell-session - - $ python3 prog.py -h - usage: prog.py [-h] echo - - positional arguments: - echo echo the string you use here - - optional arguments: - -h, --help show this help message and exit - -Now, how about doing something even more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number") - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 5, in <module> - print(args.square**2) - TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int' - -That didn't go so well. That's because :mod:`argparse` treats the options we -give it as strings, unless we tell it otherwise. So, let's tell -:mod:`argparse` to treat that input as an integer:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number", - type=int) - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py four - usage: prog.py [-h] square - prog.py: error: argument square: invalid int value: 'four' - -That went well. The program now even helpfully quits on bad illegal input -before proceeding. - - -Introducing Optional arguments -============================== - -So far we have been playing with positional arguments. Let us -have a look on how to add optional ones:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbosity", help="increase output verbosity") - args = parser.parse_args() - if args.verbosity: - print("verbosity turned on") - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py --verbosity 1 - verbosity turned on - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] [--verbosity VERBOSITY] - - optional arguments: - -h, --help show this help message and exit - --verbosity VERBOSITY - increase output verbosity - $ python3 prog.py --verbosity - usage: prog.py [-h] [--verbosity VERBOSITY] - prog.py: error: argument --verbosity: expected one argument - -Here is what is happening: - -* The program is written so as to display something when ``--verbosity`` is - specified and display nothing when not. - -* To show that the option is actually optional, there is no error when running - the program without it. Note that by default, if an optional argument isn't - used, the relevant variable, in this case :attr:`args.verbosity`, is - given ``None`` as a value, which is the reason it fails the truth - test of the :keyword:`if` statement. - -* The help message is a bit different. - -* When using the ``--verbosity`` option, one must also specify some value, - any value. - -The above example accepts arbitrary integer values for ``--verbosity``, but for -our simple program, only two values are actually useful, ``True`` or ``False``. -Let's modify the code accordingly:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py --verbose - verbosity turned on - $ python3 prog.py --verbose 1 - usage: prog.py [-h] [--verbose] - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py --help - usage: prog.py [-h] [--verbose] - - optional arguments: - -h, --help show this help message and exit - --verbose increase output verbosity - -Here is what is happening: - -* The option is now more of a flag than something that requires a value. - We even changed the name of the option to match that idea. - Note that we now specify a new keyword, ``action``, and give it the value - ``"store_true"``. This means that, if the option is specified, - assign the value ``True`` to :data:`args.verbose`. - Not specifying it implies ``False``. - -* It complains when you specify a value, in true spirit of what flags - actually are. - -* Notice the different help text. - - -Short options -------------- - -If you are familiar with command line usage, -you will notice that I haven't yet touched on the topic of short -versions of the options. It's quite simple:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And here goes: - -.. code-block:: shell-session - - $ python3 prog.py -v - verbosity turned on - $ python3 prog.py --help - usage: prog.py [-h] [-v] - - optional arguments: - -h, --help show this help message and exit - -v, --verbose increase output verbosity - -Note that the new ability is also reflected in the help text. - - -Combining Positional and Optional arguments -=========================================== - -Our program keeps growing in complexity:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbose", action="store_true", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbose: - print("the square of {} equals {}".format(args.square, answer)) - else: - print(answer) - -And now the output: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] [-v] square - prog.py: error: the following arguments are required: square - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 --verbose - the square of 4 equals 16 - $ python3 prog.py --verbose 4 - the square of 4 equals 16 - -* We've brought back a positional argument, hence the complaint. - -* Note that the order does not matter. - -How about we give this program of ours back the ability to have -multiple verbosity values, and actually get to use them:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - usage: prog.py [-h] [-v VERBOSITY] square - prog.py: error: argument -v/--verbosity: expected one argument - $ python3 prog.py 4 -v 1 - 4^2 == 16 - $ python3 prog.py 4 -v 2 - the square of 4 equals 16 - $ python3 prog.py 4 -v 3 - 16 - -These all look good except the last one, which exposes a bug in our program. -Let's fix it by restricting the values the ``--verbosity`` option can accept:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, choices=[0, 1, 2], - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 -v 3 - usage: prog.py [-h] [-v {0,1,2}] square - prog.py: error: argument -v/--verbosity: invalid choice: 3 (choose from 0, 1, 2) - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v {0,1,2}] square - - positional arguments: - square display a square of a given number - - optional arguments: - -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} - increase output verbosity - -Note that the change also reflects both in the error message as well as the -help string. - -Now, let's use a different approach of playing with verbosity, which is pretty -common. It also matches the way the CPython executable handles its own -verbosity argument (check the output of ``python --help``):: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display the square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -We have introduced another action, "count", -to count the number of occurrences of a specific optional arguments: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - 4^2 == 16 - $ python3 prog.py 4 -vv - the square of 4 equals 16 - $ python3 prog.py 4 --verbosity --verbosity - the square of 4 equals 16 - $ python3 prog.py 4 -v 1 - usage: prog.py [-h] [-v] square - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v] square - - positional arguments: - square display a square of a given number - - optional arguments: - -h, --help show this help message and exit - -v, --verbosity increase output verbosity - $ python3 prog.py 4 -vvv - 16 - -* Yes, it's now more of a flag (similar to ``action="store_true"``) in the - previous version of our script. That should explain the complaint. - -* It also behaves similar to "store_true" action. - -* Now here's a demonstration of what the "count" action gives. You've probably - seen this sort of usage before. - -* And if you don't specify the ``-v`` flag, that flag is considered to have - ``None`` value. - -* As should be expected, specifying the long form of the flag, we should get - the same output. - -* Sadly, our help output isn't very informative on the new ability our script - has acquired, but that can always be fixed by improving the documentation for - our script (e.g. via the ``help`` keyword argument). - -* That last output exposes a bug in our program. - - -Let's fix:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - - # bugfix: replace == with >= - if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And this is what it gives: - -.. code-block:: shell-session - - $ python3 prog.py 4 -vvv - the square of 4 equals 16 - $ python3 prog.py 4 -vvvv - the square of 4 equals 16 - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 11, in <module> - if args.verbosity >= 2: - TypeError: '>=' not supported between instances of 'NoneType' and 'int' - - -* First output went well, and fixes the bug we had before. - That is, we want any value >= 2 to be as verbose as possible. - -* Third output not so good. - -Let's fix that bug:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", default=0, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -We've just introduced yet another keyword, ``default``. -We've set it to ``0`` in order to make it comparable to the other int values. -Remember that by default, -if an optional argument isn't specified, -it gets the ``None`` value, and that cannot be compared to an int value -(hence the :exc:`TypeError` exception). - -And: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - -You can go quite far just with what we've learned so far, -and we have only scratched the surface. -The :mod:`argparse` module is very powerful, -and we'll explore a bit more of it before we end this tutorial. - - -Getting a little more advanced -============================== - -What if we wanted to expand our tiny program to perform other powers, -not just squares:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - elif args.verbosity >= 1: - print("{}^{} == {}".format(args.x, args.y, answer)) - else: - print(answer) - -Output: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] [-v] x y - prog.py: error: the following arguments are required: x, y - $ python3 prog.py -h - usage: prog.py [-h] [-v] x y - - positional arguments: - x the base - y the exponent - - optional arguments: - -h, --help show this help message and exit - -v, --verbosity - $ python3 prog.py 4 2 -v - 4^2 == 16 - - -Notice that so far we've been using verbosity level to *change* the text -that gets displayed. The following example instead uses verbosity level -to display *more* text instead:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print("Running '{}'".format(__file__)) - if args.verbosity >= 1: - print("{}^{} == ".format(args.x, args.y), end="") - print(answer) - -Output: - -.. code-block:: shell-session - - $ python3 prog.py 4 2 - 16 - $ python3 prog.py 4 2 -v - 4^2 == 16 - $ python3 prog.py 4 2 -vv - Running 'prog.py' - 4^2 == 16 - - -Conflicting options -------------------- - -So far, we have been working with two methods of an -:class:`argparse.ArgumentParser` instance. Let's introduce a third one, -:meth:`add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program so that -the new functionality makes more sense: -we'll introduce the ``--quiet`` option, -which will be the opposite of the ``--verbose`` one:: - - import argparse - - parser = argparse.ArgumentParser() - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - else: - print("{}^{} == {}".format(args.x, args.y, answer)) - -Our program is now simpler, and we've lost some functionality for the sake of -demonstration. Anyways, here's the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 2 - 4^2 == 16 - $ python3 prog.py 4 2 -q - 16 - $ python3 prog.py 4 2 -v - 4 to the power 2 equals 16 - $ python3 prog.py 4 2 -vq - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - $ python3 prog.py 4 2 -v --quiet - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - -That should be easy to follow. I've added that last output so you can see the -sort of flexibility you get, i.e. mixing long form options with short form -ones. - -Before we conclude, you probably want to tell your users the main purpose of -your program, just in case they don't know:: - - import argparse - - parser = argparse.ArgumentParser(description="calculate X to the power of Y") - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - else: - print("{}^{} == {}".format(args.x, args.y, answer)) - -Note that slight difference in the usage text. Note the ``[-v | -q]``, -which tells us that we can either use ``-v`` or ``-q``, -but not both at the same time: - -.. code-block:: shell-session - - $ python3 prog.py --help - usage: prog.py [-h] [-v | -q] x y - - calculate X to the power of Y - - positional arguments: - x the base - y the exponent - - optional arguments: - -h, --help show this help message and exit - -v, --verbose - -q, --quiet - - -Conclusion -========== - -The :mod:`argparse` module offers a lot more than shown here. -Its docs are quite detailed and thorough, and full of examples. -Having gone through this tutorial, you should easily digest them -without feeling overwhelmed. diff --git a/third_party/python/Doc/howto/clinic.rst b/third_party/python/Doc/howto/clinic.rst deleted file mode 100644 index 77b2aae08..000000000 --- a/third_party/python/Doc/howto/clinic.rst +++ /dev/null @@ -1,1734 +0,0 @@ -.. highlightlang:: c - -********************** -Argument Clinic How-To -********************** - -:author: Larry Hastings - - -.. topic:: Abstract - - Argument Clinic is a preprocessor for CPython C files. - Its purpose is to automate all the boilerplate involved - with writing argument parsing code for "builtins". - This document shows you how to convert your first C - function to work with Argument Clinic, and then introduces - some advanced topics on Argument Clinic usage. - - Currently Argument Clinic is considered internal-only - for CPython. Its use is not supported for files outside - CPython, and no guarantees are made regarding backwards - compatibility for future versions. In other words: if you - maintain an external C extension for CPython, you're welcome - to experiment with Argument Clinic in your own code. But the - version of Argument Clinic that ships with the next version - of CPython *could* be totally incompatible and break all your code. - -The Goals Of Argument Clinic -============================ - -Argument Clinic's primary goal -is to take over responsibility for all argument parsing code -inside CPython. This means that, when you convert a function -to work with Argument Clinic, that function should no longer -do any of its own argument parsing—the code generated by -Argument Clinic should be a "black box" to you, where CPython -calls in at the top, and your code gets called at the bottom, -with ``PyObject *args`` (and maybe ``PyObject *kwargs``) -magically converted into the C variables and types you need. - -In order for Argument Clinic to accomplish its primary goal, -it must be easy to use. Currently, working with CPython's -argument parsing library is a chore, requiring maintaining -redundant information in a surprising number of places. -When you use Argument Clinic, you don't have to repeat yourself. - -Obviously, no one would want to use Argument Clinic unless -it's solving their problem—and without creating new problems of -its own. -So it's paramount that Argument Clinic generate correct code. -It'd be nice if the code was faster, too, but at the very least -it should not introduce a major speed regression. (Eventually Argument -Clinic *should* make a major speedup possible—we could -rewrite its code generator to produce tailor-made argument -parsing code, rather than calling the general-purpose CPython -argument parsing library. That would make for the fastest -argument parsing possible!) - -Additionally, Argument Clinic must be flexible enough to -work with any approach to argument parsing. Python has -some functions with some very strange parsing behaviors; -Argument Clinic's goal is to support all of them. - -Finally, the original motivation for Argument Clinic was -to provide introspection "signatures" for CPython builtins. -It used to be, the introspection query functions would throw -an exception if you passed in a builtin. With Argument -Clinic, that's a thing of the past! - -One idea you should keep in mind, as you work with -Argument Clinic: the more information you give it, the -better job it'll be able to do. -Argument Clinic is admittedly relatively simple right -now. But as it evolves it will get more sophisticated, -and it should be able to do many interesting and smart -things with all the information you give it. - - -Basic Concepts And Usage -======================== - -Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. -If you run that script, specifying a C file as an argument: - -.. code-block:: shell-session - - $ python3 Tools/clinic/clinic.py foo.c - -Argument Clinic will scan over the file looking for lines that -look exactly like this: - -.. code-block:: none - - /*[clinic input] - -When it finds one, it reads everything up to a line that looks -exactly like this: - -.. code-block:: none - - [clinic start generated code]*/ - -Everything in between these two lines is input for Argument Clinic. -All of these lines, including the beginning and ending comment -lines, are collectively called an Argument Clinic "block". - -When Argument Clinic parses one of these blocks, it -generates output. This output is rewritten into the C file -immediately after the block, followed by a comment containing a checksum. -The Argument Clinic block now looks like this: - -.. code-block:: none - - /*[clinic input] - ... clinic input goes here ... - [clinic start generated code]*/ - ... clinic output goes here ... - /*[clinic end generated code: checksum=...]*/ - -If you run Argument Clinic on the same file a second time, Argument Clinic -will discard the old output and write out the new output with a fresh checksum -line. However, if the input hasn't changed, the output won't change either. - -You should never modify the output portion of an Argument Clinic block. Instead, -change the input until it produces the output you want. (That's the purpose of the -checksum—to detect if someone changed the output, as these edits would be lost -the next time Argument Clinic writes out fresh output.) - -For the sake of clarity, here's the terminology we'll use with Argument Clinic: - -* The first line of the comment (``/*[clinic input]``) is the *start line*. -* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*. -* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*. -* In between the start line and the end line is the *input*. -* In between the end line and the checksum line is the *output*. -* All the text collectively, from the start line to the checksum line inclusively, - is the *block*. (A block that hasn't been successfully processed by Argument - Clinic yet doesn't have output or a checksum line, but it's still considered - a block.) - - -Converting Your First Function -============================== - -The best way to get a sense of how Argument Clinic works is to -convert a function to work with it. Here, then, are the bare -minimum steps you'd need to follow to convert a function to -work with Argument Clinic. Note that for code you plan to -check in to CPython, you really should take the conversion farther, -using some of the advanced concepts you'll see later on in -the document (like "return converters" and "self converters"). -But we'll keep it simple for this walkthrough so you can learn. - -Let's dive in! - -0. Make sure you're working with a freshly updated checkout - of the CPython trunk. - -1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted - to work with Argument Clinic yet. - For my example I'm using ``_pickle.Pickler.dump()``. - -2. If the call to the ``PyArg_Parse`` function uses any of the - following format units: - - .. code-block:: none - - O& - O! - es - es# - et - et# - - or if it has multiple calls to :c:func:`PyArg_ParseTuple`, - you should choose a different function. Argument Clinic *does* - support all of these scenarios. But these are advanced - topics—let's do something simpler for your first function. - - Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different - types for the same argument, or if the function uses something besides - PyArg_Parse functions to parse its arguments, it probably - isn't suitable for conversion to Argument Clinic. Argument Clinic - doesn't support generic functions or polymorphic parameters. - -3. Add the following boilerplate above the function, creating our block:: - - /*[clinic input] - [clinic start generated code]*/ - -4. Cut the docstring and paste it in between the ``[clinic]`` lines, - removing all the junk that makes it a properly quoted C string. - When you're done you should have just the text, based at the left - margin, with no line wider than 80 characters. - (Argument Clinic will preserve indents inside the docstring.) - - If the old docstring had a first line that looked like a function - signature, throw that line away. (The docstring doesn't need it - anymore—when you use ``help()`` on your builtin in the future, - the first line will be built automatically based on the function's - signature.) - - Sample:: - - /*[clinic input] - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -5. If your docstring doesn't have a "summary" line, Argument Clinic will - complain. So let's make sure it has one. The "summary" line should - be a paragraph consisting of a single 80-column line - at the beginning of the docstring. - - (Our example docstring consists solely of a summary line, so the sample - code doesn't have to change for this step.) - -6. Above the docstring, enter the name of the function, followed - by a blank line. This should be the Python name of the function, - and should be the full dotted path - to the function—it should start with the name of the module, - include any sub-modules, and if the function is a method on - a class it should include the class name too. - - Sample:: - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -7. If this is the first time that module or class has been used with Argument - Clinic in this C file, - you must declare the module and/or class. Proper Argument Clinic hygiene - prefers declaring these in a separate block somewhere near the - top of the C file, in the same way that include files and statics go at - the top. (In our sample code we'll just show the two blocks next to - each other.) - - The name of the class and module should be the same as the one - seen by Python. Check the name defined in the :c:type:`PyModuleDef` - or :c:type:`PyTypeObject` as appropriate. - - When you declare a class, you must also specify two aspects of its type - in C: the type declaration you'd use for a pointer to an instance of - this class, and a pointer to the :c:type:`PyTypeObject` for this class. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - - - -8. Declare each of the parameters to the function. Each parameter - should get its own line. All the parameter lines should be - indented from the function name and the docstring. - - The general form of these parameter lines is as follows: - - .. code-block:: none - - name_of_parameter: converter - - If the parameter has a default value, add that after the - converter: - - .. code-block:: none - - name_of_parameter: converter = default_value - - Argument Clinic's support for "default values" is quite sophisticated; - please see :ref:`the section below on default values <default_values>` - for more information. - - Add a blank line below the parameters. - - What's a "converter"? It establishes both the type - of the variable used in C, and the method to convert the Python - value into a C value at runtime. - For now you're going to use what's called a "legacy converter"—a - convenience syntax intended to make porting old code into Argument - Clinic easier. - - For each parameter, copy the "format unit" for that - parameter from the ``PyArg_Parse()`` format argument and - specify *that* as its converter, as a quoted - string. ("format unit" is the formal name for the one-to-three - character substring of the ``format`` parameter that tells - the argument parsing function what the type of the variable - is and how to convert it. For more on format units please - see :ref:`arg-parsing`.) - - For multicharacter format units like ``z#``, use the - entire two-or-three character string. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -9. If your function has ``|`` in the format string, meaning some - parameters have default values, you can ignore it. Argument - Clinic infers which parameters are optional based on whether - or not they have default values. - - If your function has ``$`` in the format string, meaning it - takes keyword-only arguments, specify ``*`` on a line by - itself before the first keyword-only argument, indented the - same as the parameter lines. - - (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.) - - -10. If the existing C function calls :c:func:`PyArg_ParseTuple` - (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its - arguments are positional-only. - - To mark all parameters as positional-only in Argument Clinic, - add a ``/`` on a line by itself after the last parameter, - indented the same as the parameter lines. - - Currently this is all-or-nothing; either all parameters are - positional-only, or none of them are. (In the future Argument - Clinic may relax this restriction.) - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -11. It's helpful to write a per-parameter docstring for each parameter. - But per-parameter docstrings are optional; you can skip this step - if you prefer. - - Here's how to add a per-parameter docstring. The first line - of the per-parameter docstring must be indented further than the - parameter definition. The left margin of this first line establishes - the left margin for the whole per-parameter docstring; all the text - you write will be outdented by this amount. You can write as much - text as you like, across multiple lines if you wish. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -12. Save and close the file, then run ``Tools/clinic/clinic.py`` on - it. With luck everything worked---your block now has output, and - a ``.c.h`` file has been generated! Reopen the file in your - text editor to see:: - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - static PyObject * - _pickle_Pickler_dump(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/ - - Obviously, if Argument Clinic didn't produce any output, it's because - it found an error in your input. Keep fixing your errors and retrying - until Argument Clinic processes your file without complaint. - - For readability, most of the glue code has been generated to a ``.c.h`` - file. You'll need to include that in your original ``.c`` file, - typically right after the clinic module block:: - - #include "clinic/_pickle.c.h" - -13. Double-check that the argument-parsing code Argument Clinic generated - looks basically the same as the existing code. - - First, ensure both places use the same argument-parsing function. - The existing code must call either - :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`; - ensure that the code generated by Argument Clinic calls the - *exact* same function. - - Second, the format string passed in to :c:func:`PyArg_ParseTuple` or - :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same - as the hand-written one in the existing function, up to the colon - or semi-colon. - - (Argument Clinic always generates its format strings - with a ``:`` followed by the name of the function. If the - existing code's format string ends with ``;``, to provide - usage help, this change is harmless—don't worry about it.) - - Third, for parameters whose format units require two arguments - (like a length variable, or an encoding string, or a pointer - to a conversion function), ensure that the second argument is - *exactly* the same between the two invocations. - - Fourth, inside the output portion of the block you'll find a preprocessor - macro defining the appropriate static :c:type:`PyMethodDef` structure for - this builtin:: - - #define __PICKLE_PICKLER_DUMP_METHODDEF \ - {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__}, - - This static structure should be *exactly* the same as the existing static - :c:type:`PyMethodDef` structure for this builtin. - - If any of these items differ in *any way*, - adjust your Argument Clinic function specification and rerun - ``Tools/clinic/clinic.py`` until they *are* the same. - - -14. Notice that the last line of its output is the declaration - of your "impl" function. This is where the builtin's implementation goes. - Delete the existing prototype of the function you're modifying, but leave - the opening curly brace. Now delete its argument parsing code and the - declarations of all the variables it dumps the arguments into. - Notice how the Python arguments are now arguments to this impl function; - if the implementation used different names for these variables, fix it. - - Let's reiterate, just because it's kind of weird. Your code should now - look like this:: - - static return_type - your_function_impl(...) - /*[clinic end generated code: checksum=...]*/ - { - ... - - Argument Clinic generated the checksum line and the function prototype just - above it. You should write the opening (and closing) curly braces for the - function, and the implementation inside. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - PyDoc_STRVAR(__pickle_Pickler_dump__doc__, - "Write a pickled representation of obj to the open file.\n" - "\n" - ... - static PyObject * - _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ - { - /* Check whether the Pickler was initialized correctly (issue3664). - Developers often forget to call __init__() in their subclasses, which - would trigger a segfault without this check. */ - if (self->write == NULL) { - PyErr_Format(PicklingError, - "Pickler.__init__() was not called by %s.__init__()", - Py_TYPE(self)->tp_name); - return NULL; - } - - if (_Pickler_ClearBuffer(self) < 0) - return NULL; - - ... - -15. Remember the macro with the :c:type:`PyMethodDef` structure for this - function? Find the existing :c:type:`PyMethodDef` structure for this - function and replace it with a reference to the macro. (If the builtin - is at module scope, this will probably be very near the end of the file; - if the builtin is a class method, this will probably be below but relatively - near to the implementation.) - - Note that the body of the macro contains a trailing comma. So when you - replace the existing static :c:type:`PyMethodDef` structure with the macro, - *don't* add a comma to the end. - - Sample:: - - static struct PyMethodDef Pickler_methods[] = { - __PICKLE_PICKLER_DUMP_METHODDEF - __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF - {NULL, NULL} /* sentinel */ - }; - - -16. Compile, then run the relevant portions of the regression-test suite. - This change should not introduce any new compile-time warnings or errors, - and there should be no externally-visible change to Python's behavior. - - Well, except for one difference: ``inspect.signature()`` run on your function - should now provide a valid signature! - - Congratulations, you've ported your first function to work with Argument Clinic! - -Advanced Topics -=============== - -Now that you've had some experience working with Argument Clinic, it's time -for some advanced topics. - - -Symbolic default values ------------------------ - -The default value you provide for a parameter can't be any arbitrary -expression. Currently the following are explicitly supported: - -* Numeric constants (integer and float) -* String constants -* ``True``, ``False``, and ``None`` -* Simple symbolic constants like ``sys.maxsize``, which must - start with the name of the module - -In case you're curious, this is implemented in ``from_builtin()`` -in ``Lib/inspect.py``. - -(In the future, this may need to get even more elaborate, -to allow full expressions like ``CONSTANT - 1``.) - - -Renaming the C functions and variables generated by Argument Clinic -------------------------------------------------------------------- - -Argument Clinic automatically names the functions it generates for you. -Occasionally this may cause a problem, if the generated name collides with -the name of an existing C function. There's an easy solution: override the names -used for the C functions. Just add the keyword ``"as"`` -to your function declaration line, followed by the function name you wish to use. -Argument Clinic will use that function name for the base (generated) function, -then add ``"_impl"`` to the end and use that for the name of the impl function. - -For example, if we wanted to rename the C function names generated for -``pickle.Pickler.dump``, it'd look like this:: - - /*[clinic input] - pickle.Pickler.dump as pickler_dumper - - ... - -The base function would now be named ``pickler_dumper()``, -and the impl function would now be named ``pickler_dumper_impl()``. - - -Similarly, you may have a problem where you want to give a parameter -a specific Python name, but that name may be inconvenient in C. Argument -Clinic allows you to give a parameter different names in Python and in C, -using the same ``"as"`` syntax:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - file as file_obj: object - protocol: object = NULL - * - fix_imports: bool = True - -Here, the name used in Python (in the signature and the ``keywords`` -array) would be ``file``, but the C variable would be named ``file_obj``. - -You can use this to rename the ``self`` parameter too! - - -Converting functions using PyArg_UnpackTuple --------------------------------------------- - -To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, -simply write out all the arguments, specifying each as an ``object``. You -may specify the ``type`` argument to cast the type as appropriate. All -arguments should be marked positional-only (add a ``/`` on a line by itself -after the last argument). - -Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this -will change soon. - -Optional Groups ---------------- - -Some legacy functions have a tricky approach to parsing their arguments: -they count the number of positional arguments, then use a ``switch`` statement -to call one of several different :c:func:`PyArg_ParseTuple` calls depending on -how many positional arguments there are. (These functions cannot accept -keyword-only arguments.) This approach was used to simulate optional -arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created. - -While functions using this approach can often be converted to -use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values, -it's not always possible. Some of these legacy functions have -behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support. -The most obvious example is the builtin function ``range()``, which has -an optional argument on the *left* side of its required argument! -Another example is ``curses.window.addch()``, which has a group of two -arguments that must always be specified together. (The arguments are -called ``x`` and ``y``; if you call the function passing in ``x``, -you must also pass in ``y``—and if you don't pass in ``x`` you may not -pass in ``y`` either.) - -In any case, the goal of Argument Clinic is to support argument parsing -for all existing CPython builtins without changing their semantics. -Therefore Argument Clinic supports -this alternate approach to parsing, using what are called *optional groups*. -Optional groups are groups of arguments that must all be passed in together. -They can be to the left or the right of the required arguments. They -can *only* be used with positional-only parameters. - -.. note:: Optional groups are *only* intended for use when converting - functions that make multiple calls to :c:func:`PyArg_ParseTuple`! - Functions that use *any* other approach for parsing arguments - should *almost never* be converted to Argument Clinic using - optional groups. Functions using optional groups currently - cannot have accurate signatures in Python, because Python just - doesn't understand the concept. Please avoid using optional - groups wherever possible. - -To specify an optional group, add a ``[`` on a line by itself before -the parameters you wish to group together, and a ``]`` on a line by itself -after these parameters. As an example, here's how ``curses.window.addch`` -uses optional groups to make the first two parameters and the last -parameter optional:: - - /*[clinic input] - - curses.window.addch - - [ - x: int - X-coordinate. - y: int - Y-coordinate. - ] - - ch: object - Character to add. - - [ - attr: long - Attributes for the character. - ] - / - - ... - - -Notes: - -* For every optional group, one additional parameter will be passed into the - impl function representing the group. The parameter will be an int named - ``group_{direction}_{number}``, - where ``{direction}`` is either ``right`` or ``left`` depending on whether the group - is before or after the required parameters, and ``{number}`` is a monotonically - increasing number (starting at 1) indicating how far away the group is from - the required parameters. When the impl is called, this parameter will be set - to zero if this group was unused, and set to non-zero if this group was used. - (By used or unused, I mean whether or not the parameters received arguments - in this invocation.) - -* If there are no required arguments, the optional groups will behave - as if they're to the right of the required arguments. - -* In the case of ambiguity, the argument parsing code - favors parameters on the left (before the required parameters). - -* Optional groups can only contain positional-only parameters. - -* Optional groups are *only* intended for legacy code. Please do not - use optional groups for new code. - - -Using real Argument Clinic converters, instead of "legacy converters" ---------------------------------------------------------------------- - -To save time, and to minimize how much you need to learn -to achieve your first port to Argument Clinic, the walkthrough above tells -you to use "legacy converters". "Legacy converters" are a convenience, -designed explicitly to make porting existing code to Argument Clinic -easier. And to be clear, their use is acceptable when porting code for -Python 3.4. - -However, in the long term we probably want all our blocks to -use Argument Clinic's real syntax for converters. Why? A couple -reasons: - -* The proper converters are far easier to read and clearer in their intent. -* There are some format units that are unsupported as "legacy converters", - because they require arguments, and the legacy converter syntax doesn't - support specifying arguments. -* In the future we may have a new argument parsing library that isn't - restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility - won't be available to parameters using legacy converters. - -Therefore, if you don't mind a little extra effort, please use the normal -converters instead of legacy converters. - -In a nutshell, the syntax for Argument Clinic (non-legacy) converters -looks like a Python function call. However, if there are no explicit -arguments to the function (all functions take their default values), -you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly -the same converters. - -All arguments to Argument Clinic converters are keyword-only. -All Argument Clinic converters accept the following arguments: - - ``c_default`` - The default value for this parameter when defined in C. - Specifically, this will be the initializer for the variable declared - in the "parse function". See :ref:`the section on default values <default_values>` - for how to use this. - Specified as a string. - - ``annotation`` - The annotation value for this parameter. Not currently supported, - because PEP 8 mandates that the Python library may not use - annotations. - -In addition, some converters accept additional arguments. Here is a list -of these arguments, along with their meanings: - - ``accept`` - A set of Python types (and possibly pseudo-types); - this restricts the allowable Python argument to values of these types. - (This is not a general-purpose facility; as a rule it only supports - specific lists of types as shown in the legacy converter table.) - - To accept ``None``, add ``NoneType`` to this set. - - ``bitwise`` - Only supported for unsigned integers. The native integer value of this - Python argument will be written to the parameter without any range checking, - even for negative values. - - ``converter`` - Only supported by the ``object`` converter. Specifies the name of a - :ref:`C "converter function" <o_ampersand>` - to use to convert this object to a native type. - - ``encoding`` - Only supported for strings. Specifies the encoding to use when converting - this string from a Python str (Unicode) value into a C ``char *`` value. - - - ``subclass_of`` - Only supported for the ``object`` converter. Requires that the Python - value be a subclass of a Python type, as expressed in C. - - ``type`` - Only supported for the ``object`` and ``self`` converters. Specifies - the C type that will be used to declare the variable. Default value is - ``"PyObject *"``. - - ``zeroes`` - Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are - permitted inside the value. The length of the string will be passed in - to the impl function, just after the string parameter, as a parameter named - ``<parameter_name>_length``. - -Please note, not every possible combination of arguments will work. -Usually these arguments are implemented by specific ``PyArg_ParseTuple`` -*format units*, with specific behavior. For example, currently you cannot -call ``unsigned_short`` without also specifying ``bitwise=True``. -Although it's perfectly reasonable to think this would work, these semantics don't -map to any existing format unit. So Argument Clinic doesn't support it. (Or, at -least, not yet.) - -Below is a table showing the mapping of legacy converters into real -Argument Clinic converters. On the left is the legacy converter, -on the right is the text you'd replace it with. - -========= ================================================================================= -``'B'`` ``unsigned_char(bitwise=True)`` -``'b'`` ``unsigned_char`` -``'c'`` ``char`` -``'C'`` ``int(accept={str})`` -``'d'`` ``double`` -``'D'`` ``Py_complex`` -``'es'`` ``str(encoding='name_of_encoding')`` -``'es#'`` ``str(encoding='name_of_encoding', zeroes=True)`` -``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})`` -``'et#'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str}, zeroes=True)`` -``'f'`` ``float`` -``'h'`` ``short`` -``'H'`` ``unsigned_short(bitwise=True)`` -``'i'`` ``int`` -``'I'`` ``unsigned_int(bitwise=True)`` -``'k'`` ``unsigned_long(bitwise=True)`` -``'K'`` ``unsigned_long_long(bitwise=True)`` -``'l'`` ``long`` -``'L'`` ``long long`` -``'n'`` ``Py_ssize_t`` -``'O'`` ``object`` -``'O!'`` ``object(subclass_of='&PySomething_Type')`` -``'O&'`` ``object(converter='name_of_c_function')`` -``'p'`` ``bool`` -``'S'`` ``PyBytesObject`` -``'s'`` ``str`` -``'s#'`` ``str(zeroes=True)`` -``'s*'`` ``Py_buffer(accept={buffer, str})`` -``'U'`` ``unicode`` -``'u'`` ``Py_UNICODE`` -``'u#'`` ``Py_UNICODE(zeroes=True)`` -``'w*'`` ``Py_buffer(accept={rwbuffer})`` -``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(accept={bytes})`` -``'y#'`` ``str(accept={robuffer}, zeroes=True)`` -``'y*'`` ``Py_buffer`` -``'Z'`` ``Py_UNICODE(accept={str, NoneType})`` -``'Z#'`` ``Py_UNICODE(accept={str, NoneType}, zeroes=True)`` -``'z'`` ``str(accept={str, NoneType})`` -``'z#'`` ``str(accept={str, NoneType}, zeroes=True)`` -``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})`` -========= ================================================================================= - -As an example, here's our sample ``pickle.Pickler.dump`` using the proper -converter:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -Argument Clinic will show you all the converters it has -available. For each converter it'll show you all the parameters -it accepts, along with the default value for each parameter. -Just run ``Tools/clinic/clinic.py --converters`` to see the full list. - -Py_buffer ---------- - -When using the ``Py_buffer`` converter -(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters), -you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. -Argument Clinic generates code that does it for you (in the parsing function). - - - -Advanced converters -------------------- - -Remember those format units you skipped for your first -time because they were advanced? Here's how to handle those too. - -The trick is, all those format units take arguments—either -conversion functions, or types, or strings specifying an encoding. -(But "legacy converters" don't support arguments. That's why we -skipped them for your first function.) The argument you specified -to the format unit is now an argument to the converter; this -argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``), -or ``encoding`` (for all the format units that start with ``e``). - -When using ``subclass_of``, you may also want to use the other -custom argument for ``object()``: ``type``, which lets you set the type -actually used for the parameter. For example, if you want to ensure -that the object is a subclass of ``PyUnicode_Type``, you probably want -to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``. - -One possible problem with using Argument Clinic: it takes away some possible -flexibility for the format units starting with ``e``. When writing a -``PyArg_Parse`` call by hand, you could theoretically decide at runtime what -encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must -be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate; -it made supporting this format unit much easier, and may allow for future optimizations. -This restriction doesn't seem unreasonable; CPython itself always passes in static -hard-coded encoding strings for parameters whose format units start with ``e``. - - -.. _default_values: - -Parameter default values ------------------------- - -Default values for parameters can be any of a number of values. -At their simplest, they can be string, int, or float literals: - -.. code-block:: none - - foo: str = "abc" - bar: int = 123 - bat: float = 45.6 - -They can also use any of Python's built-in constants: - -.. code-block:: none - - yep: bool = True - nope: bool = False - nada: object = None - -There's also special support for a default value of ``NULL``, and -for simple expressions, documented in the following sections. - - -The ``NULL`` default value --------------------------- - -For string and object parameters, you can set them to ``None`` to indicate -that there's no default. However, that means the C variable will be -initialized to ``Py_None``. For convenience's sakes, there's a special -value called ``NULL`` for just this reason: from Python's perspective it -behaves like a default value of ``None``, but the C variable is initialized -with ``NULL``. - -Expressions specified as default values ---------------------------------------- - -The default value for a parameter can be more than just a literal value. -It can be an entire expression, using math operators and looking up attributes -on objects. However, this support isn't exactly simple, because of some -non-obvious semantics. - -Consider the following example: - -.. code-block:: none - - foo: Py_ssize_t = sys.maxsize - 1 - -``sys.maxsize`` can have different values on different platforms. Therefore -Argument Clinic can't simply evaluate that expression locally and hard-code it -in C. So it stores the default in such a way that it will get evaluated at -runtime, when the user asks for the function's signature. - -What namespace is available when the expression is evaluated? It's evaluated -in the context of the module the builtin came from. So, if your module has an -attribute called "``max_widgets``", you may simply use it: - -.. code-block:: none - - foo: Py_ssize_t = max_widgets - -If the symbol isn't found in the current module, it fails over to looking in -``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you -don't know in advance what modules the user will load into their interpreter, -it's best to restrict yourself to modules that are preloaded by Python itself.) - -Evaluating default values only at runtime means Argument Clinic can't compute -the correct equivalent C default value. So you need to tell it explicitly. -When you use an expression, you must also specify the equivalent expression -in C, using the ``c_default`` parameter to the converter: - -.. code-block:: none - - foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 - -Another complication: Argument Clinic can't know in advance whether or not the -expression you supply is valid. It parses it to make sure it looks legal, but -it can't *actually* know. You must be very careful when using expressions to -specify values that are guaranteed to be valid at runtime! - -Finally, because expressions must be representable as static C values, there -are many restrictions on legal expressions. Here's a list of Python features -you're not permitted to use: - -* Function calls. -* Inline if statements (``3 if foo else 5``). -* Automatic sequence unpacking (``*[1, 2, 3]``). -* List/set/dict comprehensions and generator expressions. -* Tuple/list/set/dict literals. - - - -Using a return converter ------------------------- - -By default the impl function Argument Clinic generates for you returns ``PyObject *``. -But your C function often computes some C type, then converts it into the ``PyObject *`` -at the last moment. Argument Clinic handles converting your inputs from Python types -into native C types—why not have it convert your return value from a native C type -into a Python type too? - -That's what a "return converter" does. It changes your impl function to return -some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate ``PyObject *``. - -The syntax for return converters is similar to that of parameter converters. -You specify the return converter like it was a return annotation on the -function itself. Return converters behave much the same as parameter converters; -they take arguments, the arguments are all keyword-only, and if you're not changing -any of the default arguments you can omit the parentheses. - -(If you use both ``"as"`` *and* a return converter for your function, -the ``"as"`` should come before the return converter.) - -There's one additional complication when using return converters: how do you -indicate an error has occurred? Normally, a function returns a valid (non-``NULL``) -pointer for success, and ``NULL`` for failure. But if you use an integer return converter, -all integers are valid. How can Argument Clinic detect an error? Its solution: each return -converter implicitly looks for a special value that indicates an error. If you return -that value, and an error has been set (``PyErr_Occurred()`` returns a true -value), then the generated code will propagate the error. Otherwise it will -encode the value you return like normal. - -Currently Argument Clinic supports only a few return converters: - -.. code-block:: none - - bool - int - unsigned int - long - unsigned int - size_t - Py_ssize_t - float - double - DecodeFSDefault - -None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``char *``; return a NULL -pointer to indicate an error. - -(There's also an experimental ``NoneType`` converter, which lets you -return ``Py_None`` on success or ``NULL`` on failure, without having -to increment the reference count on ``Py_None``. I'm not sure it adds -enough clarity to be worth using.) - -To see all the return converters Argument Clinic supports, along with -their parameters (if any), -just run ``Tools/clinic/clinic.py --converters`` for the full list. - - -Cloning existing functions --------------------------- - -If you have a number of functions that look similar, you may be able to -use Clinic's "clone" feature. When you clone an existing function, -you reuse: - -* its parameters, including - - * their names, - - * their converters, with all parameters, - - * their default values, - - * their per-parameter docstrings, - - * their *kind* (whether they're positional only, - positional or keyword, or keyword only), and - -* its return converter. - -The only thing not copied from the original function is its docstring; -the syntax allows you to specify a new docstring. - -Here's the syntax for cloning a function:: - - /*[clinic input] - module.class.new_function [as c_basename] = module.class.existing_function - - Docstring for new_function goes here. - [clinic start generated code]*/ - -(The functions can be in different modules or classes. I wrote -``module.class`` in the sample just to illustrate that you must -use the full path to *both* functions.) - -Sorry, there's no syntax for partially-cloning a function, or cloning a function -then modifying it. Cloning is an all-or nothing proposition. - -Also, the function you are cloning from must have been previously defined -in the current file. - -Calling Python code -------------------- - -The rest of the advanced topics require you to write Python code -which lives inside your C file and modifies Argument Clinic's -runtime state. This is simple: you simply define a Python block. - -A Python block uses different delimiter lines than an Argument -Clinic function block. It looks like this:: - - /*[python input] - # python code goes here - [python start generated code]*/ - -All the code inside the Python block is executed at the -time it's parsed. All text written to stdout inside the block -is redirected into the "output" after the block. - -As an example, here's a Python block that adds a static integer -variable to the C code:: - - /*[python input] - print('static int __ignored_unused_variable__ = 0;') - [python start generated code]*/ - static int __ignored_unused_variable__ = 0; - /*[python checksum:...]*/ - - -Using a "self converter" ------------------------- - -Argument Clinic automatically adds a "self" parameter for you -using a default converter. It automatically sets the ``type`` -of this parameter to the "pointer to an instance" you specified -when you declared the type. However, you can override -Argument Clinic's converter and specify one yourself. -Just add your own ``self`` parameter as the first parameter in a -block, and ensure that its converter is an instance of -``self_converter`` or a subclass thereof. - -What's the point? This lets you override the type of ``self``, -or give it a different default name. - -How do you specify the custom type you want to cast ``self`` to? -If you only have one or two functions with the same type for ``self``, -you can directly use Argument Clinic's existing ``self`` converter, -passing in the type you want to use as the ``type`` parameter:: - - /*[clinic input] - - _pickle.Pickler.dump - - self: self(type="PicklerObject *") - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - -On the other hand, if you have a lot of functions that will use the same -type for ``self``, it's best to create your own converter, subclassing -``self_converter`` but overwriting the ``type`` member:: - - /*[python input] - class PicklerObject_converter(self_converter): - type = "PicklerObject *" - [python start generated code]*/ - - /*[clinic input] - - _pickle.Pickler.dump - - self: PicklerObject - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - - - -Writing a custom converter --------------------------- - -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from ``CConverter``. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit—parsing this parameter means calling -a :c:func:`PyArg_ParseTuple` "converter function". - -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass ``CConverter.__init__``. Instead, you should -write a ``converter_init()`` function. ``converter_init()`` -always accepts a ``self`` parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -``converter_init()``. - -There are some additional members of ``CConverter`` you may wish -to specify in your subclass. Here's the current list: - -``type`` - The C type to use for this variable. - ``type`` should be a Python string specifying the type, e.g. ``int``. - If this is a pointer type, the type string should end with ``' *'``. - -``default`` - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - -``py_default`` - ``default`` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - -``c_default`` - ``default`` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - -``c_ignored_default`` - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups—although - properly-written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - -``converter`` - The name of the C converter function, as a string. - -``impl_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - -``parse_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - -Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``:: - - /*[python input] - - class ssize_t_converter(CConverter): - type = 'Py_ssize_t' - converter = 'ssize_t_converter' - - [python start generated code]*/ - /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ - -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type ``Py_ssize_t``, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. - -More sophisticated custom converters can insert custom C code to -handle initialization and cleanup. -You can see more examples of custom converters in the CPython -source tree; grep the C files for the string ``CConverter``. - -Writing a custom return converter ---------------------------------- - -Writing a custom return converter is much like writing -a custom converter. Except it's somewhat simpler, because return -converters are themselves much simpler. - -Return converters must subclass ``CReturnConverter``. -There are no examples yet of custom return converters, -because they are not widely used yet. If you wish to -write your own return converter, please read ``Tools/clinic/clinic.py``, -specifically the implementation of ``CReturnConverter`` and -all its subclasses. - -METH_O and METH_NOARGS ----------------------------------------------- - -To convert a function using ``METH_O``, make sure the function's -single argument is using the ``object`` converter, and mark the -arguments as positional-only:: - - /*[clinic input] - meth_o_sample - - argument: object - / - [clinic start generated code]*/ - - -To convert a function using ``METH_NOARGS``, just don't specify -any arguments. - -You can still use a self converter, a return converter, and specify -a ``type`` argument to the object converter for ``METH_O``. - -tp_new and tp_init functions ----------------------------------------------- - -You can convert ``tp_new`` and ``tp_init`` functions. Just name -them ``__new__`` or ``__init__`` as appropriate. Notes: - -* The function name generated for ``__new__`` doesn't end in ``__new__`` - like it would by default. It's just the name of the class, converted - into a valid C identifier. - -* No ``PyMethodDef`` ``#define`` is generated for these functions. - -* ``__init__`` functions return ``int``, not ``PyObject *``. - -* Use the docstring as the class docstring. - -* Although ``__new__`` and ``__init__`` functions must always - accept both the ``args`` and ``kwargs`` objects, when converting - you may specify any signature for these functions that you like. - (If your function doesn't support keywords, the parsing function - generated will throw an exception if it receives any.) - -Changing and redirecting Clinic's output ----------------------------------------- - -It can be inconvenient to have Clinic's output interspersed with -your conventional hand-edited C code. Luckily, Clinic is configurable: -you can buffer up its output for printing later (or earlier!), or write -its output to a separate file. You can also add a prefix or suffix to -every line of Clinic's generated output. - -While changing Clinic's output in this manner can be a boon to readability, -it may result in Clinic code using types before they are defined, or -your code attempting to use Clinic-generated code before it is defined. -These problems can be easily solved by rearranging the declarations in your file, -or moving where Clinic's generated code goes. (This is why the default behavior -of Clinic is to output everything into the current block; while many people -consider this hampers readability, it will never require rearranging your -code to fix definition-before-use problems.) - -Let's start with defining some terminology: - -*field* - A field, in this context, is a subsection of Clinic's output. - For example, the ``#define`` for the ``PyMethodDef`` structure - is a field, called ``methoddef_define``. Clinic has seven - different fields it can output per function definition: - - .. code-block:: none - - docstring_prototype - docstring_definition - methoddef_define - impl_prototype - parser_prototype - parser_definition - impl_definition - - All the names are of the form ``"<a>_<b>"``, - where ``"<a>"`` is the semantic object represented (the parsing function, - the impl function, the docstring, or the methoddef structure) and ``"<b>"`` - represents what kind of statement the field is. Field names that end in - ``"_prototype"`` - represent forward declarations of that thing, without the actual body/data - of the thing; field names that end in ``"_definition"`` represent the actual - definition of the thing, with the body/data of the thing. (``"methoddef"`` - is special, it's the only one that ends with ``"_define"``, representing that - it's a preprocessor #define.) - -*destination* - A destination is a place Clinic can write output to. There are - five built-in destinations: - - ``block`` - The default destination: printed in the output section of - the current Clinic block. - - ``buffer`` - A text buffer where you can save text for later. Text sent - here is appended to the end of any existing text. It's an - error to have any text left in the buffer when Clinic finishes - processing a file. - - ``file`` - A separate "clinic file" that will be created automatically by Clinic. - The filename chosen for the file is ``{basename}.clinic{extension}``, - where ``basename`` and ``extension`` were assigned the output - from ``os.path.splitext()`` run on the current file. (Example: - the ``file`` destination for ``_pickle.c`` would be written to - ``_pickle.clinic.c``.) - - **Important: When using a** ``file`` **destination, you** - *must check in* **the generated file!** - - ``two-pass`` - A buffer like ``buffer``. However, a two-pass buffer can only - be dumped once, and it prints out all text sent to it during - all processing, even from Clinic blocks *after* the dumping point. - - ``suppress`` - The text is suppressed—thrown away. - - -Clinic defines five new directives that let you reconfigure its output. - -The first new directive is ``dump``: - -.. code-block:: none - - dump <destination> - -This dumps the current contents of the named destination into the output of -the current block, and empties it. This only works with ``buffer`` and -``two-pass`` destinations. - -The second new directive is ``output``. The most basic form of ``output`` -is like this: - -.. code-block:: none - - output <field> <destination> - -This tells Clinic to output *field* to *destination*. ``output`` also -supports a special meta-destination, called ``everything``, which tells -Clinic to output *all* fields to that *destination*. - -``output`` has a number of other functions: - -.. code-block:: none - - output push - output pop - output preset <preset> - - -``output push`` and ``output pop`` allow you to push and pop -configurations on an internal configuration stack, so that you -can temporarily modify the output configuration, then easily restore -the previous configuration. Simply push before your change to save -the current configuration, then pop when you wish to restore the -previous configuration. - -``output preset`` sets Clinic's output to one of several built-in -preset configurations, as follows: - - ``block`` - Clinic's original starting configuration. Writes everything - immediately after the input block. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write everything else to ``block``. - - ``file`` - Designed to write everything to the "clinic file" that it can. - You then ``#include`` this file near the top of your file. - You may need to rearrange your file to make this work, though - usually this just means creating forward declarations for various - ``typedef`` and ``PyTypeObject`` definitions. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - The default filename is ``"{dirname}/clinic/{basename}.h"``. - - ``buffer`` - Save up most of the output from Clinic, to be written into - your file near the end. For Python files implementing modules - or builtin types, it's recommended that you dump the buffer - just above the static structures for your module or - builtin type; these are normally very near the end. Using - ``buffer`` may require even more editing than ``file``, if - your file has static ``PyMethodDef`` arrays defined in the - middle of the file. - - Suppress the ``parser_prototype``, ``impl_prototype``, - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - ``two-pass`` - Similar to the ``buffer`` preset, but writes forward declarations to - the ``two-pass`` buffer, and definitions to the ``buffer``. - This is similar to the ``buffer`` preset, but may require - less editing than ``buffer``. Dump the ``two-pass`` buffer - near the top of your file, and dump the ``buffer`` near - the end just like you would when using the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``impl_definition`` - to ``block``, write ``docstring_prototype``, ``methoddef_define``, - and ``parser_prototype`` to ``two-pass``, write everything else - to ``buffer``. - - ``partial-buffer`` - Similar to the ``buffer`` preset, but writes more things to ``block``, - only writing the really big chunks of generated code to ``buffer``. - This avoids the definition-before-use problem of ``buffer`` completely, - at the small cost of having slightly more stuff in the block's output. - Dump the ``buffer`` near the end, just like you would when using - the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``docstring_definition`` - and ``parser_definition`` to ``buffer``, write everything else to ``block``. - -The third new directive is ``destination``: - -.. code-block:: none - - destination <name> <command> [...] - -This performs an operation on the destination named ``name``. - -There are two defined subcommands: ``new`` and ``clear``. - -The ``new`` subcommand works like this: - -.. code-block:: none - - destination <name> new <type> - -This creates a new destination with name ``<name>`` and type ``<type>``. - -There are five destination types: - - ``suppress`` - Throws the text away. - - ``block`` - Writes the text to the current block. This is what Clinic - originally did. - - ``buffer`` - A simple text buffer, like the "buffer" builtin destination above. - - ``file`` - A text file. The file destination takes an extra argument, - a template to use for building the filename, like so: - - destination <name> new <type> <file_template> - - The template can use three strings internally that will be replaced - by bits of the filename: - - {path} - The full path to the file, including directory and full filename. - {dirname} - The name of the directory the file is in. - {basename} - Just the name of the file, not including the directory. - {basename_root} - Basename with the extension clipped off - (everything up to but not including the last '.'). - {basename_extension} - The last '.' and everything after it. If the basename - does not contain a period, this will be the empty string. - - If there are no periods in the filename, {basename} and {filename} - are the same, and {extension} is empty. "{basename}{extension}" - is always exactly the same as "{filename}"." - - ``two-pass`` - A two-pass buffer, like the "two-pass" builtin destination above. - - -The ``clear`` subcommand works like this: - -.. code-block:: none - - destination <name> clear - -It removes all the accumulated text up to this point in the destination. -(I don't know what you'd need this for, but I thought maybe it'd be -useful while someone's experimenting.) - -The fourth new directive is ``set``: - -.. code-block:: none - - set line_prefix "string" - set line_suffix "string" - -``set`` lets you set two internal variables in Clinic. -``line_prefix`` is a string that will be prepended to every line of Clinic's output; -``line_suffix`` is a string that will be appended to every line of Clinic's output. - -Both of these support two format strings: - - ``{block comment start}`` - Turns into the string ``/*``, the start-comment text sequence for C files. - - ``{block comment end}`` - Turns into the string ``*/``, the end-comment text sequence for C files. - -The final new directive is one you shouldn't need to use directly, -called ``preserve``: - -.. code-block:: none - - preserve - -This tells Clinic that the current contents of the output should be kept, unmodified. -This is used internally by Clinic when dumping output into ``file`` files; wrapping -it in a Clinic block lets Clinic use its existing checksum functionality to ensure -the file was not modified by hand before it gets overwritten. - - -The #ifdef trick ----------------------------------------------- - -If you're converting a function that isn't available on all platforms, -there's a trick you can use to make life a little easier. The existing -code probably looks like this:: - - #ifdef HAVE_FUNCTIONNAME - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -And then in the ``PyMethodDef`` structure at the bottom the existing code -will have: - -.. code-block:: none - - #ifdef HAVE_FUNCTIONNAME - {'functionname', ... }, - #endif /* HAVE_FUNCTIONNAME */ - -In this scenario, you should enclose the body of your impl function inside the ``#ifdef``, -like so:: - - #ifdef HAVE_FUNCTIONNAME - /*[clinic input] - module.functionname - ... - [clinic start generated code]*/ - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -Then, remove those three lines from the ``PyMethodDef`` structure, -replacing them with the macro Argument Clinic generated: - -.. code-block:: none - - MODULE_FUNCTIONNAME_METHODDEF - -(You can find the real name for this macro inside the generated code. -Or you can calculate it yourself: it's the name of your function as defined -on the first line of your block, but with periods changed to underscores, -uppercased, and ``"_METHODDEF"`` added to the end.) - -Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined? -The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either! - -Here's where Argument Clinic gets very clever. It actually detects that the -Argument Clinic block might be deactivated by the ``#ifdef``. When that -happens, it generates a little extra code that looks like this:: - - #ifndef MODULE_FUNCTIONNAME_METHODDEF - #define MODULE_FUNCTIONNAME_METHODDEF - #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */ - -That means the macro always works. If the function is defined, this turns -into the correct structure, including the trailing comma. If the function is -undefined, this turns into nothing. - -However, this causes one ticklish problem: where should Argument Clinic put this -extra code when using the "block" output preset? It can't go in the output block, -because that could be deactivated by the ``#ifdef``. (That's the whole point!) - -In this situation, Argument Clinic writes the extra code to the "buffer" destination. -This may mean that you get a complaint from Argument Clinic: - -.. code-block:: none - - Warning in file "Modules/posixmodule.c" on line 12357: - Destination buffer 'buffer' not empty at end of file, emptying. - -When this happens, just open your file, find the ``dump buffer`` block that -Argument Clinic added to your file (it'll be at the very bottom), then -move it above the ``PyMethodDef`` structure where that macro is used. - - - -Using Argument Clinic in Python files -------------------------------------- - -It's actually possible to use Argument Clinic to preprocess Python files. -There's no point to using Argument Clinic blocks, of course, as the output -wouldn't make any sense to the Python interpreter. But using Argument Clinic -to run Python blocks lets you use Python as a Python preprocessor! - -Since Python comments are different from C comments, Argument Clinic -blocks embedded in Python files look slightly different. They look like this: - -.. code-block:: python3 - - #/*[python input] - #print("def foo(): pass") - #[python start generated code]*/ - def foo(): pass - #/*[python checksum:...]*/ diff --git a/third_party/python/Doc/howto/cporting.rst b/third_party/python/Doc/howto/cporting.rst deleted file mode 100644 index 7cacb0aec..000000000 --- a/third_party/python/Doc/howto/cporting.rst +++ /dev/null @@ -1,257 +0,0 @@ -.. highlightlang:: c - -.. _cporting-howto: - -************************************* -Porting Extension Modules to Python 3 -************************************* - -:author: Benjamin Peterson - - -.. topic:: Abstract - - Although changing the C-API was not one of Python 3's objectives, - the many Python-level changes made leaving Python 2's API intact - impossible. In fact, some changes such as :func:`int` and - :func:`long` unification are more obvious on the C level. This - document endeavors to document incompatibilities and how they can - be worked around. - - -Conditional compilation -======================= - -The easiest way to compile only some code for Python 3 is to check -if :c:macro:`PY_MAJOR_VERSION` is greater than or equal to 3. :: - - #if PY_MAJOR_VERSION >= 3 - #define IS_PY3K - #endif - -API functions that are not present can be aliased to their equivalents within -conditional blocks. - - -Changes to Object APIs -====================== - -Python 3 merged together some types with similar functions while cleanly -separating others. - - -str/unicode Unification ------------------------ - -Python 3's :func:`str` type is equivalent to Python 2's :func:`unicode`; the C -functions are called ``PyUnicode_*`` for both. The old 8-bit string type has become -:func:`bytes`, with C functions called ``PyBytes_*``. Python 2.6 and later provide a compatibility header, -:file:`bytesobject.h`, mapping ``PyBytes`` names to ``PyString`` ones. For best -compatibility with Python 3, :c:type:`PyUnicode` should be used for textual data and -:c:type:`PyBytes` for binary data. It's also important to remember that -:c:type:`PyBytes` and :c:type:`PyUnicode` in Python 3 are not interchangeable like -:c:type:`PyString` and :c:type:`PyUnicode` are in Python 2. The following example -shows best practices with regards to :c:type:`PyUnicode`, :c:type:`PyString`, -and :c:type:`PyBytes`. :: - - #include "stdlib.h" - #include "Python.h" - #include "bytesobject.h" - - /* text example */ - static PyObject * - say_hello(PyObject *self, PyObject *args) { - PyObject *name, *result; - - if (!PyArg_ParseTuple(args, "U:say_hello", &name)) - return NULL; - - result = PyUnicode_FromFormat("Hello, %S!", name); - return result; - } - - /* just a forward */ - static char * do_encode(PyObject *); - - /* bytes example */ - static PyObject * - encode_object(PyObject *self, PyObject *args) { - char *encoded; - PyObject *result, *myobj; - - if (!PyArg_ParseTuple(args, "O:encode_object", &myobj)) - return NULL; - - encoded = do_encode(myobj); - if (encoded == NULL) - return NULL; - result = PyBytes_FromString(encoded); - free(encoded); - return result; - } - - -long/int Unification --------------------- - -Python 3 has only one integer type, :func:`int`. But it actually -corresponds to Python 2's :func:`long` type—the :func:`int` type -used in Python 2 was removed. In the C-API, ``PyInt_*`` functions -are replaced by their ``PyLong_*`` equivalents. - - -Module initialization and state -=============================== - -Python 3 has a revamped extension module initialization system. (See -:pep:`3121`.) Instead of storing module state in globals, they should -be stored in an interpreter specific structure. Creating modules that -act correctly in both Python 2 and Python 3 is tricky. The following -simple example demonstrates how. :: - - #include "Python.h" - - struct module_state { - PyObject *error; - }; - - #if PY_MAJOR_VERSION >= 3 - #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) - #else - #define GETSTATE(m) (&_state) - static struct module_state _state; - #endif - - static PyObject * - error_out(PyObject *m) { - struct module_state *st = GETSTATE(m); - PyErr_SetString(st->error, "something bad happened"); - return NULL; - } - - static PyMethodDef myextension_methods[] = { - {"error_out", (PyCFunction)error_out, METH_NOARGS, NULL}, - {NULL, NULL} - }; - - #if PY_MAJOR_VERSION >= 3 - - static int myextension_traverse(PyObject *m, visitproc visit, void *arg) { - Py_VISIT(GETSTATE(m)->error); - return 0; - } - - static int myextension_clear(PyObject *m) { - Py_CLEAR(GETSTATE(m)->error); - return 0; - } - - - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "myextension", - NULL, - sizeof(struct module_state), - myextension_methods, - NULL, - myextension_traverse, - myextension_clear, - NULL - }; - - #define INITERROR return NULL - - PyMODINIT_FUNC - PyInit_myextension(void) - - #else - #define INITERROR return - - void - initmyextension(void) - #endif - { - #if PY_MAJOR_VERSION >= 3 - PyObject *module = PyModule_Create(&moduledef); - #else - PyObject *module = Py_InitModule("myextension", myextension_methods); - #endif - - if (module == NULL) - INITERROR; - struct module_state *st = GETSTATE(module); - - st->error = PyErr_NewException("myextension.Error", NULL, NULL); - if (st->error == NULL) { - Py_DECREF(module); - INITERROR; - } - - #if PY_MAJOR_VERSION >= 3 - return module; - #endif - } - - -CObject replaced with Capsule -============================= - -The :c:type:`Capsule` object was introduced in Python 3.1 and 2.7 to replace -:c:type:`CObject`. CObjects were useful, -but the :c:type:`CObject` API was problematic: it didn't permit distinguishing -between valid CObjects, which allowed mismatched CObjects to crash the -interpreter, and some of its APIs relied on undefined behavior in C. -(For further reading on the rationale behind Capsules, please see :issue:`5630`.) - -If you're currently using CObjects, and you want to migrate to 3.1 or newer, -you'll need to switch to Capsules. -:c:type:`CObject` was deprecated in 3.1 and 2.7 and completely removed in -Python 3.2. If you only support 2.7, or 3.1 and above, you -can simply switch to :c:type:`Capsule`. If you need to support Python 3.0, -or versions of Python earlier than 2.7, -you'll have to support both CObjects and Capsules. -(Note that Python 3.0 is no longer supported, and it is not recommended -for production use.) - -The following example header file :file:`capsulethunk.h` may -solve the problem for you. Simply write your code against the -:c:type:`Capsule` API and include this header file after -:file:`Python.h`. Your code will automatically use Capsules -in versions of Python with Capsules, and switch to CObjects -when Capsules are unavailable. - -:file:`capsulethunk.h` simulates Capsules using CObjects. However, -:c:type:`CObject` provides no place to store the capsule's "name". As a -result the simulated :c:type:`Capsule` objects created by :file:`capsulethunk.h` -behave slightly differently from real Capsules. Specifically: - - * The name parameter passed in to :c:func:`PyCapsule_New` is ignored. - - * The name parameter passed in to :c:func:`PyCapsule_IsValid` and - :c:func:`PyCapsule_GetPointer` is ignored, and no error checking - of the name is performed. - - * :c:func:`PyCapsule_GetName` always returns NULL. - - * :c:func:`PyCapsule_SetName` always raises an exception and - returns failure. (Since there's no way to store a name - in a CObject, noisy failure of :c:func:`PyCapsule_SetName` - was deemed preferable to silent failure here. If this is - inconvenient, feel free to modify your local - copy as you see fit.) - -You can find :file:`capsulethunk.h` in the Python source distribution -as :source:`Doc/includes/capsulethunk.h`. We also include it here for -your convenience: - -.. literalinclude:: ../includes/capsulethunk.h - - - -Other options -============= - -If you are writing a new extension module, you might consider `Cython -<http://cython.org/>`_. It translates a Python-like language to C. The -extension modules it creates are compatible with Python 3 and Python 2. - diff --git a/third_party/python/Doc/howto/curses.rst b/third_party/python/Doc/howto/curses.rst deleted file mode 100644 index d5c07714a..000000000 --- a/third_party/python/Doc/howto/curses.rst +++ /dev/null @@ -1,552 +0,0 @@ -.. _curses-howto: - -********************************** - Curses Programming with Python -********************************** - -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - - -.. topic:: Abstract - - This document describes how to use the :mod:`curses` extension - module to control text-mode displays. - - -What is curses? -=============== - -The curses library supplies a terminal-independent screen-painting and -keyboard-handling facility for text-based terminals; such terminals -include VT100s, the Linux console, and the simulated terminal provided -by various programs. Display terminals support various control codes -to perform common operations such as moving the cursor, scrolling the -screen, and erasing areas. Different terminals use widely differing -codes, and often have their own minor quirks. - -In a world of graphical displays, one might ask "why bother"? It's -true that character-cell display terminals are an obsolete technology, -but there are niches in which being able to do fancy things with them -are still valuable. One niche is on small-footprint or embedded -Unixes that don't run an X server. Another is tools such as OS -installers and kernel configurators that may have to run before any -graphical support is available. - -The curses library provides fairly basic functionality, providing the -programmer with an abstraction of a display containing multiple -non-overlapping windows of text. The contents of a window can be -changed in various ways---adding text, erasing it, changing its -appearance---and the curses library will figure out what control codes -need to be sent to the terminal to produce the right output. curses -doesn't provide many user-interface concepts such as buttons, checkboxes, -or dialogs; if you need such features, consider a user interface library such as -`Urwid <https://pypi.org/project/urwid/>`_. - -The curses library was originally written for BSD Unix; the later System V -versions of Unix from AT&T added many enhancements and new functions. BSD curses -is no longer maintained, having been replaced by ncurses, which is an -open-source implementation of the AT&T interface. If you're using an -open-source Unix such as Linux or FreeBSD, your system almost certainly uses -ncurses. Since most current commercial Unix versions are based on System V -code, all the functions described here will probably be available. The older -versions of curses carried by some proprietary Unixes may not support -everything, though. - -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -<https://pypi.org/project/UniCurses>`_ is available. You could -also try `the Console module <http://effbot.org/zone/console-index.htm>`_ -written by Fredrik Lundh, which doesn't -use the same API as curses but provides cursor-addressable text output -and full support for mouse and keyboard input. - - -The Python curses module ------------------------- - -The Python module is a fairly simple wrapper over the C functions provided by -curses; if you're already familiar with curses programming in C, it's really -easy to transfer that knowledge to Python. The biggest difference is that the -Python interface makes things simpler by merging different C functions such as -:c:func:`addstr`, :c:func:`mvaddstr`, and :c:func:`mvwaddstr` into a single -:meth:`~curses.window.addstr` method. You'll see this covered in more -detail later. - -This HOWTO is an introduction to writing text-mode programs with curses -and Python. It doesn't attempt to be a complete guide to the curses API; for -that, see the Python library guide's section on ncurses, and the C manual pages -for ncurses. It will, however, give you the basic ideas. - - -Starting and ending a curses application -======================================== - -Before doing anything, curses must be initialized. This is done by -calling the :func:`~curses.initscr` function, which will determine the -terminal type, send any required setup codes to the terminal, and -create various internal data structures. If successful, -:func:`initscr` returns a window object representing the entire -screen; this is usually called ``stdscr`` after the name of the -corresponding C variable. :: - - import curses - stdscr = curses.initscr() - -Usually curses applications turn off automatic echoing of keys to the -screen, in order to be able to read keys and only display them under -certain circumstances. This requires calling the -:func:`~curses.noecho` function. :: - - curses.noecho() - -Applications will also commonly need to react to keys instantly, -without requiring the Enter key to be pressed; this is called cbreak -mode, as opposed to the usual buffered input mode. :: - - curses.cbreak() - -Terminals usually return special keys, such as the cursor keys or navigation -keys such as Page Up and Home, as a multibyte escape sequence. While you could -write your application to expect such sequences and process them accordingly, -curses can do it for you, returning a special value such as -:const:`curses.KEY_LEFT`. To get curses to do the job, you'll have to enable -keypad mode. :: - - stdscr.keypad(True) - -Terminating a curses application is much easier than starting one. You'll need -to call:: - - curses.nocbreak() - stdscr.keypad(False) - curses.echo() - -to reverse the curses-friendly terminal settings. Then call the -:func:`~curses.endwin` function to restore the terminal to its original -operating mode. :: - - curses.endwin() - -A common problem when debugging a curses application is to get your terminal -messed up when the application dies without restoring the terminal to its -previous state. In Python this commonly happens when your code is buggy and -raises an uncaught exception. Keys are no longer echoed to the screen when -you type them, for example, which makes using the shell difficult. - -In Python you can avoid these complications and make debugging much easier by -importing the :func:`curses.wrapper` function and using it like this:: - - from curses import wrapper - - def main(stdscr): - # Clear screen - stdscr.clear() - - # This raises ZeroDivisionError when i == 10. - for i in range(0, 11): - v = i-10 - stdscr.addstr(i, 0, '10 divided by {} is {}'.format(v, 10/v)) - - stdscr.refresh() - stdscr.getkey() - - wrapper(main) - -The :func:`~curses.wrapper` function takes a callable object and does the -initializations described above, also initializing colors if color -support is present. :func:`wrapper` then runs your provided callable. -Once the callable returns, :func:`wrapper` will restore the original -state of the terminal. The callable is called inside a -:keyword:`try`...\ :keyword:`except` that catches exceptions, restores -the state of the terminal, and then re-raises the exception. Therefore -your terminal won't be left in a funny state on exception and you'll be -able to read the exception's message and traceback. - - -Windows and Pads -================ - -Windows are the basic abstraction in curses. A window object represents a -rectangular area of the screen, and supports methods to display text, -erase it, allow the user to input strings, and so forth. - -The ``stdscr`` object returned by the :func:`~curses.initscr` function is a -window object that covers the entire screen. Many programs may need -only this single window, but you might wish to divide the screen into -smaller windows, in order to redraw or clear them separately. The -:func:`~curses.newwin` function creates a new window of a given size, -returning the new window object. :: - - begin_x = 20; begin_y = 7 - height = 5; width = 40 - win = curses.newwin(height, width, begin_y, begin_x) - -Note that the coordinate system used in curses is unusual. -Coordinates are always passed in the order *y,x*, and the top-left -corner of a window is coordinate (0,0). This breaks the normal -convention for handling coordinates where the *x* coordinate comes -first. This is an unfortunate difference from most other computer -applications, but it's been part of curses since it was first written, -and it's too late to change things now. - -Your application can determine the size of the screen by using the -:data:`curses.LINES` and :data:`curses.COLS` variables to obtain the *y* and -*x* sizes. Legal coordinates will then extend from ``(0,0)`` to -``(curses.LINES - 1, curses.COLS - 1)``. - -When you call a method to display or erase text, the effect doesn't -immediately show up on the display. Instead you must call the -:meth:`~curses.window.refresh` method of window objects to update the -screen. - -This is because curses was originally written with slow 300-baud -terminal connections in mind; with these terminals, minimizing the -time required to redraw the screen was very important. Instead curses -accumulates changes to the screen and displays them in the most -efficient manner when you call :meth:`refresh`. For example, if your -program displays some text in a window and then clears the window, -there's no need to send the original text because they're never -visible. - -In practice, explicitly telling curses to redraw a window doesn't -really complicate programming with curses much. Most programs go into a flurry -of activity, and then pause waiting for a keypress or some other action on the -part of the user. All you have to do is to be sure that the screen has been -redrawn before pausing to wait for user input, by first calling -``stdscr.refresh()`` or the :meth:`refresh` method of some other relevant -window. - -A pad is a special case of a window; it can be larger than the actual display -screen, and only a portion of the pad displayed at a time. Creating a pad -requires the pad's height and width, while refreshing a pad requires giving the -coordinates of the on-screen area where a subsection of the pad will be -displayed. :: - - pad = curses.newpad(100, 100) - # These loops fill the pad with letters; addch() is - # explained in the next section - for y in range(0, 99): - for x in range(0, 99): - pad.addch(y,x, ord('a') + (x*x+y*y) % 26) - - # Displays a section of the pad in the middle of the screen. - # (0,0) : coordinate of upper-left corner of pad area to display. - # (5,5) : coordinate of upper-left corner of window area to be filled - # with pad content. - # (20, 75) : coordinate of lower-right corner of window area to be - # : filled with pad content. - pad.refresh( 0,0, 5,5, 20,75) - -The :meth:`refresh` call displays a section of the pad in the rectangle -extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper -left corner of the displayed section is coordinate (0,0) on the pad. Beyond -that difference, pads are exactly like ordinary windows and support the same -methods. - -If you have multiple windows and pads on screen there is a more -efficient way to update the screen and prevent annoying screen flicker -as each part of the screen gets updated. :meth:`refresh` actually -does two things: - -1) Calls the :meth:`~curses.window.noutrefresh` method of each window - to update an underlying data structure representing the desired - state of the screen. -2) Calls the function :func:`~curses.doupdate` function to change the - physical screen to match the desired state recorded in the data structure. - -Instead you can call :meth:`noutrefresh` on a number of windows to -update the data structure, and then call :func:`doupdate` to update -the screen. - - -Displaying Text -=============== - -From a C programmer's point of view, curses may sometimes look like a -twisty maze of functions, all subtly different. For example, -:c:func:`addstr` displays a string at the current cursor location in -the ``stdscr`` window, while :c:func:`mvaddstr` moves to a given y,x -coordinate first before displaying the string. :c:func:`waddstr` is just -like :c:func:`addstr`, but allows specifying a window to use instead of -using ``stdscr`` by default. :c:func:`mvwaddstr` allows specifying both -a window and a coordinate. - -Fortunately the Python interface hides all these details. ``stdscr`` -is a window object like any other, and methods such as -:meth:`~curses.window.addstr` accept multiple argument forms. Usually there -are four different forms. - -+---------------------------------+-----------------------------------------------+ -| Form | Description | -+=================================+===============================================+ -| *str* or *ch* | Display the string *str* or character *ch* at | -| | the current position | -+---------------------------------+-----------------------------------------------+ -| *str* or *ch*, *attr* | Display the string *str* or character *ch*, | -| | using attribute *attr* at the current | -| | position | -+---------------------------------+-----------------------------------------------+ -| *y*, *x*, *str* or *ch* | Move to position *y,x* within the window, and | -| | display *str* or *ch* | -+---------------------------------+-----------------------------------------------+ -| *y*, *x*, *str* or *ch*, *attr* | Move to position *y,x* within the window, and | -| | display *str* or *ch*, using attribute *attr* | -+---------------------------------+-----------------------------------------------+ - -Attributes allow displaying text in highlighted forms such as boldface, -underline, reverse code, or in color. They'll be explained in more detail in -the next subsection. - - -The :meth:`~curses.window.addstr` method takes a Python string or -bytestring as the value to be displayed. The contents of bytestrings -are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`encoding` attribute; this defaults to -the default system encoding as returned by -:func:`locale.getpreferredencoding`. - -The :meth:`~curses.window.addch` methods take a character, which can be -either a string of length 1, a bytestring of length 1, or an integer. - -Constants are provided for extension characters; these constants are -integers greater than 255. For example, :const:`ACS_PLMINUS` is a +/- -symbol, and :const:`ACS_ULCORNER` is the upper left corner of a box -(handy for drawing borders). You can also use the appropriate Unicode -character. - -Windows remember where the cursor was left after the last operation, so if you -leave out the *y,x* coordinates, the string or character will be displayed -wherever the last operation left off. You can also move the cursor with the -``move(y,x)`` method. Because some terminals always display a flashing cursor, -you may want to ensure that the cursor is positioned in some location where it -won't be distracting; it can be confusing to have the cursor blinking at some -apparently random location. - -If your application doesn't need a blinking cursor at all, you can -call ``curs_set(False)`` to make it invisible. For compatibility -with older curses versions, there's a ``leaveok(bool)`` function -that's a synonym for :func:`~curses.curs_set`. When *bool* is true, the -curses library will attempt to suppress the flashing cursor, and you -won't need to worry about leaving it in odd locations. - - -Attributes and Color --------------------- - -Characters can be displayed in different ways. Status lines in a text-based -application are commonly shown in reverse video, or a text viewer may need to -highlight certain words. curses supports this by allowing you to specify an -attribute for each cell on the screen. - -An attribute is an integer, each bit representing a different -attribute. You can try to display text with multiple attribute bits -set, but curses doesn't guarantee that all the possible combinations -are available, or that they're all visually distinct. That depends on -the ability of the terminal being used, so it's safest to stick to the -most commonly available attributes, listed here. - -+----------------------+--------------------------------------+ -| Attribute | Description | -+======================+======================================+ -| :const:`A_BLINK` | Blinking text | -+----------------------+--------------------------------------+ -| :const:`A_BOLD` | Extra bright or bold text | -+----------------------+--------------------------------------+ -| :const:`A_DIM` | Half bright text | -+----------------------+--------------------------------------+ -| :const:`A_REVERSE` | Reverse-video text | -+----------------------+--------------------------------------+ -| :const:`A_STANDOUT` | The best highlighting mode available | -+----------------------+--------------------------------------+ -| :const:`A_UNDERLINE` | Underlined text | -+----------------------+--------------------------------------+ - -So, to display a reverse-video status line on the top line of the screen, you -could code:: - - stdscr.addstr(0, 0, "Current mode: Typing mode", - curses.A_REVERSE) - stdscr.refresh() - -The curses library also supports color on those terminals that provide it. The -most common such terminal is probably the Linux console, followed by color -xterms. - -To use color, you must call the :func:`~curses.start_color` function soon -after calling :func:`~curses.initscr`, to initialize the default color set -(the :func:`curses.wrapper` function does this automatically). Once that's -done, the :func:`~curses.has_colors` function returns TRUE if the terminal -in use can -actually display color. (Note: curses uses the American spelling 'color', -instead of the Canadian/British spelling 'colour'. If you're used to the -British spelling, you'll have to resign yourself to misspelling it for the sake -of these functions.) - -The curses library maintains a finite number of color pairs, containing a -foreground (or text) color and a background color. You can get the attribute -value corresponding to a color pair with the :func:`~curses.color_pair` -function; this can be bitwise-OR'ed with other attributes such as -:const:`A_REVERSE`, but again, such combinations are not guaranteed to work -on all terminals. - -An example, which displays a line of text using color pair 1:: - - stdscr.addstr("Pretty text", curses.color_pair(1)) - stdscr.refresh() - -As I said before, a color pair consists of a foreground and background color. -The ``init_pair(n, f, b)`` function changes the definition of color pair *n*, to -foreground color f and background color b. Color pair 0 is hard-wired to white -on black, and cannot be changed. - -Colors are numbered, and :func:`start_color` initializes 8 basic -colors when it activates color mode. They are: 0:black, 1:red, -2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The :mod:`curses` -module defines named constants for each of these colors: -:const:`curses.COLOR_BLACK`, :const:`curses.COLOR_RED`, and so forth. - -Let's put all this together. To change color 1 to red text on a white -background, you would call:: - - curses.init_pair(1, curses.COLOR_RED, curses.COLOR_WHITE) - -When you change a color pair, any text already displayed using that color pair -will change to the new colors. You can also display new text in this color -with:: - - stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1)) - -Very fancy terminals can change the definitions of the actual colors to a given -RGB value. This lets you change color 1, which is usually red, to purple or -blue or any other color you like. Unfortunately, the Linux console doesn't -support this, so I'm unable to try it out, and can't provide any examples. You -can check if your terminal can do this by calling -:func:`~curses.can_change_color`, which returns ``True`` if the capability is -there. If you're lucky enough to have such a talented terminal, consult your -system's man pages for more information. - - -User Input -========== - -The C curses library offers only very simple input mechanisms. Python's -:mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid <https://pypi.org/project/urwid/>`_ have more extensive -collections of widgets.) - -There are two methods for getting input from a window: - -* :meth:`~curses.window.getch` refreshes the screen and then waits for - the user to hit a key, displaying the key if :func:`~curses.echo` has been - called earlier. You can optionally specify a coordinate to which - the cursor should be moved before pausing. - -* :meth:`~curses.window.getkey` does the same thing but converts the - integer to a string. Individual characters are returned as - 1-character strings, and special keys such as function keys return - longer strings containing a key name such as ``KEY_UP`` or ``^G``. - -It's possible to not wait for the user using the -:meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`getch` and :meth:`getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`getch` returns -``curses.ERR`` (a value of -1) and :meth:`getkey` raises an exception. -There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`getch`; if no input becomes -available within a specified delay (measured in tenths of a second), -curses raises an exception. - -The :meth:`getch` method returns an integer; if it's between 0 and 255, it -represents the ASCII code of the key pressed. Values greater than 255 are -special keys such as Page Up, Home, or the cursor keys. You can compare the -value returned to constants such as :const:`curses.KEY_PPAGE`, -:const:`curses.KEY_HOME`, or :const:`curses.KEY_LEFT`. The main loop of -your program may look something like this:: - - while True: - c = stdscr.getch() - if c == ord('p'): - PrintDocument() - elif c == ord('q'): - break # Exit the while loop - elif c == curses.KEY_HOME: - x = y = 0 - -The :mod:`curses.ascii` module supplies ASCII class membership functions that -take either integer or 1-character string arguments; these may be useful in -writing more readable tests for such loops. It also supplies -conversion functions that take either integer or 1-character-string arguments -and return the same type. For example, :func:`curses.ascii.ctrl` returns the -control character corresponding to its argument. - -There's also a method to retrieve an entire string, -:meth:`~curses.window.getstr`. It isn't used very often, because its -functionality is quite limited; the only editing keys available are -the backspace key and the Enter key, which terminates the string. It -can optionally be limited to a fixed number of characters. :: - - curses.echo() # Enable echoing of characters - - # Get a 15-character string, with the cursor on the top line - s = stdscr.getstr(0,0, 15) - -The :mod:`curses.textpad` module supplies a text box that supports an -Emacs-like set of keybindings. Various methods of the -:class:`~curses.textpad.Textbox` class support editing with input -validation and gathering the edit results either with or without -trailing spaces. Here's an example:: - - import curses - from curses.textpad import Textbox, rectangle - - def main(stdscr): - stdscr.addstr(0, 0, "Enter IM message: (hit Ctrl-G to send)") - - editwin = curses.newwin(5,30, 2,1) - rectangle(stdscr, 1,0, 1+5+1, 1+30+1) - stdscr.refresh() - - box = Textbox(editwin) - - # Let the user edit until Ctrl-G is struck. - box.edit() - - # Get resulting contents - message = box.gather() - -See the library documentation on :mod:`curses.textpad` for more details. - - -For More Information -==================== - -This HOWTO doesn't cover some advanced topics, such as reading the -contents of the screen or capturing mouse events from an xterm -instance, but the Python library page for the :mod:`curses` module is now -reasonably complete. You should browse it next. - -If you're in doubt about the detailed behavior of the curses -functions, consult the manual pages for your curses implementation, -whether it's ncurses or a proprietary Unix vendor's. The manual pages -will document any quirks, and provide complete lists of all the -functions, attributes, and :const:`ACS_\*` characters available to -you. - -Because the curses API is so large, some functions aren't supported in -the Python interface. Often this isn't because they're difficult to -implement, but because no one has needed them yet. Also, Python -doesn't yet support the menu library associated with ncurses. -Patches adding support for these would be welcome; see -`the Python Developer's Guide <https://devguide.python.org/>`_ to -learn more about submitting patches to Python. - -* `Writing Programs with NCURSES <http://invisible-island.net/ncurses/ncurses-intro.html>`_: - a lengthy tutorial for C programmers. -* `The ncurses man page <http://linux.die.net/man/3/ncurses>`_ -* `The ncurses FAQ <http://invisible-island.net/ncurses/ncurses.faq.html>`_ -* `"Use curses... don't swear" <https://www.youtube.com/watch?v=eN1eZtjLEnU>`_: - video of a PyCon 2013 talk on controlling terminals using curses or Urwid. -* `"Console Applications with Urwid" <http://www.pyvideo.org/video/1568/console-applications-with-urwid>`_: - video of a PyCon CA 2012 talk demonstrating some applications written using - Urwid. diff --git a/third_party/python/Doc/howto/descriptor.rst b/third_party/python/Doc/howto/descriptor.rst deleted file mode 100644 index 5e85a9aa6..000000000 --- a/third_party/python/Doc/howto/descriptor.rst +++ /dev/null @@ -1,443 +0,0 @@ -====================== -Descriptor HowTo Guide -====================== - -:Author: Raymond Hettinger -:Contact: <python at rcn dot com> - -.. Contents:: - -Abstract --------- - -Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Examines a custom descriptor and several built-in python descriptors -including functions, properties, static methods, and class methods. Shows how -each works by giving a pure Python equivalent and a sample application. - -Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works and an appreciation for the -elegance of its design. - - -Definition and Introduction ---------------------------- - -In general, a descriptor is an object attribute with "binding behavior", one -whose attribute access has been overridden by methods in the descriptor -protocol. Those methods are :meth:`__get__`, :meth:`__set__`, and -:meth:`__delete__`. If any of those methods are defined for an object, it is -said to be a descriptor. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the base classes of ``type(a)`` excluding metaclasses. If the -looked-up value is an object defining one of the descriptor methods, then Python -may override the default behavior and invoke the descriptor method instead. -Where this occurs in the precedence chain depends on which descriptor methods -were defined. - -Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and :func:`super()`. -They are used throughout Python itself to implement the new style classes -introduced in version 2.2. Descriptors simplify the underlying C-code and offer -a flexible set of new tools for everyday Python programs. - - -Descriptor Protocol -------------------- - -``descr.__get__(self, obj, type=None) --> value`` - -``descr.__set__(self, obj, value) --> None`` - -``descr.__delete__(self, obj) --> None`` - -That is all there is to it. Define any of these methods and an object is -considered a descriptor and can override default behavior upon being looked up -as an attribute. - -If an object defines both :meth:`__get__` and :meth:`__set__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are typically used for methods but other uses are -possible). - -Data and non-data descriptors differ in how overrides are calculated with -respect to entries in an instance's dictionary. If an instance's dictionary -has an entry with the same name as a data descriptor, the data descriptor -takes precedence. If an instance's dictionary has an entry with the same -name as a non-data descriptor, the dictionary entry takes precedence. - -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising -placeholder is enough to make it a data descriptor. - - -Invoking Descriptors --------------------- - -A descriptor can be called directly by its method name. For example, -``d.__get__(obj)``. - -Alternatively, it is more common for a descriptor to be invoked automatically -upon attribute access. For example, ``obj.d`` looks up ``d`` in the dictionary -of ``obj``. If ``d`` defines the method :meth:`__get__`, then ``d.__get__(obj)`` -is invoked according to the precedence rules listed below. - -The details of invocation depend on whether ``obj`` is an object or a class. - -For objects, the machinery is in :meth:`object.__getattribute__` which -transforms ``b.x`` into ``type(b).__dict__['x'].__get__(b, type(b))``. The -implementation works through a precedence chain that gives data descriptors -priority over instance variables, instance variables priority over non-data -descriptors, and assigns lowest priority to :meth:`__getattr__` if provided. -The full C implementation can be found in :c:func:`PyObject_GenericGetAttr()` in -:source:`Objects/object.c`. - -For classes, the machinery is in :meth:`type.__getattribute__` which transforms -``B.x`` into ``B.__dict__['x'].__get__(None, B)``. In pure Python, it looks -like:: - - def __getattribute__(self, key): - "Emulate type_getattro() in Objects/typeobject.c" - v = object.__getattribute__(self, key) - if hasattr(v, '__get__'): - return v.__get__(None, self) - return v - -The important points to remember are: - -* descriptors are invoked by the :meth:`__getattribute__` method -* overriding :meth:`__getattribute__` prevents automatic descriptor calls -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. -* data descriptors always override instance dictionaries. -* non-data descriptors may be overridden by instance dictionaries. - -The object returned by ``super()`` also has a custom :meth:`__getattribute__` -method for invoking descriptors. The call ``super(B, obj).m()`` searches -``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` -and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, -``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a -search using :meth:`object.__getattribute__`. - -The implementation details are in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. and a pure Python equivalent can be found in -`Guido's Tutorial`_. - -.. _`Guido's Tutorial`: https://www.python.org/download/releases/2.2.3/descrintro/#cooperation - -The details above show that the mechanism for descriptors is embedded in the -:meth:`__getattribute__()` methods for :class:`object`, :class:`type`, and -:func:`super`. Classes inherit this machinery when they derive from -:class:`object` or if they have a meta-class providing similar functionality. -Likewise, classes can turn-off descriptor invocation by overriding -:meth:`__getattribute__()`. - - -Descriptor Example ------------------- - -The following code creates a class whose objects are data descriptors which -print a message for each get or set. Overriding :meth:`__getattribute__` is -alternate approach that could do this for every attribute. However, this -descriptor is useful for monitoring just a few chosen attributes:: - - class RevealAccess(object): - """A data descriptor that sets and returns values - normally and prints a message logging their access. - """ - - def __init__(self, initval=None, name='var'): - self.val = initval - self.name = name - - def __get__(self, obj, objtype): - print('Retrieving', self.name) - return self.val - - def __set__(self, obj, val): - print('Updating', self.name) - self.val = val - - >>> class MyClass(object): - ... x = RevealAccess(10, 'var "x"') - ... y = 5 - ... - >>> m = MyClass() - >>> m.x - Retrieving var "x" - 10 - >>> m.x = 20 - Updating var "x" - >>> m.x - Retrieving var "x" - 20 - >>> m.y - 5 - -The protocol is simple and offers exciting possibilities. Several use cases are -so common that they have been packaged into individual function calls. -Properties, bound methods, static methods, and class methods are all -based on the descriptor protocol. - - -Properties ----------- - -Calling :func:`property` is a succinct way of building a data descriptor that -triggers function calls upon access to an attribute. Its signature is:: - - property(fget=None, fset=None, fdel=None, doc=None) -> property attribute - -The documentation shows a typical use to define a managed attribute ``x``:: - - class C(object): - def getx(self): return self.__x - def setx(self, value): self.__x = value - def delx(self): del self.__x - x = property(getx, setx, delx, "I'm the 'x' property.") - -To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent:: - - class Property(object): - "Emulate PyProperty_Type() in Objects/descrobject.c" - - def __init__(self, fget=None, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ - self.__doc__ = doc - - def __get__(self, obj, objtype=None): - if obj is None: - return self - if self.fget is None: - raise AttributeError("unreadable attribute") - return self.fget(obj) - - def __set__(self, obj, value): - if self.fset is None: - raise AttributeError("can't set attribute") - self.fset(obj, value) - - def __delete__(self, obj): - if self.fdel is None: - raise AttributeError("can't delete attribute") - self.fdel(obj) - - def getter(self, fget): - return type(self)(fget, self.fset, self.fdel, self.__doc__) - - def setter(self, fset): - return type(self)(self.fget, fset, self.fdel, self.__doc__) - - def deleter(self, fdel): - return type(self)(self.fget, self.fset, fdel, self.__doc__) - -The :func:`property` builtin helps whenever a user interface has granted -attribute access and then subsequent changes require the intervention of a -method. - -For instance, a spreadsheet class may grant access to a cell value through -``Cell('b10').value``. Subsequent improvements to the program require the cell -to be recalculated on every access; however, the programmer does not want to -affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor:: - - class Cell(object): - . . . - def getvalue(self): - "Recalculate the cell before returning value" - self.recalc() - return self._value - value = property(getvalue) - - -Functions and Methods ---------------------- - -Python's object oriented features are built upon a function based environment. -Using non-data descriptors, the two are merged seamlessly. - -Class dictionaries store methods as functions. In a class definition, methods -are written using :keyword:`def` or :keyword:`lambda`, the usual tools for -creating functions. Methods only differ from regular functions in that the -first argument is reserved for the object instance. By Python convention, the -instance reference is called *self* but may be called *this* or any other -variable name. - -To support method calls, functions include the :meth:`__get__` method for -binding methods during attribute access. This means that all functions are -non-data descriptors which return bound methods when they are invoked from an -object. In pure python, it works like this:: - - class Function(object): - . . . - def __get__(self, obj, objtype=None): - "Simulate func_descr_get() in Objects/funcobject.c" - if obj is None: - return self - return types.MethodType(self, obj) - -Running the interpreter shows how the function descriptor works in practice:: - - >>> class D(object): - ... def f(self, x): - ... return x - ... - >>> d = D() - - # Access through the class dictionary does not invoke __get__. - # It just returns the underlying function object. - >>> D.__dict__['f'] - <function D.f at 0x00C45070> - - # Dotted access from a class calls __get__() which just returns - # the underlying function unchanged. - >>> D.f - <function D.f at 0x00C45070> - - # The function has a __qualname__ attribute to support introspection - >>> D.f.__qualname__ - 'D.f' - - # Dotted access from an instance calls __get__() which returns the - # function wrapped in a bound method object - >>> d.f - <bound method D.f of <__main__.D object at 0x00B18C90>> - - # Internally, the bound method stores the underlying function, - # the bound instance, and the class of the bound instance. - >>> d.f.__func__ - <function D.f at 0x1012e5ae8> - >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> - >>> d.f.__class__ - <class 'method'> - - -Static Methods and Class Methods --------------------------------- - -Non-data descriptors provide a simple mechanism for variations on the usual -patterns of binding functions into methods. - -To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``klass.f(*args)`` -becomes ``f(*args)``. - -This chart summarizes the binding and its two most useful variants: - - +-----------------+----------------------+------------------+ - | Transformation | Called from an | Called from a | - | | Object | Class | - +=================+======================+==================+ - | function | f(obj, \*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | staticmethod | f(\*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(klass, \*args) | - +-----------------+----------------------+------------------+ - -Static methods return the underlying function without changes. Calling either -``c.f`` or ``C.f`` is the equivalent of a direct lookup into -``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a -result, the function becomes identically accessible from either an object or a -class. - -Good candidates for static methods are methods that do not reference the -``self`` variable. - -For instance, a statistics package may include a container class for -experimental data. The class provides normal methods for computing the average, -mean, median, and other descriptive statistics that depend on the data. However, -there may be useful functions which are conceptually related but do not depend -on the data. For instance, ``erf(x)`` is handy conversion routine that comes up -in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. - -Since staticmethods return the underlying function with no changes, the example -calls are unexciting:: - - >>> class E(object): - ... def f(x): - ... print(x) - ... f = staticmethod(f) - ... - >>> print(E.f(3)) - 3 - >>> print(E().f(3)) - 3 - -Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this:: - - class StaticMethod(object): - "Emulate PyStaticMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, objtype=None): - return self.f - -Unlike static methods, class methods prepend the class reference to the -argument list before calling the function. This format is the same -for whether the caller is an object or a class:: - - >>> class E(object): - ... def f(klass, x): - ... return klass.__name__, x - ... f = classmethod(f) - ... - >>> print(E.f(3)) - ('E', 3) - >>> print(E().f(3)) - ('E', 3) - - -This behavior is useful whenever the function only needs to have a class -reference and does not care about any underlying data. One use for classmethods -is to create alternate class constructors. In Python 2.3, the classmethod -:func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure -Python equivalent is:: - - class Dict(object): - . . . - def fromkeys(klass, iterable, value=None): - "Emulate dict_fromkeys() in Objects/dictobject.c" - d = klass() - for key in iterable: - d[key] = value - return d - fromkeys = classmethod(fromkeys) - -Now a new dictionary of unique keys can be constructed like this:: - - >>> Dict.fromkeys('abracadabra') - {'a': None, 'r': None, 'b': None, 'c': None, 'd': None} - -Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this:: - - class ClassMethod(object): - "Emulate PyClassMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, klass=None): - if klass is None: - klass = type(obj) - def newfunc(*args): - return self.f(klass, *args) - return newfunc - diff --git a/third_party/python/Doc/howto/functional.rst b/third_party/python/Doc/howto/functional.rst deleted file mode 100644 index 7e21aa76a..000000000 --- a/third_party/python/Doc/howto/functional.rst +++ /dev/null @@ -1,1261 +0,0 @@ -******************************** - Functional Programming HOWTO -******************************** - -:Author: A. M. Kuchling -:Release: 0.32 - -In this document, we'll take a tour of Python's features suitable for -implementing programs in a functional style. After an introduction to the -concepts of functional programming, we'll look at language features such as -:term:`iterator`\s and :term:`generator`\s and relevant library modules such as -:mod:`itertools` and :mod:`functools`. - - -Introduction -============ - -This section explains the basic concept of functional programming; if -you're just interested in learning about Python language features, -skip to the next section on :ref:`functional-howto-iterators`. - -Programming languages support decomposing problems in several different ways: - -* Most programming languages are **procedural**: programs are lists of - instructions that tell the computer what to do with the program's input. C, - Pascal, and even Unix shells are procedural languages. - -* In **declarative** languages, you write a specification that describes the - problem to be solved, and the language implementation figures out how to - perform the computation efficiently. SQL is the declarative language you're - most likely to be familiar with; a SQL query describes the data set you want - to retrieve, and the SQL engine decides whether to scan tables or use indexes, - which subclauses should be performed first, etc. - -* **Object-oriented** programs manipulate collections of objects. Objects have - internal state and support methods that query or modify this internal state in - some way. Smalltalk and Java are object-oriented languages. C++ and Python - are languages that support object-oriented programming, but don't force the - use of object-oriented features. - -* **Functional** programming decomposes a problem into a set of functions. - Ideally, functions only take inputs and produce outputs, and don't have any - internal state that affects the output produced for a given input. Well-known - functional languages include the ML family (Standard ML, OCaml, and other - variants) and Haskell. - -The designers of some computer languages choose to emphasize one -particular approach to programming. This often makes it difficult to -write programs that use a different approach. Other languages are -multi-paradigm languages that support several different approaches. -Lisp, C++, and Python are multi-paradigm; you can write programs or -libraries that are largely procedural, object-oriented, or functional -in all of these languages. In a large program, different sections -might be written using different approaches; the GUI might be -object-oriented while the processing logic is procedural or -functional, for example. - -In a functional program, input flows through a set of functions. Each function -operates on its input and produces some output. Functional style discourages -functions with side effects that modify internal state or make other changes -that aren't visible in the function's return value. Functions that have no side -effects at all are called **purely functional**. Avoiding side effects means -not using data structures that get updated as a program runs; every function's -output must only depend on its input. - -Some languages are very strict about purity and don't even have assignment -statements such as ``a=3`` or ``c = a + b``, but it's difficult to avoid all -side effects. Printing to the screen or writing to a disk file are side -effects, for example. For example, in Python a call to the :func:`print` or -:func:`time.sleep` function both return no useful value; they're only called for -their side effects of sending some text to the screen or pausing execution for a -second. - -Python programs written in functional style usually won't go to the extreme of -avoiding all I/O or all assignments; instead, they'll provide a -functional-appearing interface but will use non-functional features internally. -For example, the implementation of a function will still use assignments to -local variables, but won't modify global variables or have other side effects. - -Functional programming can be considered the opposite of object-oriented -programming. Objects are little capsules containing some internal state along -with a collection of method calls that let you modify this state, and programs -consist of making the right set of state changes. Functional programming wants -to avoid state changes as much as possible and works with data flowing between -functions. In Python you might combine the two approaches by writing functions -that take and return instances representing objects in your application (e-mail -messages, transactions, etc.). - -Functional design may seem like an odd constraint to work under. Why should you -avoid objects and side effects? There are theoretical and practical advantages -to the functional style: - -* Formal provability. -* Modularity. -* Composability. -* Ease of debugging and testing. - - -Formal provability ------------------- - -A theoretical benefit is that it's easier to construct a mathematical proof that -a functional program is correct. - -For a long time researchers have been interested in finding ways to -mathematically prove programs correct. This is different from testing a program -on numerous inputs and concluding that its output is usually correct, or reading -a program's source code and concluding that the code looks right; the goal is -instead a rigorous proof that a program produces the right result for all -possible inputs. - -The technique used to prove programs correct is to write down **invariants**, -properties of the input data and of the program's variables that are always -true. For each line of code, you then show that if invariants X and Y are true -**before** the line is executed, the slightly different invariants X' and Y' are -true **after** the line is executed. This continues until you reach the end of -the program, at which point the invariants should match the desired conditions -on the program's output. - -Functional programming's avoidance of assignments arose because assignments are -difficult to handle with this technique; assignments can break invariants that -were true before the assignment without producing any new invariants that can be -propagated onward. - -Unfortunately, proving programs correct is largely impractical and not relevant -to Python software. Even trivial programs require proofs that are several pages -long; the proof of correctness for a moderately complicated program would be -enormous, and few or none of the programs you use daily (the Python interpreter, -your XML parser, your web browser) could be proven correct. Even if you wrote -down or generated a proof, there would then be the question of verifying the -proof; maybe there's an error in it, and you wrongly believe you've proved the -program correct. - - -Modularity ----------- - -A more practical benefit of functional programming is that it forces you to -break apart your problem into small pieces. Programs are more modular as a -result. It's easier to specify and write a small function that does one thing -than a large function that performs a complicated transformation. Small -functions are also easier to read and to check for errors. - - -Ease of debugging and testing ------------------------------ - -Testing and debugging a functional-style program is easier. - -Debugging is simplified because functions are generally small and clearly -specified. When a program doesn't work, each function is an interface point -where you can check that the data are correct. You can look at the intermediate -inputs and outputs to quickly isolate the function that's responsible for a bug. - -Testing is easier because each function is a potential subject for a unit test. -Functions don't depend on system state that needs to be replicated before -running a test; instead you only have to synthesize the right input and then -check that the output matches expectations. - - -Composability -------------- - -As you work on a functional-style program, you'll write a number of functions -with varying inputs and outputs. Some of these functions will be unavoidably -specialized to a particular application, but others will be useful in a wide -variety of programs. For example, a function that takes a directory path and -returns all the XML files in the directory, or a function that takes a filename -and returns its contents, can be applied to many different situations. - -Over time you'll form a personal library of utilities. Often you'll assemble -new programs by arranging existing functions in a new configuration and writing -a few functions specialized for the current task. - - -.. _functional-howto-iterators: - -Iterators -========= - -I'll start by looking at a Python language feature that's an important -foundation for writing functional-style programs: iterators. - -An iterator is an object representing a stream of data; this object returns the -data one element at a time. A Python iterator must support a method called -:meth:`~iterator.__next__` that takes no arguments and always returns the next -element of the stream. If there are no more elements in the stream, -:meth:`~iterator.__next__` must raise the :exc:`StopIteration` exception. -Iterators don't have to be finite, though; it's perfectly reasonable to write -an iterator that produces an infinite stream of data. - -The built-in :func:`iter` function takes an arbitrary object and tries to return -an iterator that will return the object's contents or elements, raising -:exc:`TypeError` if the object doesn't support iteration. Several of Python's -built-in data types support iteration, the most common being lists and -dictionaries. An object is called :term:`iterable` if you can get an iterator -for it. - -You can experiment with the iteration interface manually: - - >>> L = [1, 2, 3] - >>> it = iter(L) - >>> it #doctest: +ELLIPSIS - <...iterator object at ...> - >>> it.__next__() # same as next(it) - 1 - >>> next(it) - 2 - >>> next(it) - 3 - >>> next(it) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - >>> - -Python expects iterable objects in several different contexts, the most -important being the :keyword:`for` statement. In the statement ``for X in Y``, -Y must be an iterator or some object for which :func:`iter` can create an -iterator. These two statements are equivalent:: - - - for i in iter(obj): - print(i) - - for i in obj: - print(i) - -Iterators can be materialized as lists or tuples by using the :func:`list` or -:func:`tuple` constructor functions: - - >>> L = [1, 2, 3] - >>> iterator = iter(L) - >>> t = tuple(iterator) - >>> t - (1, 2, 3) - -Sequence unpacking also supports iterators: if you know an iterator will return -N elements, you can unpack them into an N-tuple: - - >>> L = [1, 2, 3] - >>> iterator = iter(L) - >>> a, b, c = iterator - >>> a, b, c - (1, 2, 3) - -Built-in functions such as :func:`max` and :func:`min` can take a single -iterator argument and will return the largest or smallest element. The ``"in"`` -and ``"not in"`` operators also support iterators: ``X in iterator`` is true if -X is found in the stream returned by the iterator. You'll run into obvious -problems if the iterator is infinite; :func:`max`, :func:`min` -will never return, and if the element X never appears in the stream, the -``"in"`` and ``"not in"`` operators won't return either. - -Note that you can only go forward in an iterator; there's no way to get the -previous element, reset the iterator, or make a copy of it. Iterator objects -can optionally provide these additional capabilities, but the iterator protocol -only specifies the :meth:`~iterator.__next__` method. Functions may therefore -consume all of the iterator's output, and if you need to do something different -with the same stream, you'll have to create a new iterator. - - - -Data Types That Support Iterators ---------------------------------- - -We've already seen how lists and tuples support iterators. In fact, any Python -sequence type, such as strings, will automatically support creation of an -iterator. - -Calling :func:`iter` on a dictionary returns an iterator that will loop over the -dictionary's keys:: - - >>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, - ... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} - >>> for key in m: #doctest: +SKIP - ... print(key, m[key]) - Mar 3 - Feb 2 - Aug 8 - Sep 9 - Apr 4 - Jun 6 - Jul 7 - Jan 1 - May 5 - Nov 11 - Dec 12 - Oct 10 - -Note that the order is essentially random, because it's based on the hash -ordering of the objects in the dictionary. - -Applying :func:`iter` to a dictionary always loops over the keys, but -dictionaries have methods that return other iterators. If you want to iterate -over values or key/value pairs, you can explicitly call the -:meth:`~dict.values` or :meth:`~dict.items` methods to get an appropriate -iterator. - -The :func:`dict` constructor can accept an iterator that returns a finite stream -of ``(key, value)`` tuples: - - >>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')] - >>> dict(iter(L)) #doctest: +SKIP - {'Italy': 'Rome', 'US': 'Washington DC', 'France': 'Paris'} - -Files also support iteration by calling the :meth:`~io.TextIOBase.readline` -method until there are no more lines in the file. This means you can read each -line of a file like this:: - - for line in file: - # do something for each line - ... - -Sets can take their contents from an iterable and let you iterate over the set's -elements:: - - S = {2, 3, 5, 7, 11, 13} - for i in S: - print(i) - - - -Generator expressions and list comprehensions -============================================= - -Two common operations on an iterator's output are 1) performing some operation -for every element, 2) selecting a subset of elements that meet some condition. -For example, given a list of strings, you might want to strip off trailing -whitespace from each line or extract all the strings containing a given -substring. - -List comprehensions and generator expressions (short form: "listcomps" and -"genexps") are a concise notation for such operations, borrowed from the -functional programming language Haskell (https://www.haskell.org/). You can strip -all the whitespace from a stream of strings with the following code:: - - line_list = [' line 1\n', 'line 2 \n', ...] - - # Generator expression -- returns iterator - stripped_iter = (line.strip() for line in line_list) - - # List comprehension -- returns list - stripped_list = [line.strip() for line in line_list] - -You can select only certain elements by adding an ``"if"`` condition:: - - stripped_list = [line.strip() for line in line_list - if line != ""] - -With a list comprehension, you get back a Python list; ``stripped_list`` is a -list containing the resulting lines, not an iterator. Generator expressions -return an iterator that computes the values as necessary, not needing to -materialize all the values at once. This means that list comprehensions aren't -useful if you're working with iterators that return an infinite stream or a very -large amount of data. Generator expressions are preferable in these situations. - -Generator expressions are surrounded by parentheses ("()") and list -comprehensions are surrounded by square brackets ("[]"). Generator expressions -have the form:: - - ( expression for expr in sequence1 - if condition1 - for expr2 in sequence2 - if condition2 - for expr3 in sequence3 ... - if condition3 - for exprN in sequenceN - if conditionN ) - -Again, for a list comprehension only the outside brackets are different (square -brackets instead of parentheses). - -The elements of the generated output will be the successive values of -``expression``. The ``if`` clauses are all optional; if present, ``expression`` -is only evaluated and added to the result when ``condition`` is true. - -Generator expressions always have to be written inside parentheses, but the -parentheses signalling a function call also count. If you want to create an -iterator that will be immediately passed to a function you can write:: - - obj_total = sum(obj.count for obj in list_all_objects()) - -The ``for...in`` clauses contain the sequences to be iterated over. The -sequences do not have to be the same length, because they are iterated over from -left to right, **not** in parallel. For each element in ``sequence1``, -``sequence2`` is looped over from the beginning. ``sequence3`` is then looped -over for each resulting pair of elements from ``sequence1`` and ``sequence2``. - -To put it another way, a list comprehension or generator expression is -equivalent to the following Python code:: - - for expr1 in sequence1: - if not (condition1): - continue # Skip this element - for expr2 in sequence2: - if not (condition2): - continue # Skip this element - ... - for exprN in sequenceN: - if not (conditionN): - continue # Skip this element - - # Output the value of - # the expression. - -This means that when there are multiple ``for...in`` clauses but no ``if`` -clauses, the length of the resulting output will be equal to the product of the -lengths of all the sequences. If you have two lists of length 3, the output -list is 9 elements long: - - >>> seq1 = 'abc' - >>> seq2 = (1, 2, 3) - >>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE - [('a', 1), ('a', 2), ('a', 3), - ('b', 1), ('b', 2), ('b', 3), - ('c', 1), ('c', 2), ('c', 3)] - -To avoid introducing an ambiguity into Python's grammar, if ``expression`` is -creating a tuple, it must be surrounded with parentheses. The first list -comprehension below is a syntax error, while the second one is correct:: - - # Syntax error - [x, y for x in seq1 for y in seq2] - # Correct - [(x, y) for x in seq1 for y in seq2] - - -Generators -========== - -Generators are a special class of functions that simplify the task of writing -iterators. Regular functions compute a value and return it, but generators -return an iterator that returns a stream of values. - -You're doubtless familiar with how regular function calls work in Python or C. -When you call a function, it gets a private namespace where its local variables -are created. When the function reaches a ``return`` statement, the local -variables are destroyed and the value is returned to the caller. A later call -to the same function creates a new private namespace and a fresh set of local -variables. But, what if the local variables weren't thrown away on exiting a -function? What if you could later resume the function where it left off? This -is what generators provide; they can be thought of as resumable functions. - -Here's the simplest example of a generator function: - - >>> def generate_ints(N): - ... for i in range(N): - ... yield i - -Any function containing a :keyword:`yield` keyword is a generator function; -this is detected by Python's :term:`bytecode` compiler which compiles the -function specially as a result. - -When you call a generator function, it doesn't return a single value; instead it -returns a generator object that supports the iterator protocol. On executing -the ``yield`` expression, the generator outputs the value of ``i``, similar to a -``return`` statement. The big difference between ``yield`` and a ``return`` -statement is that on reaching a ``yield`` the generator's state of execution is -suspended and local variables are preserved. On the next call to the -generator's :meth:`~generator.__next__` method, the function will resume -executing. - -Here's a sample usage of the ``generate_ints()`` generator: - - >>> gen = generate_ints(3) - >>> gen #doctest: +ELLIPSIS - <generator object generate_ints at ...> - >>> next(gen) - 0 - >>> next(gen) - 1 - >>> next(gen) - 2 - >>> next(gen) - Traceback (most recent call last): - File "stdin", line 1, in <module> - File "stdin", line 2, in generate_ints - StopIteration - -You could equally write ``for i in generate_ints(5)``, or ``a, b, c = -generate_ints(3)``. - -Inside a generator function, ``return value`` causes ``StopIteration(value)`` -to be raised from the :meth:`~generator.__next__` method. Once this happens, or -the bottom of the function is reached, the procession of values ends and the -generator cannot yield any further values. - -You could achieve the effect of generators manually by writing your own class -and storing all the local variables of the generator as instance variables. For -example, returning a list of integers could be done by setting ``self.count`` to -0, and having the :meth:`~iterator.__next__` method increment ``self.count`` and -return it. -However, for a moderately complicated generator, writing a corresponding class -can be much messier. - -The test suite included with Python's library, -:source:`Lib/test/test_generators.py`, contains -a number of more interesting examples. Here's one generator that implements an -in-order traversal of a tree using generators recursively. :: - - # A recursive generator that generates Tree leaves in in-order. - def inorder(t): - if t: - for x in inorder(t.left): - yield x - - yield t.label - - for x in inorder(t.right): - yield x - -Two other examples in ``test_generators.py`` produce solutions for the N-Queens -problem (placing N queens on an NxN chess board so that no queen threatens -another) and the Knight's Tour (finding a route that takes a knight to every -square of an NxN chessboard without visiting any square twice). - - - -Passing values into a generator -------------------------------- - -In Python 2.4 and earlier, generators only produced output. Once a generator's -code was invoked to create an iterator, there was no way to pass any new -information into the function when its execution is resumed. You could hack -together this ability by making the generator look at a global variable or by -passing in some mutable object that callers then modify, but these approaches -are messy. - -In Python 2.5 there's a simple way to pass values into a generator. -:keyword:`yield` became an expression, returning a value that can be assigned to -a variable or otherwise operated on:: - - val = (yield i) - -I recommend that you **always** put parentheses around a ``yield`` expression -when you're doing something with the returned value, as in the above example. -The parentheses aren't always necessary, but it's easier to always add them -instead of having to remember when they're needed. - -(:pep:`342` explains the exact rules, which are that a ``yield``-expression must -always be parenthesized except when it occurs at the top-level expression on the -right-hand side of an assignment. This means you can write ``val = yield i`` -but have to use parentheses when there's an operation, as in ``val = (yield i) -+ 12``.) - -Values are sent into a generator by calling its :meth:`send(value) -<generator.send>` method. This method resumes the generator's code and the -``yield`` expression returns the specified value. If the regular -:meth:`~generator.__next__` method is called, the ``yield`` returns ``None``. - -Here's a simple counter that increments by 1 and allows changing the value of -the internal counter. - -.. testcode:: - - def counter(maximum): - i = 0 - while i < maximum: - val = (yield i) - # If value provided, change counter - if val is not None: - i = val - else: - i += 1 - -And here's an example of changing the counter: - - >>> it = counter(10) #doctest: +SKIP - >>> next(it) #doctest: +SKIP - 0 - >>> next(it) #doctest: +SKIP - 1 - >>> it.send(8) #doctest: +SKIP - 8 - >>> next(it) #doctest: +SKIP - 9 - >>> next(it) #doctest: +SKIP - Traceback (most recent call last): - File "t.py", line 15, in <module> - it.next() - StopIteration - -Because ``yield`` will often be returning ``None``, you should always check for -this case. Don't just use its value in expressions unless you're sure that the -:meth:`~generator.send` method will be the only method used to resume your -generator function. - -In addition to :meth:`~generator.send`, there are two other methods on -generators: - -* :meth:`throw(type, value=None, traceback=None) <generator.throw>` is used to - raise an exception inside the generator; the exception is raised by the - ``yield`` expression where the generator's execution is paused. - -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the - generator to terminate the iteration. On receiving this exception, the - generator's code must either raise :exc:`GeneratorExit` or - :exc:`StopIteration`; catching the exception and doing anything else is - illegal and will trigger a :exc:`RuntimeError`. :meth:`~generator.close` - will also be called by Python's garbage collector when the generator is - garbage-collected. - - If you need to run cleanup code when a :exc:`GeneratorExit` occurs, I suggest - using a ``try: ... finally:`` suite instead of catching :exc:`GeneratorExit`. - -The cumulative effect of these changes is to turn generators from one-way -producers of information into both producers and consumers. - -Generators also become **coroutines**, a more generalized form of subroutines. -Subroutines are entered at one point and exited at another point (the top of the -function, and a ``return`` statement), but coroutines can be entered, exited, -and resumed at many different points (the ``yield`` statements). - - -Built-in functions -================== - -Let's look in more detail at built-in functions often used with iterators. - -Two of Python's built-in functions, :func:`map` and :func:`filter` duplicate the -features of generator expressions: - -:func:`map(f, iterA, iterB, ...) <map>` returns an iterator over the sequence - ``f(iterA[0], iterB[0]), f(iterA[1], iterB[1]), f(iterA[2], iterB[2]), ...``. - - >>> def upper(s): - ... return s.upper() - - >>> list(map(upper, ['sentence', 'fragment'])) - ['SENTENCE', 'FRAGMENT'] - >>> [upper(s) for s in ['sentence', 'fragment']] - ['SENTENCE', 'FRAGMENT'] - -You can of course achieve the same effect with a list comprehension. - -:func:`filter(predicate, iter) <filter>` returns an iterator over all the -sequence elements that meet a certain condition, and is similarly duplicated by -list comprehensions. A **predicate** is a function that returns the truth -value of some condition; for use with :func:`filter`, the predicate must take a -single value. - - >>> def is_even(x): - ... return (x % 2) == 0 - - >>> list(filter(is_even, range(10))) - [0, 2, 4, 6, 8] - - -This can also be written as a list comprehension: - - >>> list(x for x in range(10) if is_even(x)) - [0, 2, 4, 6, 8] - - -:func:`enumerate(iter, start=0) <enumerate>` counts off the elements in the -iterable returning 2-tuples containing the count (from *start*) and -each element. :: - - >>> for item in enumerate(['subject', 'verb', 'object']): - ... print(item) - (0, 'subject') - (1, 'verb') - (2, 'object') - -:func:`enumerate` is often used when looping through a list and recording the -indexes at which certain conditions are met:: - - f = open('data.txt', 'r') - for i, line in enumerate(f): - if line.strip() == '': - print('Blank line at line #%i' % i) - -:func:`sorted(iterable, key=None, reverse=False) <sorted>` collects all the -elements of the iterable into a list, sorts the list, and returns the sorted -result. The *key* and *reverse* arguments are passed through to the -constructed list's :meth:`~list.sort` method. :: - - >>> import random - >>> # Generate 8 random numbers between [0, 10000) - >>> rand_list = random.sample(range(10000), 8) - >>> rand_list #doctest: +SKIP - [769, 7953, 9828, 6431, 8442, 9878, 6213, 2207] - >>> sorted(rand_list) #doctest: +SKIP - [769, 2207, 6213, 6431, 7953, 8442, 9828, 9878] - >>> sorted(rand_list, reverse=True) #doctest: +SKIP - [9878, 9828, 8442, 7953, 6431, 6213, 2207, 769] - -(For a more detailed discussion of sorting, see the :ref:`sortinghowto`.) - - -The :func:`any(iter) <any>` and :func:`all(iter) <all>` built-ins look at the -truth values of an iterable's contents. :func:`any` returns ``True`` if any element -in the iterable is a true value, and :func:`all` returns ``True`` if all of the -elements are true values: - - >>> any([0, 1, 0]) - True - >>> any([0, 0, 0]) - False - >>> any([1, 1, 1]) - True - >>> all([0, 1, 0]) - False - >>> all([0, 0, 0]) - False - >>> all([1, 1, 1]) - True - - -:func:`zip(iterA, iterB, ...) <zip>` takes one element from each iterable and -returns them in a tuple:: - - zip(['a', 'b', 'c'], (1, 2, 3)) => - ('a', 1), ('b', 2), ('c', 3) - -It doesn't construct an in-memory list and exhaust all the input iterators -before returning; instead tuples are constructed and returned only if they're -requested. (The technical term for this behaviour is `lazy evaluation -<https://en.wikipedia.org/wiki/Lazy_evaluation>`__.) - -This iterator is intended to be used with iterables that are all of the same -length. If the iterables are of different lengths, the resulting stream will be -the same length as the shortest iterable. :: - - zip(['a', 'b'], (1, 2, 3)) => - ('a', 1), ('b', 2) - -You should avoid doing this, though, because an element may be taken from the -longer iterators and discarded. This means you can't go on to use the iterators -further because you risk skipping a discarded element. - - -The itertools module -==================== - -The :mod:`itertools` module contains a number of commonly-used iterators as well -as functions for combining several iterators. This section will introduce the -module's contents by showing small examples. - -The module's functions fall into a few broad classes: - -* Functions that create a new iterator based on an existing iterator. -* Functions for treating an iterator's elements as function arguments. -* Functions for selecting portions of an iterator's output. -* A function for grouping an iterator's output. - -Creating new iterators ----------------------- - -:func:`itertools.count(start, step) <itertools.count>` returns an infinite -stream of evenly spaced values. You can optionally supply the starting number, -which defaults to 0, and the interval between numbers, which defaults to 1:: - - itertools.count() => - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - itertools.count(10) => - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... - itertools.count(10, 5) => - 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... - -:func:`itertools.cycle(iter) <itertools.cycle>` saves a copy of the contents of -a provided iterable and returns a new iterator that returns its elements from -first to last. The new iterator will repeat these elements infinitely. :: - - itertools.cycle([1, 2, 3, 4, 5]) => - 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ... - -:func:`itertools.repeat(elem, [n]) <itertools.repeat>` returns the provided -element *n* times, or returns the element endlessly if *n* is not provided. :: - - itertools.repeat('abc') => - abc, abc, abc, abc, abc, abc, abc, abc, abc, abc, ... - itertools.repeat('abc', 5) => - abc, abc, abc, abc, abc - -:func:`itertools.chain(iterA, iterB, ...) <itertools.chain>` takes an arbitrary -number of iterables as input, and returns all the elements of the first -iterator, then all the elements of the second, and so on, until all of the -iterables have been exhausted. :: - - itertools.chain(['a', 'b', 'c'], (1, 2, 3)) => - a, b, c, 1, 2, 3 - -:func:`itertools.islice(iter, [start], stop, [step]) <itertools.islice>` returns -a stream that's a slice of the iterator. With a single *stop* argument, it -will return the first *stop* elements. If you supply a starting index, you'll -get *stop-start* elements, and if you supply a value for *step*, elements -will be skipped accordingly. Unlike Python's string and list slicing, you can't -use negative values for *start*, *stop*, or *step*. :: - - itertools.islice(range(10), 8) => - 0, 1, 2, 3, 4, 5, 6, 7 - itertools.islice(range(10), 2, 8) => - 2, 3, 4, 5, 6, 7 - itertools.islice(range(10), 2, 8, 2) => - 2, 4, 6 - -:func:`itertools.tee(iter, [n]) <itertools.tee>` replicates an iterator; it -returns *n* independent iterators that will all return the contents of the -source iterator. -If you don't supply a value for *n*, the default is 2. Replicating iterators -requires saving some of the contents of the source iterator, so this can consume -significant memory if the iterator is large and one of the new iterators is -consumed more than the others. :: - - itertools.tee( itertools.count() ) => - iterA, iterB - - where iterA -> - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - - and iterB -> - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - - -Calling functions on elements ------------------------------ - -The :mod:`operator` module contains a set of functions corresponding to Python's -operators. Some examples are :func:`operator.add(a, b) <operator.add>` (adds -two values), :func:`operator.ne(a, b) <operator.ne>` (same as ``a != b``), and -:func:`operator.attrgetter('id') <operator.attrgetter>` -(returns a callable that fetches the ``.id`` attribute). - -:func:`itertools.starmap(func, iter) <itertools.starmap>` assumes that the -iterable will return a stream of tuples, and calls *func* using these tuples as -the arguments:: - - itertools.starmap(os.path.join, - [('/bin', 'python'), ('/usr', 'bin', 'java'), - ('/usr', 'bin', 'perl'), ('/usr', 'bin', 'ruby')]) - => - /bin/python, /usr/bin/java, /usr/bin/perl, /usr/bin/ruby - - -Selecting elements ------------------- - -Another group of functions chooses a subset of an iterator's elements based on a -predicate. - -:func:`itertools.filterfalse(predicate, iter) <itertools.filterfalse>` is the -opposite of :func:`filter`, returning all elements for which the predicate -returns false:: - - itertools.filterfalse(is_even, itertools.count()) => - 1, 3, 5, 7, 9, 11, 13, 15, ... - -:func:`itertools.takewhile(predicate, iter) <itertools.takewhile>` returns -elements for as long as the predicate returns true. Once the predicate returns -false, the iterator will signal the end of its results. :: - - def less_than_10(x): - return x < 10 - - itertools.takewhile(less_than_10, itertools.count()) => - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 - - itertools.takewhile(is_even, itertools.count()) => - 0 - -:func:`itertools.dropwhile(predicate, iter) <itertools.dropwhile>` discards -elements while the predicate returns true, and then returns the rest of the -iterable's results. :: - - itertools.dropwhile(less_than_10, itertools.count()) => - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... - - itertools.dropwhile(is_even, itertools.count()) => - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ... - -:func:`itertools.compress(data, selectors) <itertools.compress>` takes two -iterators and returns only those elements of *data* for which the corresponding -element of *selectors* is true, stopping whenever either one is exhausted:: - - itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) => - 1, 2, 5 - - -Combinatoric functions ----------------------- - -The :func:`itertools.combinations(iterable, r) <itertools.combinations>` -returns an iterator giving all possible *r*-tuple combinations of the -elements contained in *iterable*. :: - - itertools.combinations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 3), (2, 4), (2, 5), - (3, 4), (3, 5), - (4, 5) - - itertools.combinations([1, 2, 3, 4, 5], 3) => - (1, 2, 3), (1, 2, 4), (1, 2, 5), (1, 3, 4), (1, 3, 5), (1, 4, 5), - (2, 3, 4), (2, 3, 5), (2, 4, 5), - (3, 4, 5) - -The elements within each tuple remain in the same order as -*iterable* returned them. For example, the number 1 is always before -2, 3, 4, or 5 in the examples above. A similar function, -:func:`itertools.permutations(iterable, r=None) <itertools.permutations>`, -removes this constraint on the order, returning all possible -arrangements of length *r*:: - - itertools.permutations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 1), (2, 3), (2, 4), (2, 5), - (3, 1), (3, 2), (3, 4), (3, 5), - (4, 1), (4, 2), (4, 3), (4, 5), - (5, 1), (5, 2), (5, 3), (5, 4) - - itertools.permutations([1, 2, 3, 4, 5]) => - (1, 2, 3, 4, 5), (1, 2, 3, 5, 4), (1, 2, 4, 3, 5), - ... - (5, 4, 3, 2, 1) - -If you don't supply a value for *r* the length of the iterable is used, -meaning that all the elements are permuted. - -Note that these functions produce all of the possible combinations by -position and don't require that the contents of *iterable* are unique:: - - itertools.permutations('aba', 3) => - ('a', 'b', 'a'), ('a', 'a', 'b'), ('b', 'a', 'a'), - ('b', 'a', 'a'), ('a', 'a', 'b'), ('a', 'b', 'a') - -The identical tuple ``('a', 'a', 'b')`` occurs twice, but the two 'a' -strings came from different positions. - -The :func:`itertools.combinations_with_replacement(iterable, r) <itertools.combinations_with_replacement>` -function relaxes a different constraint: elements can be repeated -within a single tuple. Conceptually an element is selected for the -first position of each tuple and then is replaced before the second -element is selected. :: - - itertools.combinations_with_replacement([1, 2, 3, 4, 5], 2) => - (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), - (2, 2), (2, 3), (2, 4), (2, 5), - (3, 3), (3, 4), (3, 5), - (4, 4), (4, 5), - (5, 5) - - -Grouping elements ------------------ - -The last function I'll discuss, :func:`itertools.groupby(iter, key_func=None) -<itertools.groupby>`, is the most complicated. ``key_func(elem)`` is a function -that can compute a key value for each element returned by the iterable. If you -don't supply a key function, the key is simply each element itself. - -:func:`~itertools.groupby` collects all the consecutive elements from the -underlying iterable that have the same key value, and returns a stream of -2-tuples containing a key value and an iterator for the elements with that key. - -:: - - city_list = [('Decatur', 'AL'), ('Huntsville', 'AL'), ('Selma', 'AL'), - ('Anchorage', 'AK'), ('Nome', 'AK'), - ('Flagstaff', 'AZ'), ('Phoenix', 'AZ'), ('Tucson', 'AZ'), - ... - ] - - def get_state(city_state): - return city_state[1] - - itertools.groupby(city_list, get_state) => - ('AL', iterator-1), - ('AK', iterator-2), - ('AZ', iterator-3), ... - - where - iterator-1 => - ('Decatur', 'AL'), ('Huntsville', 'AL'), ('Selma', 'AL') - iterator-2 => - ('Anchorage', 'AK'), ('Nome', 'AK') - iterator-3 => - ('Flagstaff', 'AZ'), ('Phoenix', 'AZ'), ('Tucson', 'AZ') - -:func:`~itertools.groupby` assumes that the underlying iterable's contents will -already be sorted based on the key. Note that the returned iterators also use -the underlying iterable, so you have to consume the results of iterator-1 before -requesting iterator-2 and its corresponding key. - - -The functools module -==================== - -The :mod:`functools` module in Python 2.5 contains some higher-order functions. -A **higher-order function** takes one or more functions as input and returns a -new function. The most useful tool in this module is the -:func:`functools.partial` function. - -For programs written in a functional style, you'll sometimes want to construct -variants of existing functions that have some of the parameters filled in. -Consider a Python function ``f(a, b, c)``; you may wish to create a new function -``g(b, c)`` that's equivalent to ``f(1, b, c)``; you're filling in a value for -one of ``f()``'s parameters. This is called "partial function application". - -The constructor for :func:`~functools.partial` takes the arguments -``(function, arg1, arg2, ..., kwarg1=value1, kwarg2=value2)``. The resulting -object is callable, so you can just call it to invoke ``function`` with the -filled-in arguments. - -Here's a small but realistic example:: - - import functools - - def log(message, subsystem): - """Write the contents of 'message' to the specified subsystem.""" - print('%s: %s' % (subsystem, message)) - ... - - server_log = functools.partial(log, subsystem='server') - server_log('Unable to open socket') - -:func:`functools.reduce(func, iter, [initial_value]) <functools.reduce>` -cumulatively performs an operation on all the iterable's elements and, -therefore, can't be applied to infinite iterables. *func* must be a function -that takes two elements and returns a single value. :func:`functools.reduce` -takes the first two elements A and B returned by the iterator and calculates -``func(A, B)``. It then requests the third element, C, calculates -``func(func(A, B), C)``, combines this result with the fourth element returned, -and continues until the iterable is exhausted. If the iterable returns no -values at all, a :exc:`TypeError` exception is raised. If the initial value is -supplied, it's used as a starting point and ``func(initial_value, A)`` is the -first calculation. :: - - >>> import operator, functools - >>> functools.reduce(operator.concat, ['A', 'BB', 'C']) - 'ABBC' - >>> functools.reduce(operator.concat, []) - Traceback (most recent call last): - ... - TypeError: reduce() of empty sequence with no initial value - >>> functools.reduce(operator.mul, [1, 2, 3], 1) - 6 - >>> functools.reduce(operator.mul, [], 1) - 1 - -If you use :func:`operator.add` with :func:`functools.reduce`, you'll add up all the -elements of the iterable. This case is so common that there's a special -built-in called :func:`sum` to compute it: - - >>> import functools, operator - >>> functools.reduce(operator.add, [1, 2, 3, 4], 0) - 10 - >>> sum([1, 2, 3, 4]) - 10 - >>> sum([]) - 0 - -For many uses of :func:`functools.reduce`, though, it can be clearer to just -write the obvious :keyword:`for` loop:: - - import functools - # Instead of: - product = functools.reduce(operator.mul, [1, 2, 3], 1) - - # You can write: - product = 1 - for i in [1, 2, 3]: - product *= i - -A related function is :func:`itertools.accumulate(iterable, func=operator.add) -<itertools.accumulate>`. It performs the same calculation, but instead of -returning only the final result, :func:`accumulate` returns an iterator that -also yields each partial result:: - - itertools.accumulate([1, 2, 3, 4, 5]) => - 1, 3, 6, 10, 15 - - itertools.accumulate([1, 2, 3, 4, 5], operator.mul) => - 1, 2, 6, 24, 120 - - -The operator module -------------------- - -The :mod:`operator` module was mentioned earlier. It contains a set of -functions corresponding to Python's operators. These functions are often useful -in functional-style code because they save you from writing trivial functions -that perform a single operation. - -Some of the functions in this module are: - -* Math operations: ``add()``, ``sub()``, ``mul()``, ``floordiv()``, ``abs()``, ... -* Logical operations: ``not_()``, ``truth()``. -* Bitwise operations: ``and_()``, ``or_()``, ``invert()``. -* Comparisons: ``eq()``, ``ne()``, ``lt()``, ``le()``, ``gt()``, and ``ge()``. -* Object identity: ``is_()``, ``is_not()``. - -Consult the operator module's documentation for a complete list. - - -Small functions and the lambda expression -========================================= - -When writing functional-style programs, you'll often need little functions that -act as predicates or that combine elements in some way. - -If there's a Python built-in or a module function that's suitable, you don't -need to define a new function at all:: - - stripped_lines = [line.strip() for line in lines] - existing_files = filter(os.path.exists, file_list) - -If the function you need doesn't exist, you need to write it. One way to write -small functions is to use the :keyword:`lambda` statement. ``lambda`` takes a -number of parameters and an expression combining these parameters, and creates -an anonymous function that returns the value of the expression:: - - adder = lambda x, y: x+y - - print_assign = lambda name, value: name + '=' + str(value) - -An alternative is to just use the ``def`` statement and define a function in the -usual way:: - - def adder(x, y): - return x + y - - def print_assign(name, value): - return name + '=' + str(value) - -Which alternative is preferable? That's a style question; my usual course is to -avoid using ``lambda``. - -One reason for my preference is that ``lambda`` is quite limited in the -functions it can define. The result has to be computable as a single -expression, which means you can't have multiway ``if... elif... else`` -comparisons or ``try... except`` statements. If you try to do too much in a -``lambda`` statement, you'll end up with an overly complicated expression that's -hard to read. Quick, what's the following code doing? :: - - import functools - total = functools.reduce(lambda a, b: (0, a[1] + b[1]), items)[1] - -You can figure it out, but it takes time to disentangle the expression to figure -out what's going on. Using a short nested ``def`` statements makes things a -little bit better:: - - import functools - def combine(a, b): - return 0, a[1] + b[1] - - total = functools.reduce(combine, items)[1] - -But it would be best of all if I had simply used a ``for`` loop:: - - total = 0 - for a, b in items: - total += b - -Or the :func:`sum` built-in and a generator expression:: - - total = sum(b for a, b in items) - -Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops. - -Fredrik Lundh once suggested the following set of rules for refactoring uses of -``lambda``: - -1. Write a lambda function. -2. Write a comment explaining what the heck that lambda does. -3. Study the comment for a while, and think of a name that captures the essence - of the comment. -4. Convert the lambda to a def statement, using that name. -5. Remove the comment. - -I really like these rules, but you're free to disagree -about whether this lambda-free style is better. - - -Revision History and Acknowledgements -===================================== - -The author would like to thank the following people for offering suggestions, -corrections and assistance with various drafts of this article: Ian Bicking, -Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro -Lameiro, Jussi Salmela, Collin Winter, Blake Winton. - -Version 0.1: posted June 30 2006. - -Version 0.11: posted July 1 2006. Typo fixes. - -Version 0.2: posted July 10 2006. Merged genexp and listcomp sections into one. -Typo fixes. - -Version 0.21: Added more references suggested on the tutor mailing list. - -Version 0.30: Adds a section on the ``functional`` module written by Collin -Winter; adds short section on the operator module; a few other edits. - - -References -========== - -General -------- - -**Structure and Interpretation of Computer Programs**, by Harold Abelson and -Gerald Jay Sussman with Julie Sussman. Full text at -https://mitpress.mit.edu/sicp/. In this classic textbook of computer science, -chapters 2 and 3 discuss the use of sequences and streams to organize the data -flow inside a program. The book uses Scheme for its examples, but many of the -design approaches described in these chapters are applicable to functional-style -Python code. - -http://www.defmacro.org/ramblings/fp.html: A general introduction to functional -programming that uses Java examples and has a lengthy historical introduction. - -https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry -describing functional programming. - -https://en.wikipedia.org/wiki/Coroutine: Entry for coroutines. - -https://en.wikipedia.org/wiki/Currying: Entry for the concept of currying. - -Python-specific ---------------- - -http://gnosis.cx/TPiP/: The first chapter of David Mertz's book -:title-reference:`Text Processing in Python` discusses functional programming -for text processing, in the section titled "Utilizing Higher-Order Functions in -Text Processing". - -Mertz also wrote a 3-part series of articles on functional programming -for IBM's DeveloperWorks site; see -`part 1 <https://www.ibm.com/developerworks/linux/library/l-prog/index.html>`__, -`part 2 <https://www.ibm.com/developerworks/linux/library/l-prog2/index.html>`__, and -`part 3 <https://www.ibm.com/developerworks/linux/library/l-prog3/index.html>`__, - - -Python documentation --------------------- - -Documentation for the :mod:`itertools` module. - -Documentation for the :mod:`functools` module. - -Documentation for the :mod:`operator` module. - -:pep:`289`: "Generator Expressions" - -:pep:`342`: "Coroutines via Enhanced Generators" describes the new generator -features in Python 2.5. - -.. comment - - Handy little function for printing part of an iterator -- used - while writing this document. - - import itertools - def print_iter(it): - slice = itertools.islice(it, 10) - for elem in slice[:-1]: - sys.stdout.write(str(elem)) - sys.stdout.write(', ') - print(elem[-1]) diff --git a/third_party/python/Doc/howto/index.rst b/third_party/python/Doc/howto/index.rst deleted file mode 100644 index 593341cc2..000000000 --- a/third_party/python/Doc/howto/index.rst +++ /dev/null @@ -1,32 +0,0 @@ -*************** - Python HOWTOs -*************** - -Python HOWTOs are documents that cover a single, specific topic, -and attempt to cover it fairly completely. Modelled on the Linux -Documentation Project's HOWTO collection, this collection is an -effort to foster documentation that's more detailed than the -Python Library Reference. - -Currently, the HOWTOs are: - -.. toctree:: - :maxdepth: 1 - - pyporting.rst - cporting.rst - curses.rst - descriptor.rst - functional.rst - logging.rst - logging-cookbook.rst - regex.rst - sockets.rst - sorting.rst - unicode.rst - urllib2.rst - argparse.rst - ipaddress.rst - clinic.rst - instrumentation.rst - diff --git a/third_party/python/Doc/howto/instrumentation.rst b/third_party/python/Doc/howto/instrumentation.rst deleted file mode 100644 index 54c3fe379..000000000 --- a/third_party/python/Doc/howto/instrumentation.rst +++ /dev/null @@ -1,412 +0,0 @@ -.. highlight:: shell-session - -.. _instrumentation: - -=============================================== -Instrumenting CPython with DTrace and SystemTap -=============================================== - -:author: David Malcolm -:author: Łukasz Langa - -DTrace and SystemTap are monitoring tools, each providing a way to inspect -what the processes on a computer system are doing. They both use -domain-specific languages allowing a user to write scripts which: - - - filter which processes are to be observed - - gather data from the processes of interest - - generate reports on the data - -As of Python 3.6, CPython can be built with embedded "markers", also -known as "probes", that can be observed by a DTrace or SystemTap script, -making it easier to monitor what the CPython processes on a system are -doing. - -.. impl-detail:: - - DTrace markers are implementation details of the CPython interpreter. - No guarantees are made about probe compatibility between versions of - CPython. DTrace scripts can stop working or work incorrectly without - warning when changing CPython versions. - - -Enabling the static markers ---------------------------- - -macOS comes with built-in support for DTrace. On Linux, in order to -build CPython with the embedded markers for SystemTap, the SystemTap -development tools must be installed. - -On a Linux machine, this can be done via:: - - $ yum install systemtap-sdt-devel - -or:: - - $ sudo apt-get install systemtap-sdt-dev - - -CPython must then be configured ``--with-dtrace``: - -.. code-block:: none - - checking for --with-dtrace... yes - -On macOS, you can list available DTrace probes by running a Python -process in the background and listing all probes made available by the -Python provider:: - - $ python3.6 -q & - $ sudo dtrace -l -P python$! # or: dtrace -l -m python3.6 - - ID PROVIDER MODULE FUNCTION NAME - 29564 python18035 python3.6 _PyEval_EvalFrameDefault function-entry - 29565 python18035 python3.6 dtrace_function_entry function-entry - 29566 python18035 python3.6 _PyEval_EvalFrameDefault function-return - 29567 python18035 python3.6 dtrace_function_return function-return - 29568 python18035 python3.6 collect gc-done - 29569 python18035 python3.6 collect gc-start - 29570 python18035 python3.6 _PyEval_EvalFrameDefault line - 29571 python18035 python3.6 maybe_dtrace_line line - -On Linux, you can verify if the SystemTap static markers are present in -the built binary by seeing if it contains a ".note.stapsdt" section. - -:: - - $ readelf -S ./python | grep .note.stapsdt - [30] .note.stapsdt NOTE 0000000000000000 00308d78 - -If you've built Python as a shared library (with --enable-shared), you -need to look instead within the shared library. For example:: - - $ readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt - [29] .note.stapsdt NOTE 0000000000000000 00365b68 - -Sufficiently modern readelf can print the metadata:: - - $ readelf -n ./python - - Displaying notes found at file offset 0x00000254 with length 0x00000020: - Owner Data size Description - GNU 0x00000010 NT_GNU_ABI_TAG (ABI version tag) - OS: Linux, ABI: 2.6.32 - - Displaying notes found at file offset 0x00000274 with length 0x00000024: - Owner Data size Description - GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID bitstring) - Build ID: df924a2b08a7e89f6e11251d4602022977af2670 - - Displaying notes found at file offset 0x002d6c30 with length 0x00000144: - Owner Data size Description - stapsdt 0x00000031 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: gc__start - Location: 0x00000000004371c3, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf6 - Arguments: -4@%ebx - stapsdt 0x00000030 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: gc__done - Location: 0x00000000004374e1, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf8 - Arguments: -8@%rax - stapsdt 0x00000045 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: function__entry - Location: 0x000000000053db6c, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6be8 - Arguments: 8@%rbp 8@%r12 -4@%eax - stapsdt 0x00000046 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: function__return - Location: 0x000000000053dba8, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bea - Arguments: 8@%rbp 8@%r12 -4@%eax - -The above metadata contains information for SystemTap describing how it -can patch strategically-placed machine code instructions to enable the -tracing hooks used by a SystemTap script. - - -Static DTrace probes --------------------- - -The following example DTrace script can be used to show the call/return -hierarchy of a Python script, only tracing within the invocation of -a function called "start". In other words, import-time function -invocations are not going to be listed: - -.. code-block:: none - - self int indent; - - python$target:::function-entry - /copyinstr(arg1) == "start"/ - { - self->trace = 1; - } - - python$target:::function-entry - /self->trace/ - { - printf("%d\t%*s:", timestamp, 15, probename); - printf("%*s", self->indent, ""); - printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); - self->indent++; - } - - python$target:::function-return - /self->trace/ - { - self->indent--; - printf("%d\t%*s:", timestamp, 15, probename); - printf("%*s", self->indent, ""); - printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); - } - - python$target:::function-return - /copyinstr(arg1) == "start"/ - { - self->trace = 0; - } - -It can be invoked like this:: - - $ sudo dtrace -q -s call_stack.d -c "python3.6 script.py" - -The output looks like this: - -.. code-block:: none - - 156641360502280 function-entry:call_stack.py:start:23 - 156641360518804 function-entry: call_stack.py:function_1:1 - 156641360532797 function-entry: call_stack.py:function_3:9 - 156641360546807 function-return: call_stack.py:function_3:10 - 156641360563367 function-return: call_stack.py:function_1:2 - 156641360578365 function-entry: call_stack.py:function_2:5 - 156641360591757 function-entry: call_stack.py:function_1:1 - 156641360605556 function-entry: call_stack.py:function_3:9 - 156641360617482 function-return: call_stack.py:function_3:10 - 156641360629814 function-return: call_stack.py:function_1:2 - 156641360642285 function-return: call_stack.py:function_2:6 - 156641360656770 function-entry: call_stack.py:function_3:9 - 156641360669707 function-return: call_stack.py:function_3:10 - 156641360687853 function-entry: call_stack.py:function_4:13 - 156641360700719 function-return: call_stack.py:function_4:14 - 156641360719640 function-entry: call_stack.py:function_5:18 - 156641360732567 function-return: call_stack.py:function_5:21 - 156641360747370 function-return:call_stack.py:start:28 - - -Static SystemTap markers ------------------------- - -The low-level way to use the SystemTap integration is to use the static -markers directly. This requires you to explicitly state the binary file -containing them. - -For example, this SystemTap script can be used to show the call/return -hierarchy of a Python script: - -.. code-block:: none - - probe process("python").mark("function__entry") { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - - printf("%s => %s in %s:%d\\n", - thread_indent(1), funcname, filename, lineno); - } - - probe process("python").mark("function__return") { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - - printf("%s <= %s in %s:%d\\n", - thread_indent(-1), funcname, filename, lineno); - } - -It can be invoked like this:: - - $ stap \ - show-call-hierarchy.stp \ - -c "./python test.py" - -The output looks like this: - -.. code-block:: none - - 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 - 11414 python(8274): => __getitem__ in Lib/os.py:425 - 11418 python(8274): => encode in Lib/os.py:490 - 11424 python(8274): <= encode in Lib/os.py:493 - 11428 python(8274): <= __getitem__ in Lib/os.py:426 - 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 - -where the columns are: - - - time in microseconds since start of script - - - name of executable - - - PID of process - -and the remainder indicates the call/return hierarchy as the script executes. - -For a `--enable-shared` build of CPython, the markers are contained within the -libpython shared library, and the probe's dotted path needs to reflect this. For -example, this line from the above example: - -.. code-block:: none - - probe process("python").mark("function__entry") { - -should instead read: - -.. code-block:: none - - probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") { - -(assuming a debug build of CPython 3.6) - - -Available static markers ------------------------- - -.. I'm reusing the "c:function" type for markers - -.. c:function:: function__entry(str filename, str funcname, int lineno) - - This marker indicates that execution of a Python function has begun. - It is only triggered for pure-Python (bytecode) functions. - - The filename, function name, and line number are provided back to the - tracing script as positional arguments, which must be accessed using - ``$arg1``, ``$arg2``, ``$arg3``: - - * ``$arg1`` : ``(const char *)`` filename, accessible using ``user_string($arg1)`` - - * ``$arg2`` : ``(const char *)`` function name, accessible using - ``user_string($arg2)`` - - * ``$arg3`` : ``int`` line number - -.. c:function:: function__return(str filename, str funcname, int lineno) - - This marker is the converse of :c:func:`function__entry`, and indicates that - execution of a Python function has ended (either via ``return``, or via an - exception). It is only triggered for pure-Python (bytecode) functions. - - The arguments are the same as for :c:func:`function__entry` - -.. c:function:: line(str filename, str funcname, int lineno) - - This marker indicates a Python line is about to be executed. It is - the equivalent of line-by-line tracing with a Python profiler. It is - not triggered within C functions. - - The arguments are the same as for :c:func:`function__entry`. - -.. c:function:: gc__start(int generation) - - Fires when the Python interpreter starts a garbage collection cycle. - ``arg0`` is the generation to scan, like :func:`gc.collect()`. - -.. c:function:: gc__done(long collected) - - Fires when the Python interpreter finishes a garbage collection - cycle. ``arg0`` is the number of collected objects. - - -SystemTap Tapsets ------------------ - -The higher-level way to use the SystemTap integration is to use a "tapset": -SystemTap's equivalent of a library, which hides some of the lower-level -details of the static markers. - -Here is a tapset file, based on a non-shared build of CPython: - -.. code-block:: none - - /* - Provide a higher-level wrapping around the function__entry and - function__return markers: - \*/ - probe python.function.entry = process("python").mark("function__entry") - { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - frameptr = $arg4 - } - probe python.function.return = process("python").mark("function__return") - { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - frameptr = $arg4 - } - -If this file is installed in SystemTap's tapset directory (e.g. -``/usr/share/systemtap/tapset``), then these additional probepoints become -available: - -.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) - - This probe point indicates that execution of a Python function has begun. - It is only triggered for pure-python (bytecode) functions. - -.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) - - This probe point is the converse of :c:func:`python.function.return`, and - indicates that execution of a Python function has ended (either via - ``return``, or via an exception). It is only triggered for pure-python - (bytecode) functions. - - -Examples --------- -This SystemTap script uses the tapset above to more cleanly implement the -example given above of tracing the Python function-call hierarchy, without -needing to directly name the static markers: - -.. code-block:: none - - probe python.function.entry - { - printf("%s => %s in %s:%d\n", - thread_indent(1), funcname, filename, lineno); - } - - probe python.function.return - { - printf("%s <= %s in %s:%d\n", - thread_indent(-1), funcname, filename, lineno); - } - - -The following script uses the tapset above to provide a top-like view of all -running CPython code, showing the top 20 most frequently-entered bytecode -frames, each second, across the whole system: - -.. code-block:: none - - global fn_calls; - - probe python.function.entry - { - fn_calls[pid(), filename, funcname, lineno] += 1; - } - - probe timer.ms(1000) { - printf("\033[2J\033[1;1H") /* clear screen \*/ - printf("%6s %80s %6s %30s %6s\n", - "PID", "FILENAME", "LINE", "FUNCTION", "CALLS") - foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) { - printf("%6d %80s %6d %30s %6d\n", - pid, filename, lineno, funcname, - fn_calls[pid, filename, funcname, lineno]); - } - delete fn_calls; - } - diff --git a/third_party/python/Doc/howto/ipaddress.rst b/third_party/python/Doc/howto/ipaddress.rst deleted file mode 100644 index 452e36710..000000000 --- a/third_party/python/Doc/howto/ipaddress.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. testsetup:: - - import ipaddress - -.. _ipaddress-howto: - -*************************************** -An introduction to the ipaddress module -*************************************** - -:author: Peter Moody -:author: Nick Coghlan - -.. topic:: Overview - - This document aims to provide a gentle introduction to the - :mod:`ipaddress` module. It is aimed primarily at users that aren't - already familiar with IP networking terminology, but may also be useful - to network engineers wanting an overview of how :mod:`ipaddress` - represents IP network addressing concepts. - - -Creating Address/Network/Interface objects -========================================== - -Since :mod:`ipaddress` is a module for inspecting and manipulating IP addresses, -the first thing you'll want to do is create some objects. You can use -:mod:`ipaddress` to create objects from strings and integers. - - -A Note on IP Versions ---------------------- - -For readers that aren't particularly familiar with IP addressing, it's -important to know that the Internet Protocol is currently in the process -of moving from version 4 of the protocol to version 6. This transition is -occurring largely because version 4 of the protocol doesn't provide enough -addresses to handle the needs of the whole world, especially given the -increasing number of devices with direct connections to the internet. - -Explaining the details of the differences between the two versions of the -protocol is beyond the scope of this introduction, but readers need to at -least be aware that these two versions exist, and it will sometimes be -necessary to force the use of one version or the other. - - -IP Host Addresses ------------------ - -Addresses, often referred to as "host addresses" are the most basic unit -when working with IP addressing. The simplest way to create addresses is -to use the :func:`ipaddress.ip_address` factory function, which automatically -determines whether to create an IPv4 or IPv6 address based on the passed in -value: - - >>> ipaddress.ip_address('192.0.2.1') - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address('2001:DB8::1') - IPv6Address('2001:db8::1') - -Addresses can also be created directly from integers. Values that will -fit within 32 bits are assumed to be IPv4 addresses:: - - >>> ipaddress.ip_address(3221225985) - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address(42540766411282592856903984951653826561) - IPv6Address('2001:db8::1') - -To force the use of IPv4 or IPv6 addresses, the relevant classes can be -invoked directly. This is particularly useful to force creation of IPv6 -addresses for small integers:: - - >>> ipaddress.ip_address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv4Address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv6Address(1) - IPv6Address('::1') - - -Defining Networks ------------------ - -Host addresses are usually grouped together into IP networks, so -:mod:`ipaddress` provides a way to create, inspect and manipulate network -definitions. IP network objects are constructed from strings that define the -range of host addresses that are part of that network. The simplest form -for that information is a "network address/network prefix" pair, where the -prefix defines the number of leading bits that are compared to determine -whether or not an address is part of the network and the network address -defines the expected value of those bits. - -As for addresses, a factory function is provided that determines the correct -IP version automatically:: - - >>> ipaddress.ip_network('192.0.2.0/24') - IPv4Network('192.0.2.0/24') - >>> ipaddress.ip_network('2001:db8::0/96') - IPv6Network('2001:db8::/96') - -Network objects cannot have any host bits set. The practical effect of this -is that ``192.0.2.1/24`` does not describe a network. Such definitions are -referred to as interface objects since the ip-on-a-network notation is -commonly used to describe network interfaces of a computer on a given network -and are described further in the next section. - -By default, attempting to create a network object with host bits set will -result in :exc:`ValueError` being raised. To request that the -additional bits instead be coerced to zero, the flag ``strict=False`` can -be passed to the constructor:: - - >>> ipaddress.ip_network('192.0.2.1/24') - Traceback (most recent call last): - ... - ValueError: 192.0.2.1/24 has host bits set - >>> ipaddress.ip_network('192.0.2.1/24', strict=False) - IPv4Network('192.0.2.0/24') - -While the string form offers significantly more flexibility, networks can -also be defined with integers, just like host addresses. In this case, the -network is considered to contain only the single address identified by the -integer, so the network prefix includes the entire network address:: - - >>> ipaddress.ip_network(3221225984) - IPv4Network('192.0.2.0/32') - >>> ipaddress.ip_network(42540766411282592856903984951653826560) - IPv6Network('2001:db8::/128') - -As with addresses, creation of a particular kind of network can be forced -by calling the class constructor directly instead of using the factory -function. - - -Host Interfaces ---------------- - -As mentioned just above, if you need to describe an address on a particular -network, neither the address nor the network classes are sufficient. -Notation like ``192.0.2.1/24`` is commonly used by network engineers and the -people who write tools for firewalls and routers as shorthand for "the host -``192.0.2.1`` on the network ``192.0.2.0/24``", Accordingly, :mod:`ipaddress` -provides a set of hybrid classes that associate an address with a particular -network. The interface for creation is identical to that for defining network -objects, except that the address portion isn't constrained to being a network -address. - - >>> ipaddress.ip_interface('192.0.2.1/24') - IPv4Interface('192.0.2.1/24') - >>> ipaddress.ip_interface('2001:db8::1/96') - IPv6Interface('2001:db8::1/96') - -Integer inputs are accepted (as with networks), and use of a particular IP -version can be forced by calling the relevant constructor directly. - - -Inspecting Address/Network/Interface Objects -============================================ - -You've gone to the trouble of creating an IPv(4|6)(Address|Network|Interface) -object, so you probably want to get information about it. :mod:`ipaddress` -tries to make doing this easy and intuitive. - -Extracting the IP version:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr6 = ipaddress.ip_address('2001:db8::1') - >>> addr6.version - 6 - >>> addr4.version - 4 - -Obtaining the network from an interface:: - - >>> host4 = ipaddress.ip_interface('192.0.2.1/24') - >>> host4.network - IPv4Network('192.0.2.0/24') - >>> host6 = ipaddress.ip_interface('2001:db8::1/96') - >>> host6.network - IPv6Network('2001:db8::/96') - -Finding out how many individual addresses are in a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.num_addresses - 256 - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.num_addresses - 4294967296 - -Iterating through the "usable" addresses on a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> for x in net4.hosts(): - ... print(x) # doctest: +ELLIPSIS - 192.0.2.1 - 192.0.2.2 - 192.0.2.3 - 192.0.2.4 - ... - 192.0.2.252 - 192.0.2.253 - 192.0.2.254 - - -Obtaining the netmask (i.e. set bits corresponding to the network prefix) or -the hostmask (any bits that are not part of the netmask): - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.netmask - IPv4Address('255.255.255.0') - >>> net4.hostmask - IPv4Address('0.0.0.255') - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.netmask - IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') - >>> net6.hostmask - IPv6Address('::ffff:ffff') - - -Exploding or compressing the address:: - - >>> addr6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0001' - >>> addr6.compressed - '2001:db8::1' - >>> net6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0000/96' - >>> net6.compressed - '2001:db8::/96' - -While IPv4 doesn't support explosion or compression, the associated objects -still provide the relevant properties so that version neutral code can -easily ensure the most concise or most verbose form is used for IPv6 -addresses while still correctly handling IPv4 addresses. - - -Networks as lists of Addresses -============================== - -It's sometimes useful to treat networks as lists. This means it is possible -to index them like this:: - - >>> net4[1] - IPv4Address('192.0.2.1') - >>> net4[-1] - IPv4Address('192.0.2.255') - >>> net6[1] - IPv6Address('2001:db8::1') - >>> net6[-1] - IPv6Address('2001:db8::ffff:ffff') - - -It also means that network objects lend themselves to using the list -membership test syntax like this:: - - if address in network: - # do something - -Containment testing is done efficiently based on the network prefix:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr4 in ipaddress.ip_network('192.0.2.0/24') - True - >>> addr4 in ipaddress.ip_network('192.0.3.0/24') - False - - -Comparisons -=========== - -:mod:`ipaddress` provides some simple, hopefully intuitive ways to compare -objects, where it makes sense:: - - >>> ipaddress.ip_address('192.0.2.1') < ipaddress.ip_address('192.0.2.2') - True - -A :exc:`TypeError` exception is raised if you try to compare objects of -different versions or different types. - - -Using IP Addresses with other modules -===================================== - -Other modules that use IP addresses (such as :mod:`socket`) usually won't -accept objects from this module directly. Instead, they must be coerced to -an integer or string that the other module will accept:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> str(addr4) - '192.0.2.1' - >>> int(addr4) - 3221225985 - - -Getting more detail when instance creation fails -================================================ - -When creating address/network/interface objects using the version-agnostic -factory functions, any errors will be reported as :exc:`ValueError` with -a generic error message that simply says the passed in value was not -recognized as an object of that type. The lack of a specific error is -because it's necessary to know whether the value is *supposed* to be IPv4 -or IPv6 in order to provide more detail on why it has been rejected. - -To support use cases where it is useful to have access to this additional -detail, the individual class constructors actually raise the -:exc:`ValueError` subclasses :exc:`ipaddress.AddressValueError` and -:exc:`ipaddress.NetmaskValueError` to indicate exactly which part of -the definition failed to parse correctly. - -The error messages are significantly more detailed when using the -class constructors directly. For example:: - - >>> ipaddress.ip_address("192.168.0.256") - Traceback (most recent call last): - ... - ValueError: '192.168.0.256' does not appear to be an IPv4 or IPv6 address - >>> ipaddress.IPv4Address("192.168.0.256") - Traceback (most recent call last): - ... - ipaddress.AddressValueError: Octet 256 (> 255) not permitted in '192.168.0.256' - - >>> ipaddress.ip_network("192.168.0.1/64") - Traceback (most recent call last): - ... - ValueError: '192.168.0.1/64' does not appear to be an IPv4 or IPv6 network - >>> ipaddress.IPv4Network("192.168.0.1/64") - Traceback (most recent call last): - ... - ipaddress.NetmaskValueError: '64' is not a valid netmask - -However, both of the module specific exceptions have :exc:`ValueError` as their -parent class, so if you're not concerned with the particular type of error, -you can still write code like the following:: - - try: - network = ipaddress.IPv4Network(address) - except ValueError: - print('address/netmask is invalid for IPv4:', address) - diff --git a/third_party/python/Doc/howto/logging-cookbook.rst b/third_party/python/Doc/howto/logging-cookbook.rst deleted file mode 100644 index 96d550c45..000000000 --- a/third_party/python/Doc/howto/logging-cookbook.rst +++ /dev/null @@ -1,2551 +0,0 @@ -.. _logging-cookbook: - -================ -Logging Cookbook -================ - -:Author: Vinay Sajip <vinay_sajip at red-dove dot com> - -This page contains a number of recipes related to logging, which have been found -useful in the past. - -.. currentmodule:: logging - -Using logging in multiple modules ---------------------------------- - -Multiple calls to ``logging.getLogger('someLogger')`` return a reference to the -same logger object. This is true not only within the same module, but also -across modules as long as it is in the same Python interpreter process. It is -true for references to the same object; additionally, application code can -define and configure a parent logger in one module and create (but not -configure) a child logger in a separate module, and all logger calls to the -child will pass up to the parent. Here is a main module:: - - import logging - import auxiliary_module - - # create logger with 'spam_application' - logger = logging.getLogger('spam_application') - logger.setLevel(logging.DEBUG) - # create file handler which logs even debug messages - fh = logging.FileHandler('spam.log') - fh.setLevel(logging.DEBUG) - # create console handler with a higher log level - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - ch.setFormatter(formatter) - # add the handlers to the logger - logger.addHandler(fh) - logger.addHandler(ch) - - logger.info('creating an instance of auxiliary_module.Auxiliary') - a = auxiliary_module.Auxiliary() - logger.info('created an instance of auxiliary_module.Auxiliary') - logger.info('calling auxiliary_module.Auxiliary.do_something') - a.do_something() - logger.info('finished auxiliary_module.Auxiliary.do_something') - logger.info('calling auxiliary_module.some_function()') - auxiliary_module.some_function() - logger.info('done with auxiliary_module.some_function()') - -Here is the auxiliary module:: - - import logging - - # create logger - module_logger = logging.getLogger('spam_application.auxiliary') - - class Auxiliary: - def __init__(self): - self.logger = logging.getLogger('spam_application.auxiliary.Auxiliary') - self.logger.info('creating an instance of Auxiliary') - - def do_something(self): - self.logger.info('doing something') - a = 1 + 1 - self.logger.info('done doing something') - - def some_function(): - module_logger.info('received a call to "some_function"') - -The output looks like this: - -.. code-block:: none - - 2005-03-23 23:47:11,663 - spam_application - INFO - - creating an instance of auxiliary_module.Auxiliary - 2005-03-23 23:47:11,665 - spam_application.auxiliary.Auxiliary - INFO - - creating an instance of Auxiliary - 2005-03-23 23:47:11,665 - spam_application - INFO - - created an instance of auxiliary_module.Auxiliary - 2005-03-23 23:47:11,668 - spam_application - INFO - - calling auxiliary_module.Auxiliary.do_something - 2005-03-23 23:47:11,668 - spam_application.auxiliary.Auxiliary - INFO - - doing something - 2005-03-23 23:47:11,669 - spam_application.auxiliary.Auxiliary - INFO - - done doing something - 2005-03-23 23:47:11,670 - spam_application - INFO - - finished auxiliary_module.Auxiliary.do_something - 2005-03-23 23:47:11,671 - spam_application - INFO - - calling auxiliary_module.some_function() - 2005-03-23 23:47:11,672 - spam_application.auxiliary - INFO - - received a call to 'some_function' - 2005-03-23 23:47:11,673 - spam_application - INFO - - done with auxiliary_module.some_function() - -Logging from multiple threads ------------------------------ - -Logging from multiple threads requires no special effort. The following example -shows logging from the main (initial) thread and another thread:: - - import logging - import threading - import time - - def worker(arg): - while not arg['stop']: - logging.debug('Hi from myfunc') - time.sleep(0.5) - - def main(): - logging.basicConfig(level=logging.DEBUG, format='%(relativeCreated)6d %(threadName)s %(message)s') - info = {'stop': False} - thread = threading.Thread(target=worker, args=(info,)) - thread.start() - while True: - try: - logging.debug('Hello from main') - time.sleep(0.75) - except KeyboardInterrupt: - info['stop'] = True - break - thread.join() - - if __name__ == '__main__': - main() - -When run, the script should print something like the following: - -.. code-block:: none - - 0 Thread-1 Hi from myfunc - 3 MainThread Hello from main - 505 Thread-1 Hi from myfunc - 755 MainThread Hello from main - 1007 Thread-1 Hi from myfunc - 1507 MainThread Hello from main - 1508 Thread-1 Hi from myfunc - 2010 Thread-1 Hi from myfunc - 2258 MainThread Hello from main - 2512 Thread-1 Hi from myfunc - 3009 MainThread Hello from main - 3013 Thread-1 Hi from myfunc - 3515 Thread-1 Hi from myfunc - 3761 MainThread Hello from main - 4017 Thread-1 Hi from myfunc - 4513 MainThread Hello from main - 4518 Thread-1 Hi from myfunc - -This shows the logging output interspersed as one might expect. This approach -works for more threads than shown here, of course. - -Multiple handlers and formatters --------------------------------- - -Loggers are plain Python objects. The :meth:`~Logger.addHandler` method has no -minimum or maximum quota for the number of handlers you may add. Sometimes it -will be beneficial for an application to log all messages of all severities to a -text file while simultaneously logging errors or above to the console. To set -this up, simply configure the appropriate handlers. The logging calls in the -application code will remain unchanged. Here is a slight modification to the -previous simple module-based configuration example:: - - import logging - - logger = logging.getLogger('simple_example') - logger.setLevel(logging.DEBUG) - # create file handler which logs even debug messages - fh = logging.FileHandler('spam.log') - fh.setLevel(logging.DEBUG) - # create console handler with a higher log level - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - fh.setFormatter(formatter) - # add the handlers to logger - logger.addHandler(ch) - logger.addHandler(fh) - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warn('warn message') - logger.error('error message') - logger.critical('critical message') - -Notice that the 'application' code does not care about multiple handlers. All -that changed was the addition and configuration of a new handler named *fh*. - -The ability to create new handlers with higher- or lower-severity filters can be -very helpful when writing and testing an application. Instead of using many -``print`` statements for debugging, use ``logger.debug``: Unlike the print -statements, which you will have to delete or comment out later, the logger.debug -statements can remain intact in the source code and remain dormant until you -need them again. At that time, the only change that needs to happen is to -modify the severity level of the logger and/or handler to debug. - -.. _multiple-destinations: - -Logging to multiple destinations --------------------------------- - -Let's say you want to log to console and file with different message formats and -in differing circumstances. Say you want to log messages with levels of DEBUG -and higher to file, and those messages at level INFO and higher to the console. -Let's also assume that the file should contain timestamps, but the console -messages should not. Here's how you can achieve this:: - - import logging - - # set up logging to file - see previous section for more details - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M', - filename='/temp/myapp.log', - filemode='w') - # define a Handler which writes INFO messages or higher to the sys.stderr - console = logging.StreamHandler() - console.setLevel(logging.INFO) - # set a format which is simpler for console use - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - # tell the handler to use this format - console.setFormatter(formatter) - # add the handler to the root logger - logging.getLogger('').addHandler(console) - - # Now, we can log to the root logger, or any other logger. First the root... - logging.info('Jackdaws love my big sphinx of quartz.') - - # Now, define a couple of other loggers which might represent areas in your - # application: - - logger1 = logging.getLogger('myapp.area1') - logger2 = logging.getLogger('myapp.area2') - - logger1.debug('Quick zephyrs blow, vexing daft Jim.') - logger1.info('How quickly daft jumping zebras vex.') - logger2.warning('Jail zesty vixen who grabbed pay from quack.') - logger2.error('The five boxing wizards jump quickly.') - -When you run this, on the console you will see - -.. code-block:: none - - root : INFO Jackdaws love my big sphinx of quartz. - myapp.area1 : INFO How quickly daft jumping zebras vex. - myapp.area2 : WARNING Jail zesty vixen who grabbed pay from quack. - myapp.area2 : ERROR The five boxing wizards jump quickly. - -and in the file you will see something like - -.. code-block:: none - - 10-22 22:19 root INFO Jackdaws love my big sphinx of quartz. - 10-22 22:19 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim. - 10-22 22:19 myapp.area1 INFO How quickly daft jumping zebras vex. - 10-22 22:19 myapp.area2 WARNING Jail zesty vixen who grabbed pay from quack. - 10-22 22:19 myapp.area2 ERROR The five boxing wizards jump quickly. - -As you can see, the DEBUG message only shows up in the file. The other messages -are sent to both destinations. - -This example uses console and file handlers, but you can use any number and -combination of handlers you choose. - - -Configuration server example ----------------------------- - -Here is an example of a module using the logging configuration server:: - - import logging - import logging.config - import time - import os - - # read initial config file - logging.config.fileConfig('logging.conf') - - # create and start listener on port 9999 - t = logging.config.listen(9999) - t.start() - - logger = logging.getLogger('simpleExample') - - try: - # loop through logging calls to see the difference - # new configurations make, until Ctrl+C is pressed - while True: - logger.debug('debug message') - logger.info('info message') - logger.warn('warn message') - logger.error('error message') - logger.critical('critical message') - time.sleep(5) - except KeyboardInterrupt: - # cleanup - logging.config.stopListening() - t.join() - -And here is a script that takes a filename and sends that file to the server, -properly preceded with the binary-encoded length, as the new logging -configuration:: - - #!/usr/bin/env python - import socket, sys, struct - - with open(sys.argv[1], 'rb') as f: - data_to_send = f.read() - - HOST = 'localhost' - PORT = 9999 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - print('connecting...') - s.connect((HOST, PORT)) - print('sending config...') - s.send(struct.pack('>L', len(data_to_send))) - s.send(data_to_send) - s.close() - print('complete') - - -Dealing with handlers that block --------------------------------- - -.. currentmodule:: logging.handlers - -Sometimes you have to get your logging handlers to do their work without -blocking the thread you're logging from. This is common in Web applications, -though of course it also occurs in other scenarios. - -A common culprit which demonstrates sluggish behaviour is the -:class:`SMTPHandler`: sending emails can take a long time, for a -number of reasons outside the developer's control (for example, a poorly -performing mail or network infrastructure). But almost any network-based -handler can block: Even a :class:`SocketHandler` operation may do a -DNS query under the hood which is too slow (and this query can be deep in the -socket library code, below the Python layer, and outside your control). - -One solution is to use a two-part approach. For the first part, attach only a -:class:`QueueHandler` to those loggers which are accessed from -performance-critical threads. They simply write to their queue, which can be -sized to a large enough capacity or initialized with no upper bound to their -size. The write to the queue will typically be accepted quickly, though you -will probably need to catch the :exc:`queue.Full` exception as a precaution -in your code. If you are a library developer who has performance-critical -threads in their code, be sure to document this (together with a suggestion to -attach only ``QueueHandlers`` to your loggers) for the benefit of other -developers who will use your code. - -The second part of the solution is :class:`QueueListener`, which has been -designed as the counterpart to :class:`QueueHandler`. A -:class:`QueueListener` is very simple: it's passed a queue and some handlers, -and it fires up an internal thread which listens to its queue for LogRecords -sent from ``QueueHandlers`` (or any other source of ``LogRecords``, for that -matter). The ``LogRecords`` are removed from the queue and passed to the -handlers for processing. - -The advantage of having a separate :class:`QueueListener` class is that you -can use the same instance to service multiple ``QueueHandlers``. This is more -resource-friendly than, say, having threaded versions of the existing handler -classes, which would eat up one thread per handler for no particular benefit. - -An example of using these two classes follows (imports omitted):: - - que = queue.Queue(-1) # no limit on size - queue_handler = QueueHandler(que) - handler = logging.StreamHandler() - listener = QueueListener(que, handler) - root = logging.getLogger() - root.addHandler(queue_handler) - formatter = logging.Formatter('%(threadName)s: %(message)s') - handler.setFormatter(formatter) - listener.start() - # The log output will display the thread which generated - # the event (the main thread) rather than the internal - # thread which monitors the internal queue. This is what - # you want to happen. - root.warning('Look out!') - listener.stop() - -which, when run, will produce: - -.. code-block:: none - - MainThread: Look out! - -.. versionchanged:: 3.5 - Prior to Python 3.5, the :class:`QueueListener` always passed every message - received from the queue to every handler it was initialized with. (This was - because it was assumed that level filtering was all done on the other side, - where the queue is filled.) From 3.5 onwards, this behaviour can be changed - by passing a keyword argument ``respect_handler_level=True`` to the - listener's constructor. When this is done, the listener compares the level - of each message with the handler's level, and only passes a message to a - handler if it's appropriate to do so. - -.. _network-logging: - -Sending and receiving logging events across a network ------------------------------------------------------ - -Let's say you want to send logging events across a network, and handle them at -the receiving end. A simple way of doing this is attaching a -:class:`SocketHandler` instance to the root logger at the sending end:: - - import logging, logging.handlers - - rootLogger = logging.getLogger('') - rootLogger.setLevel(logging.DEBUG) - socketHandler = logging.handlers.SocketHandler('localhost', - logging.handlers.DEFAULT_TCP_LOGGING_PORT) - # don't bother with a formatter, since a socket handler sends the event as - # an unformatted pickle - rootLogger.addHandler(socketHandler) - - # Now, we can log to the root logger, or any other logger. First the root... - logging.info('Jackdaws love my big sphinx of quartz.') - - # Now, define a couple of other loggers which might represent areas in your - # application: - - logger1 = logging.getLogger('myapp.area1') - logger2 = logging.getLogger('myapp.area2') - - logger1.debug('Quick zephyrs blow, vexing daft Jim.') - logger1.info('How quickly daft jumping zebras vex.') - logger2.warning('Jail zesty vixen who grabbed pay from quack.') - logger2.error('The five boxing wizards jump quickly.') - -At the receiving end, you can set up a receiver using the :mod:`socketserver` -module. Here is a basic working example:: - - import pickle - import logging - import logging.handlers - import socketserver - import struct - - - class LogRecordStreamHandler(socketserver.StreamRequestHandler): - """Handler for a streaming logging request. - - This basically logs the record using whatever logging policy is - configured locally. - """ - - def handle(self): - """ - Handle multiple requests - each expected to be a 4-byte length, - followed by the LogRecord in pickle format. Logs the record - according to whatever policy is configured locally. - """ - while True: - chunk = self.connection.recv(4) - if len(chunk) < 4: - break - slen = struct.unpack('>L', chunk)[0] - chunk = self.connection.recv(slen) - while len(chunk) < slen: - chunk = chunk + self.connection.recv(slen - len(chunk)) - obj = self.unPickle(chunk) - record = logging.makeLogRecord(obj) - self.handleLogRecord(record) - - def unPickle(self, data): - return pickle.loads(data) - - def handleLogRecord(self, record): - # if a name is specified, we use the named logger rather than the one - # implied by the record. - if self.server.logname is not None: - name = self.server.logname - else: - name = record.name - logger = logging.getLogger(name) - # N.B. EVERY record gets logged. This is because Logger.handle - # is normally called AFTER logger-level filtering. If you want - # to do filtering, do it at the client end to save wasting - # cycles and network bandwidth! - logger.handle(record) - - class LogRecordSocketReceiver(socketserver.ThreadingTCPServer): - """ - Simple TCP socket-based logging receiver suitable for testing. - """ - - allow_reuse_address = True - - def __init__(self, host='localhost', - port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, - handler=LogRecordStreamHandler): - socketserver.ThreadingTCPServer.__init__(self, (host, port), handler) - self.abort = 0 - self.timeout = 1 - self.logname = None - - def serve_until_stopped(self): - import select - abort = 0 - while not abort: - rd, wr, ex = select.select([self.socket.fileno()], - [], [], - self.timeout) - if rd: - self.handle_request() - abort = self.abort - - def main(): - logging.basicConfig( - format='%(relativeCreated)5d %(name)-15s %(levelname)-8s %(message)s') - tcpserver = LogRecordSocketReceiver() - print('About to start TCP server...') - tcpserver.serve_until_stopped() - - if __name__ == '__main__': - main() - -First run the server, and then the client. On the client side, nothing is -printed on the console; on the server side, you should see something like: - -.. code-block:: none - - About to start TCP server... - 59 root INFO Jackdaws love my big sphinx of quartz. - 59 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim. - 69 myapp.area1 INFO How quickly daft jumping zebras vex. - 69 myapp.area2 WARNING Jail zesty vixen who grabbed pay from quack. - 69 myapp.area2 ERROR The five boxing wizards jump quickly. - -Note that there are some security issues with pickle in some scenarios. If -these affect you, you can use an alternative serialization scheme by overriding -the :meth:`~handlers.SocketHandler.makePickle` method and implementing your -alternative there, as well as adapting the above script to use your alternative -serialization. - - -.. _context-info: - -Adding contextual information to your logging output ----------------------------------------------------- - -Sometimes you want logging output to contain contextual information in -addition to the parameters passed to the logging call. For example, in a -networked application, it may be desirable to log client-specific information -in the log (e.g. remote client's username, or IP address). Although you could -use the *extra* parameter to achieve this, it's not always convenient to pass -the information in this way. While it might be tempting to create -:class:`Logger` instances on a per-connection basis, this is not a good idea -because these instances are not garbage collected. While this is not a problem -in practice, when the number of :class:`Logger` instances is dependent on the -level of granularity you want to use in logging an application, it could -be hard to manage if the number of :class:`Logger` instances becomes -effectively unbounded. - - -Using LoggerAdapters to impart contextual information -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An easy way in which you can pass contextual information to be output along -with logging event information is to use the :class:`LoggerAdapter` class. -This class is designed to look like a :class:`Logger`, so that you can call -:meth:`debug`, :meth:`info`, :meth:`warning`, :meth:`error`, -:meth:`exception`, :meth:`critical` and :meth:`log`. These methods have the -same signatures as their counterparts in :class:`Logger`, so you can use the -two types of instances interchangeably. - -When you create an instance of :class:`LoggerAdapter`, you pass it a -:class:`Logger` instance and a dict-like object which contains your contextual -information. When you call one of the logging methods on an instance of -:class:`LoggerAdapter`, it delegates the call to the underlying instance of -:class:`Logger` passed to its constructor, and arranges to pass the contextual -information in the delegated call. Here's a snippet from the code of -:class:`LoggerAdapter`:: - - def debug(self, msg, *args, **kwargs): - """ - Delegate a debug call to the underlying logger, after adding - contextual information from this adapter instance. - """ - msg, kwargs = self.process(msg, kwargs) - self.logger.debug(msg, *args, **kwargs) - -The :meth:`~LoggerAdapter.process` method of :class:`LoggerAdapter` is where the -contextual information is added to the logging output. It's passed the message -and keyword arguments of the logging call, and it passes back (potentially) -modified versions of these to use in the call to the underlying logger. The -default implementation of this method leaves the message alone, but inserts -an 'extra' key in the keyword argument whose value is the dict-like object -passed to the constructor. Of course, if you had passed an 'extra' keyword -argument in the call to the adapter, it will be silently overwritten. - -The advantage of using 'extra' is that the values in the dict-like object are -merged into the :class:`LogRecord` instance's __dict__, allowing you to use -customized strings with your :class:`Formatter` instances which know about -the keys of the dict-like object. If you need a different method, e.g. if you -want to prepend or append the contextual information to the message string, -you just need to subclass :class:`LoggerAdapter` and override -:meth:`~LoggerAdapter.process` to do what you need. Here is a simple example:: - - class CustomAdapter(logging.LoggerAdapter): - """ - This example adapter expects the passed in dict-like object to have a - 'connid' key, whose value in brackets is prepended to the log message. - """ - def process(self, msg, kwargs): - return '[%s] %s' % (self.extra['connid'], msg), kwargs - -which you can use like this:: - - logger = logging.getLogger(__name__) - adapter = CustomAdapter(logger, {'connid': some_conn_id}) - -Then any events that you log to the adapter will have the value of -``some_conn_id`` prepended to the log messages. - -Using objects other than dicts to pass contextual information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You don't need to pass an actual dict to a :class:`LoggerAdapter` - you could -pass an instance of a class which implements ``__getitem__`` and ``__iter__`` so -that it looks like a dict to logging. This would be useful if you want to -generate values dynamically (whereas the values in a dict would be constant). - - -.. _filters-contextual: - -Using Filters to impart contextual information -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can also add contextual information to log output using a user-defined -:class:`Filter`. ``Filter`` instances are allowed to modify the ``LogRecords`` -passed to them, including adding additional attributes which can then be output -using a suitable format string, or if needed a custom :class:`Formatter`. - -For example in a web application, the request being processed (or at least, -the interesting parts of it) can be stored in a threadlocal -(:class:`threading.local`) variable, and then accessed from a ``Filter`` to -add, say, information from the request - say, the remote IP address and remote -user's username - to the ``LogRecord``, using the attribute names 'ip' and -'user' as in the ``LoggerAdapter`` example above. In that case, the same format -string can be used to get similar output to that shown above. Here's an example -script:: - - import logging - from random import choice - - class ContextFilter(logging.Filter): - """ - This is a filter which injects contextual information into the log. - - Rather than use actual contextual information, we just use random - data in this demo. - """ - - USERS = ['jim', 'fred', 'sheila'] - IPS = ['123.231.231.123', '127.0.0.1', '192.168.0.1'] - - def filter(self, record): - - record.ip = choice(ContextFilter.IPS) - record.user = choice(ContextFilter.USERS) - return True - - if __name__ == '__main__': - levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)-15s %(name)-5s %(levelname)-8s IP: %(ip)-15s User: %(user)-8s %(message)s') - a1 = logging.getLogger('a.b.c') - a2 = logging.getLogger('d.e.f') - - f = ContextFilter() - a1.addFilter(f) - a2.addFilter(f) - a1.debug('A debug message') - a1.info('An info message with %s', 'some parameters') - for x in range(10): - lvl = choice(levels) - lvlname = logging.getLevelName(lvl) - a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') - -which, when run, produces something like: - -.. code-block:: none - - 2010-09-06 22:38:15,292 a.b.c DEBUG IP: 123.231.231.123 User: fred A debug message - 2010-09-06 22:38:15,300 a.b.c INFO IP: 192.168.0.1 User: sheila An info message with some parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 127.0.0.1 User: sheila A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f ERROR IP: 127.0.0.1 User: jim A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f DEBUG IP: 127.0.0.1 User: sheila A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f ERROR IP: 123.231.231.123 User: fred A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 192.168.0.1 User: jim A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 127.0.0.1 User: sheila A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f DEBUG IP: 192.168.0.1 User: jim A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f ERROR IP: 127.0.0.1 User: sheila A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f DEBUG IP: 123.231.231.123 User: fred A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f INFO IP: 123.231.231.123 User: fred A message at INFO level with 2 parameters - - -.. _multiple-processes: - -Logging to a single file from multiple processes ------------------------------------------------- - -Although logging is thread-safe, and logging to a single file from multiple -threads in a single process *is* supported, logging to a single file from -*multiple processes* is *not* supported, because there is no standard way to -serialize access to a single file across multiple processes in Python. If you -need to log to a single file from multiple processes, one way of doing this is -to have all the processes log to a :class:`~handlers.SocketHandler`, and have a -separate process which implements a socket server which reads from the socket -and logs to file. (If you prefer, you can dedicate one thread in one of the -existing processes to perform this function.) -:ref:`This section <network-logging>` documents this approach in more detail and -includes a working socket receiver which can be used as a starting point for you -to adapt in your own applications. - -If you are using a recent version of Python which includes the -:mod:`multiprocessing` module, you could write your own handler which uses the -:class:`~multiprocessing.Lock` class from this module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide -working lock functionality on all platforms (see -https://bugs.python.org/issue3770). - -.. currentmodule:: logging.handlers - -Alternatively, you can use a ``Queue`` and a :class:`QueueHandler` to send -all logging events to one of the processes in your multi-process application. -The following example script demonstrates how you can do this; in the example -a separate listener process listens for events sent by other processes and logs -them according to its own logging configuration. Although the example only -demonstrates one way of doing it (for example, you may want to use a listener -thread rather than a separate listener process -- the implementation would be -analogous) it does allow for completely different logging configurations for -the listener and the other processes in your application, and can be used as -the basis for code meeting your own specific requirements:: - - # You'll need these imports in your own code - import logging - import logging.handlers - import multiprocessing - - # Next two import lines for this demo only - from random import choice, random - import time - - # - # Because you'll want to define the logging configurations for listener and workers, the - # listener and worker process functions take a configurer parameter which is a callable - # for configuring logging for that process. These functions are also passed the queue, - # which they use for communication. - # - # In practice, you can configure the listener however you want, but note that in this - # simple example, the listener does not apply level or filter logic to received records. - # In practice, you would probably want to do this logic in the worker processes, to avoid - # sending events which would be filtered out between processes. - # - # The size of the rotated files is made small so you can see the results easily. - def listener_configurer(): - root = logging.getLogger() - h = logging.handlers.RotatingFileHandler('mptest.log', 'a', 300, 10) - f = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s') - h.setFormatter(f) - root.addHandler(h) - - # This is the listener process top-level loop: wait for logging events - # (LogRecords)on the queue and handle them, quit when you get a None for a - # LogRecord. - def listener_process(queue, configurer): - configurer() - while True: - try: - record = queue.get() - if record is None: # We send this as a sentinel to tell the listener to quit. - break - logger = logging.getLogger(record.name) - logger.handle(record) # No level or filter logic applied - just do it! - except Exception: - import sys, traceback - print('Whoops! Problem:', file=sys.stderr) - traceback.print_exc(file=sys.stderr) - - # Arrays used for random selections in this demo - - LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, - logging.ERROR, logging.CRITICAL] - - LOGGERS = ['a.b.c', 'd.e.f'] - - MESSAGES = [ - 'Random message #1', - 'Random message #2', - 'Random message #3', - ] - - # The worker configuration is done at the start of the worker process run. - # Note that on Windows you can't rely on fork semantics, so each process - # will run the logging configuration code when it starts. - def worker_configurer(queue): - h = logging.handlers.QueueHandler(queue) # Just the one handler needed - root = logging.getLogger() - root.addHandler(h) - # send all messages, for demo; no other level or filter logic applied. - root.setLevel(logging.DEBUG) - - # This is the worker process top-level loop, which just logs ten events with - # random intervening delays before terminating. - # The print messages are just so you know it's doing something! - def worker_process(queue, configurer): - configurer(queue) - name = multiprocessing.current_process().name - print('Worker started: %s' % name) - for i in range(10): - time.sleep(random()) - logger = logging.getLogger(choice(LOGGERS)) - level = choice(LEVELS) - message = choice(MESSAGES) - logger.log(level, message) - print('Worker finished: %s' % name) - - # Here's where the demo gets orchestrated. Create the queue, create and start - # the listener, create ten workers and start them, wait for them to finish, - # then send a None to the queue to tell the listener to finish. - def main(): - queue = multiprocessing.Queue(-1) - listener = multiprocessing.Process(target=listener_process, - args=(queue, listener_configurer)) - listener.start() - workers = [] - for i in range(10): - worker = multiprocessing.Process(target=worker_process, - args=(queue, worker_configurer)) - workers.append(worker) - worker.start() - for w in workers: - w.join() - queue.put_nowait(None) - listener.join() - - if __name__ == '__main__': - main() - -A variant of the above script keeps the logging in the main process, in a -separate thread:: - - import logging - import logging.config - import logging.handlers - from multiprocessing import Process, Queue - import random - import threading - import time - - def logger_thread(q): - while True: - record = q.get() - if record is None: - break - logger = logging.getLogger(record.name) - logger.handle(record) - - - def worker_process(q): - qh = logging.handlers.QueueHandler(q) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(qh) - levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL] - loggers = ['foo', 'foo.bar', 'foo.bar.baz', - 'spam', 'spam.ham', 'spam.ham.eggs'] - for i in range(100): - lvl = random.choice(levels) - logger = logging.getLogger(random.choice(loggers)) - logger.log(lvl, 'Message no. %d', i) - - if __name__ == '__main__': - q = Queue() - d = { - 'version': 1, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - }, - 'file': { - 'class': 'logging.FileHandler', - 'filename': 'mplog.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'foofile': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-foo.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'errors': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-errors.log', - 'mode': 'w', - 'level': 'ERROR', - 'formatter': 'detailed', - }, - }, - 'loggers': { - 'foo': { - 'handlers': ['foofile'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console', 'file', 'errors'] - }, - } - workers = [] - for i in range(5): - wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(q,)) - workers.append(wp) - wp.start() - logging.config.dictConfig(d) - lp = threading.Thread(target=logger_thread, args=(q,)) - lp.start() - # At this point, the main process could do some useful work of its own - # Once it's done that, it can wait for the workers to terminate... - for wp in workers: - wp.join() - # And now tell the logging thread to finish up, too - q.put(None) - lp.join() - -This variant shows how you can e.g. apply configuration for particular loggers -- e.g. the ``foo`` logger has a special handler which stores all events in the -``foo`` subsystem in a file ``mplog-foo.log``. This will be used by the logging -machinery in the main process (even though the logging events are generated in -the worker processes) to direct the messages to the appropriate destinations. - -Using file rotation -------------------- - -.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see <https://pymotw.com/3/logging/>) - -Sometimes you want to let a log file grow to a certain size, then open a new -file and log to that. You may want to keep a certain number of these files, and -when that many files have been created, rotate the files so that the number of -files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`~handlers.RotatingFileHandler`:: - - import glob - import logging - import logging.handlers - - LOG_FILENAME = 'logging_rotatingfile_example.out' - - # Set up a specific logger with our desired output level - my_logger = logging.getLogger('MyLogger') - my_logger.setLevel(logging.DEBUG) - - # Add the log message handler to the logger - handler = logging.handlers.RotatingFileHandler( - LOG_FILENAME, maxBytes=20, backupCount=5) - - my_logger.addHandler(handler) - - # Log some messages - for i in range(20): - my_logger.debug('i = %d' % i) - - # See what files are created - logfiles = glob.glob('%s*' % LOG_FILENAME) - - for filename in logfiles: - print(filename) - -The result should be 6 separate files, each with part of the log history for the -application: - -.. code-block:: none - - logging_rotatingfile_example.out - logging_rotatingfile_example.out.1 - logging_rotatingfile_example.out.2 - logging_rotatingfile_example.out.3 - logging_rotatingfile_example.out.4 - logging_rotatingfile_example.out.5 - -The most current file is always :file:`logging_rotatingfile_example.out`, -and each time it reaches the size limit it is renamed with the suffix -``.1``. Each of the existing backup files is renamed to increment the suffix -(``.1`` becomes ``.2``, etc.) and the ``.6`` file is erased. - -Obviously this example sets the log length much too small as an extreme -example. You would want to set *maxBytes* to an appropriate value. - -.. _format-styles: - -Use of alternative formatting styles ------------------------------------- - -When logging was added to the Python standard library, the only way of -formatting messages with variable content was to use the %-formatting -method. Since then, Python has gained two new formatting approaches: -:class:`string.Template` (added in Python 2.4) and :meth:`str.format` -(added in Python 2.6). - -Logging (as of 3.2) provides improved support for these two additional -formatting styles. The :class:`Formatter` class been enhanced to take an -additional, optional keyword parameter named ``style``. This defaults to -``'%'``, but other possible values are ``'{'`` and ``'$'``, which correspond -to the other two formatting styles. Backwards compatibility is maintained by -default (as you would expect), but by explicitly specifying a style parameter, -you get the ability to specify format strings which work with -:meth:`str.format` or :class:`string.Template`. Here's an example console -session to show the possibilities: - -.. code-block:: pycon - - >>> import logging - >>> root = logging.getLogger() - >>> root.setLevel(logging.DEBUG) - >>> handler = logging.StreamHandler() - >>> bf = logging.Formatter('{asctime} {name} {levelname:8s} {message}', - ... style='{') - >>> handler.setFormatter(bf) - >>> root.addHandler(handler) - >>> logger = logging.getLogger('foo.bar') - >>> logger.debug('This is a DEBUG message') - 2010-10-28 15:11:55,341 foo.bar DEBUG This is a DEBUG message - >>> logger.critical('This is a CRITICAL message') - 2010-10-28 15:12:11,526 foo.bar CRITICAL This is a CRITICAL message - >>> df = logging.Formatter('$asctime $name ${levelname} $message', - ... style='$') - >>> handler.setFormatter(df) - >>> logger.debug('This is a DEBUG message') - 2010-10-28 15:13:06,924 foo.bar DEBUG This is a DEBUG message - >>> logger.critical('This is a CRITICAL message') - 2010-10-28 15:13:11,494 foo.bar CRITICAL This is a CRITICAL message - >>> - -Note that the formatting of logging messages for final output to logs is -completely independent of how an individual logging message is constructed. -That can still use %-formatting, as shown here:: - - >>> logger.error('This is an%s %s %s', 'other,', 'ERROR,', 'message') - 2010-10-28 15:19:29,833 foo.bar ERROR This is another, ERROR, message - >>> - -Logging calls (``logger.debug()``, ``logger.info()`` etc.) only take -positional parameters for the actual logging message itself, with keyword -parameters used only for determining options for how to handle the actual -logging call (e.g. the ``exc_info`` keyword parameter to indicate that -traceback information should be logged, or the ``extra`` keyword parameter -to indicate additional contextual information to be added to the log). So -you cannot directly make logging calls using :meth:`str.format` or -:class:`string.Template` syntax, because internally the logging package -uses %-formatting to merge the format string and the variable arguments. -There would no changing this while preserving backward compatibility, since -all logging calls which are out there in existing code will be using %-format -strings. - -There is, however, a way that you can use {}- and $- formatting to construct -your individual log messages. Recall that for a message you can use an -arbitrary object as a message format string, and that the logging package will -call ``str()`` on that object to get the actual format string. Consider the -following two classes:: - - class BraceMessage: - def __init__(self, fmt, *args, **kwargs): - self.fmt = fmt - self.args = args - self.kwargs = kwargs - - def __str__(self): - return self.fmt.format(*self.args, **self.kwargs) - - class DollarMessage: - def __init__(self, fmt, **kwargs): - self.fmt = fmt - self.kwargs = kwargs - - def __str__(self): - from string import Template - return Template(self.fmt).substitute(**self.kwargs) - -Either of these can be used in place of a format string, to allow {}- or -$-formatting to be used to build the actual "message" part which appears in the -formatted log output in place of "%(message)s" or "{message}" or "$message". -It's a little unwieldy to use the class names whenever you want to log -something, but it's quite palatable if you use an alias such as __ (double -underscore --- not to be confused with _, the single underscore used as a -synonym/alias for :func:`gettext.gettext` or its brethren). - -The above classes are not included in Python, though they're easy enough to -copy and paste into your own code. They can be used as follows (assuming that -they're declared in a module called ``wherever``): - -.. code-block:: pycon - - >>> from wherever import BraceMessage as __ - >>> print(__('Message with {0} {name}', 2, name='placeholders')) - Message with 2 placeholders - >>> class Point: pass - ... - >>> p = Point() - >>> p.x = 0.5 - >>> p.y = 0.5 - >>> print(__('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', - ... point=p)) - Message with coordinates: (0.50, 0.50) - >>> from wherever import DollarMessage as __ - >>> print(__('Message with $num $what', num=2, what='placeholders')) - Message with 2 placeholders - >>> - -While the above examples use ``print()`` to show how the formatting works, you -would of course use ``logger.debug()`` or similar to actually log using this -approach. - -One thing to note is that you pay no significant performance penalty with this -approach: the actual formatting happens not when you make the logging call, but -when (and if) the logged message is actually about to be output to a log by a -handler. So the only slightly unusual thing which might trip you up is that the -parentheses go around the format string and the arguments, not just the format -string. That's because the __ notation is just syntax sugar for a constructor -call to one of the XXXMessage classes. - -If you prefer, you can use a :class:`LoggerAdapter` to achieve a similar effect -to the above, as in the following example:: - - import logging - - class Message(object): - def __init__(self, fmt, args): - self.fmt = fmt - self.args = args - - def __str__(self): - return self.fmt.format(*self.args) - - class StyleAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super(StyleAdapter, self).__init__(logger, extra or {}) - - def log(self, level, msg, *args, **kwargs): - if self.isEnabledFor(level): - msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, Message(msg, args), (), **kwargs) - - logger = StyleAdapter(logging.getLogger(__name__)) - - def main(): - logger.debug('Hello, {}', 'world!') - - if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - main() - -The above script should log the message ``Hello, world!`` when run with -Python 3.2 or later. - - -.. currentmodule:: logging - -.. _custom-logrecord: - -Customizing ``LogRecord`` -------------------------- - -Every logging event is represented by a :class:`LogRecord` instance. -When an event is logged and not filtered out by a logger's level, a -:class:`LogRecord` is created, populated with information about the event and -then passed to the handlers for that logger (and its ancestors, up to and -including the logger where further propagation up the hierarchy is disabled). -Before Python 3.2, there were only two places where this creation was done: - -* :meth:`Logger.makeRecord`, which is called in the normal process of - logging an event. This invoked :class:`LogRecord` directly to create an - instance. -* :func:`makeLogRecord`, which is called with a dictionary containing - attributes to be added to the LogRecord. This is typically invoked when a - suitable dictionary has been received over the network (e.g. in pickle form - via a :class:`~handlers.SocketHandler`, or in JSON form via an - :class:`~handlers.HTTPHandler`). - -This has usually meant that if you need to do anything special with a -:class:`LogRecord`, you've had to do one of the following. - -* Create your own :class:`Logger` subclass, which overrides - :meth:`Logger.makeRecord`, and set it using :func:`~logging.setLoggerClass` - before any loggers that you care about are instantiated. -* Add a :class:`Filter` to a logger or handler, which does the - necessary special manipulation you need when its - :meth:`~Filter.filter` method is called. - -The first approach would be a little unwieldy in the scenario where (say) -several different libraries wanted to do different things. Each would attempt -to set its own :class:`Logger` subclass, and the one which did this last would -win. - -The second approach works reasonably well for many cases, but does not allow -you to e.g. use a specialized subclass of :class:`LogRecord`. Library -developers can set a suitable filter on their loggers, but they would have to -remember to do this every time they introduced a new logger (which they would -do simply by adding new packages or modules and doing :: - - logger = logging.getLogger(__name__) - -at module level). It's probably one too many things to think about. Developers -could also add the filter to a :class:`~logging.NullHandler` attached to their -top-level logger, but this would not be invoked if an application developer -attached a handler to a lower-level library logger --- so output from that -handler would not reflect the intentions of the library developer. - -In Python 3.2 and later, :class:`~logging.LogRecord` creation is done through a -factory, which you can specify. The factory is just a callable you can set with -:func:`~logging.setLogRecordFactory`, and interrogate with -:func:`~logging.getLogRecordFactory`. The factory is invoked with the same -signature as the :class:`~logging.LogRecord` constructor, as :class:`LogRecord` -is the default setting for the factory. - -This approach allows a custom factory to control all aspects of LogRecord -creation. For example, you could return a subclass, or just add some additional -attributes to the record once created, using a pattern similar to this:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - -This pattern allows different libraries to chain factories together, and as -long as they don't overwrite each other's attributes or unintentionally -overwrite the attributes provided as standard, there should be no surprises. -However, it should be borne in mind that each link in the chain adds run-time -overhead to all logging operations, and the technique should only be used when -the use of a :class:`Filter` does not provide the desired result. - - -.. _zeromq-handlers: - -Subclassing QueueHandler - a ZeroMQ example -------------------------------------------- - -You can use a :class:`QueueHandler` subclass to send messages to other kinds -of queues, for example a ZeroMQ 'publish' socket. In the example below,the -socket is created separately and passed to the handler (as its 'queue'):: - - import zmq # using pyzmq, the Python binding for ZeroMQ - import json # for serializing records portably - - ctx = zmq.Context() - sock = zmq.Socket(ctx, zmq.PUB) # or zmq.PUSH, or other suitable value - sock.bind('tcp://*:5556') # or wherever - - class ZeroMQSocketHandler(QueueHandler): - def enqueue(self, record): - self.queue.send_json(record.__dict__) - - - handler = ZeroMQSocketHandler(sock) - - -Of course there are other ways of organizing this, for example passing in the -data needed by the handler to create the socket:: - - class ZeroMQSocketHandler(QueueHandler): - def __init__(self, uri, socktype=zmq.PUB, ctx=None): - self.ctx = ctx or zmq.Context() - socket = zmq.Socket(self.ctx, socktype) - socket.bind(uri) - super().__init__(socket) - - def enqueue(self, record): - self.queue.send_json(record.__dict__) - - def close(self): - self.queue.close() - - -Subclassing QueueListener - a ZeroMQ example --------------------------------------------- - -You can also subclass :class:`QueueListener` to get messages from other kinds -of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: - - class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, *handlers, **kwargs): - self.ctx = kwargs.get('ctx') or zmq.Context() - socket = zmq.Socket(self.ctx, zmq.SUB) - socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything - socket.connect(uri) - super().__init__(socket, *handlers, **kwargs) - - def dequeue(self): - msg = self.queue.recv_json() - return logging.makeLogRecord(msg) - - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :ref:`A basic logging tutorial <logging-basic-tutorial>` - - :ref:`A more advanced logging tutorial <logging-advanced-tutorial>` - - -An example dictionary-based configuration ------------------------------------------ - -Below is an example of a logging configuration dictionary - it's taken from -the `documentation on the Django project <https://docs.djangoproject.com/en/1.9/topics/logging/#configuring-logging>`_. -This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - }, - 'filters': { - 'special': { - '()': 'project.logging.SpecialFilter', - 'foo': 'bar', - } - }, - 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' - }, - 'mail_admins': { - 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler', - 'filters': ['special'] - } - }, - 'loggers': { - 'django': { - 'handlers':['null'], - 'propagate': True, - 'level':'INFO', - }, - 'django.request': { - 'handlers': ['mail_admins'], - 'level': 'ERROR', - 'propagate': False, - }, - 'myproject.custom': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - 'filters': ['special'] - } - } - } - -For more information about this configuration, you can see the `relevant -section <https://docs.djangoproject.com/en/1.9/topics/logging/#configuring-logging>`_ -of the Django documentation. - -.. _cookbook-rotator-namer: - -Using a rotator and namer to customize log rotation processing --------------------------------------------------------------- - -An example of how you can define a namer and rotator is given in the following -snippet, which shows zlib-based compression of the log file:: - - def namer(name): - return name + ".gz" - - def rotator(source, dest): - with open(source, "rb") as sf: - data = sf.read() - compressed = zlib.compress(data, 9) - with open(dest, "wb") as df: - df.write(compressed) - os.remove(source) - - rh = logging.handlers.RotatingFileHandler(...) - rh.rotator = rotator - rh.namer = namer - -These are not "true" .gz files, as they are bare compressed data, with no -"container" such as you’d find in an actual gzip file. This snippet is just -for illustration purposes. - -A more elaborate multiprocessing example ----------------------------------------- - -The following working example shows how logging can be used with multiprocessing -using configuration files. The configurations are fairly simple, but serve to -illustrate how more complex ones could be implemented in a real multiprocessing -scenario. - -In the example, the main process spawns a listener process and some worker -processes. Each of the main process, the listener and the workers have three -separate configurations (the workers all share the same configuration). We can -see logging in the main process, how the workers log to a QueueHandler and how -the listener implements a QueueListener and a more complex logging -configuration, and arranges to dispatch events received via the queue to the -handlers specified in the configuration. Note that these configurations are -purely illustrative, but you should be able to adapt this example to your own -scenario. - -Here's the script - the docstrings and the comments hopefully explain how it -works:: - - import logging - import logging.config - import logging.handlers - from multiprocessing import Process, Queue, Event, current_process - import os - import random - import time - - class MyHandler: - """ - A simple handler for logging events. It runs in the listener process and - dispatches events to loggers based on the name in the received record, - which then get dispatched, by the logging system, to the handlers - configured for those loggers. - """ - def handle(self, record): - logger = logging.getLogger(record.name) - # The process name is transformed just to show that it's the listener - # doing the logging to files and console - record.processName = '%s (for %s)' % (current_process().name, record.processName) - logger.handle(record) - - def listener_process(q, stop_event, config): - """ - This could be done in the main process, but is just done in a separate - process for illustrative purposes. - - This initialises logging according to the specified configuration, - starts the listener and waits for the main process to signal completion - via the event. The listener is then stopped, and the process exits. - """ - logging.config.dictConfig(config) - listener = logging.handlers.QueueListener(q, MyHandler()) - listener.start() - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - stop_event.wait() - listener.stop() - - def worker_process(config): - """ - A number of these are spawned for the purpose of illustration. In - practice, they could be a heterogeneous bunch of processes rather than - ones which are identical to each other. - - This initialises logging according to the specified configuration, - and logs a hundred messages with random levels to randomly selected - loggers. - - A small sleep is added to allow other processes a chance to run. This - is not strictly needed, but it mixes the output from the different - processes a bit more than if it's left out. - """ - logging.config.dictConfig(config) - levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL] - loggers = ['foo', 'foo.bar', 'foo.bar.baz', - 'spam', 'spam.ham', 'spam.ham.eggs'] - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - for i in range(100): - lvl = random.choice(levels) - logger = logging.getLogger(random.choice(loggers)) - logger.log(lvl, 'Message no. %d', i) - time.sleep(0.01) - - def main(): - q = Queue() - # The main process gets a simple configuration which prints to the console. - config_initial = { - 'version': 1, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'] - }, - } - # The worker process configuration is just a QueueHandler attached to the - # root logger, which allows all messages to be sent to the queue. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_worker = { - 'version': 1, - 'disable_existing_loggers': True, - 'handlers': { - 'queue': { - 'class': 'logging.handlers.QueueHandler', - 'queue': q, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['queue'] - }, - } - # The listener process configuration shows that the full flexibility of - # logging configuration is available to dispatch events to handlers however - # you want. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_listener = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - }, - 'simple': { - 'class': 'logging.Formatter', - 'format': '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'simple', - }, - 'file': { - 'class': 'logging.FileHandler', - 'filename': 'mplog.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'foofile': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-foo.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'errors': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-errors.log', - 'mode': 'w', - 'level': 'ERROR', - 'formatter': 'detailed', - }, - }, - 'loggers': { - 'foo': { - 'handlers': ['foofile'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console', 'file', 'errors'] - }, - } - # Log some initial events, just to show that logging in the parent works - # normally. - logging.config.dictConfig(config_initial) - logger = logging.getLogger('setup') - logger.info('About to create workers ...') - workers = [] - for i in range(5): - wp = Process(target=worker_process, name='worker %d' % (i + 1), - args=(config_worker,)) - workers.append(wp) - wp.start() - logger.info('Started worker: %s', wp.name) - logger.info('About to create listener ...') - stop_event = Event() - lp = Process(target=listener_process, name='listener', - args=(q, stop_event, config_listener)) - lp.start() - logger.info('Started listener') - # We now hang around for the workers to finish their work. - for wp in workers: - wp.join() - # Workers all done, listening can now stop. - # Logging in the parent still works normally. - logger.info('Telling listener to stop ...') - stop_event.set() - lp.join() - logger.info('All done.') - - if __name__ == '__main__': - main() - - -Inserting a BOM into messages sent to a SysLogHandler ------------------------------------------------------ - -:rfc:`5424` requires that a -Unicode message be sent to a syslog daemon as a set of bytes which have the -following structure: an optional pure-ASCII component, followed by a UTF-8 Byte -Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the -:rfc:`relevant section of the specification <5424#section-6>`.) - -In Python 3.1, code was added to -:class:`~logging.handlers.SysLogHandler` to insert a BOM into the message, but -unfortunately, it was implemented incorrectly, with the BOM appearing at the -beginning of the message and hence not allowing any pure-ASCII component to -appear before it. - -As this behaviour is broken, the incorrect BOM insertion code is being removed -from Python 3.2.4 and later. However, it is not being replaced, and if you -want to produce :rfc:`5424`-compliant messages which include a BOM, an optional -pure-ASCII sequence before it and arbitrary Unicode after it, encoded using -UTF-8, then you need to do the following: - -#. Attach a :class:`~logging.Formatter` instance to your - :class:`~logging.handlers.SysLogHandler` instance, with a format string - such as:: - - 'ASCII section\ufeffUnicode section' - - The Unicode code point U+FEFF, when encoded using UTF-8, will be - encoded as a UTF-8 BOM -- the byte-string ``b'\xef\xbb\xbf'``. - -#. Replace the ASCII section with whatever placeholders you like, but make sure - that the data that appears in there after substitution is always ASCII (that - way, it will remain unchanged after UTF-8 encoding). - -#. Replace the Unicode section with whatever placeholders you like; if the data - which appears there after substitution contains characters outside the ASCII - range, that's fine -- it will be encoded using UTF-8. - -The formatted message *will* be encoded using UTF-8 encoding by -``SysLogHandler``. If you follow the above rules, you should be able to produce -:rfc:`5424`-compliant messages. If you don't, logging may not complain, but your -messages will not be RFC 5424-compliant, and your syslog daemon may complain. - - -Implementing structured logging -------------------------------- - -Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be circumstances where you want to output -messages in a structured format which *is* capable of being parsed by a program -(without needing complex regular expressions to parse the log message). This is -straightforward to achieve using the logging package. There are a number of -ways in which this could be achieved, but the following is a simple approach -which uses JSON to serialise the event in a machine-parseable manner:: - - import json - import logging - - class StructuredMessage(object): - def __init__(self, message, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - return '%s >>> %s' % (self.message, json.dumps(self.kwargs)) - - _ = StructuredMessage # optional, to improve readability - - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456)) - -If the above script is run, it prints: - -.. code-block:: none - - message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"} - -Note that the order of items might be different according to the version of -Python used. - -If you need more specialised processing, you can use a custom JSON encoder, -as in the following complete example:: - - from __future__ import unicode_literals - - import json - import logging - - # This next bit is to ensure the script runs unchanged on 2.x and 3.x - try: - unicode - except NameError: - unicode = str - - class Encoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, set): - return tuple(o) - elif isinstance(o, unicode): - return o.encode('unicode_escape').decode('ascii') - return super(Encoder, self).default(o) - - class StructuredMessage(object): - def __init__(self, message, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - s = Encoder().encode(self.kwargs) - return '%s >>> %s' % (self.message, s) - - _ = StructuredMessage # optional, to improve readability - - def main(): - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', set_value={1, 2, 3}, snowman='\u2603')) - - if __name__ == '__main__': - main() - -When the above script is run, it prints: - -.. code-block:: none - - message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]} - -Note that the order of items might be different according to the version of -Python used. - - -.. _custom-handlers: - -.. currentmodule:: logging.config - -Customizing handlers with :func:`dictConfig` --------------------------------------------- - -There are times when you want to customize logging handlers in particular ways, -and if you use :func:`dictConfig` you may be able to do this without -subclassing. As an example, consider that you may want to set the ownership of a -log file. On POSIX, this is easily done using :func:`shutil.chown`, but the file -handlers in the stdlib don't offer built-in support. You can customize handler -creation using a plain function such as:: - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - -You can then specify, in a logging configuration passed to :func:`dictConfig`, -that a logging handler be created by calling this function:: - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - -In this example I am setting the ownership using the ``pulse`` user and group, -just for the purposes of illustration. Putting it together into a working -script, ``chowntest.py``:: - - import logging, logging.config, os, shutil - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(LOGGING) - logger = logging.getLogger('mylogger') - logger.debug('A debug message') - -To run this, you will probably need to run as ``root``: - -.. code-block:: shell-session - - $ sudo python3.3 chowntest.py - $ cat chowntest.log - 2013-11-05 09:34:51,128 DEBUG mylogger A debug message - $ ls -l chowntest.log - -rw-r--r-- 1 pulse pulse 55 2013-11-05 09:34 chowntest.log - -Note that this example uses Python 3.3 because that's where :func:`shutil.chown` -makes an appearance. This approach should work with any Python version that -supports :func:`dictConfig` - namely, Python 2.7, 3.2 or later. With pre-3.3 -versions, you would need to implement the actual ownership change using e.g. -:func:`os.chown`. - -In practice, the handler-creating function may be in a utility module somewhere -in your project. Instead of the line in the configuration:: - - '()': owned_file_handler, - -you could use e.g.:: - - '()': 'ext://project.util.owned_file_handler', - -where ``project.util`` can be replaced with the actual name of the package -where the function resides. In the above working script, using -``'ext://__main__.owned_file_handler'`` should work. Here, the actual callable -is resolved by :func:`dictConfig` from the ``ext://`` specification. - -This example hopefully also points the way to how you could implement other -types of file change - e.g. setting specific POSIX permission bits - in the -same way, using :func:`os.chmod`. - -Of course, the approach could also be extended to types of handler other than a -:class:`~logging.FileHandler` - for example, one of the rotating file handlers, -or a different type of handler altogether. - - -.. currentmodule:: logging - -.. _formatting-styles: - -Using particular formatting styles throughout your application --------------------------------------------------------------- - -In Python 3.2, the :class:`~logging.Formatter` gained a ``style`` keyword -parameter which, while defaulting to ``%`` for backward compatibility, allowed -the specification of ``{`` or ``$`` to support the formatting approaches -supported by :meth:`str.format` and :class:`string.Template`. Note that this -governs the formatting of logging messages for final output to logs, and is -completely orthogonal to how an individual logging message is constructed. - -Logging calls (:meth:`~Logger.debug`, :meth:`~Logger.info` etc.) only take -positional parameters for the actual logging message itself, with keyword -parameters used only for determining options for how to handle the logging call -(e.g. the ``exc_info`` keyword parameter to indicate that traceback information -should be logged, or the ``extra`` keyword parameter to indicate additional -contextual information to be added to the log). So you cannot directly make -logging calls using :meth:`str.format` or :class:`string.Template` syntax, -because internally the logging package uses %-formatting to merge the format -string and the variable arguments. There would no changing this while preserving -backward compatibility, since all logging calls which are out there in existing -code will be using %-format strings. - -There have been suggestions to associate format styles with specific loggers, -but that approach also runs into backward compatibility problems because any -existing code could be using a given logger name and using %-formatting. - -For logging to work interoperably between any third-party libraries and your -code, decisions about formatting need to be made at the level of the -individual logging call. This opens up a couple of ways in which alternative -formatting styles can be accommodated. - - -Using LogRecord factories -^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Python 3.2, along with the :class:`~logging.Formatter` changes mentioned -above, the logging package gained the ability to allow users to set their own -:class:`LogRecord` subclasses, using the :func:`setLogRecordFactory` function. -You can use this to set your own subclass of :class:`LogRecord`, which does the -Right Thing by overriding the :meth:`~LogRecord.getMessage` method. The base -class implementation of this method is where the ``msg % args`` formatting -happens, and where you can substitute your alternate formatting; however, you -should be careful to support all formatting styles and allow %-formatting as -the default, to ensure interoperability with other code. Care should also be -taken to call ``str(self.msg)``, just as the base implementation does. - -Refer to the reference documentation on :func:`setLogRecordFactory` and -:class:`LogRecord` for more information. - - -Using custom message objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There is another, perhaps simpler way that you can use {}- and $- formatting to -construct your individual log messages. You may recall (from -:ref:`arbitrary-object-messages`) that when logging you can use an arbitrary -object as a message format string, and that the logging package will call -:func:`str` on that object to get the actual format string. Consider the -following two classes:: - - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): - self.fmt = fmt - self.args = args - self.kwargs = kwargs - - def __str__(self): - return self.fmt.format(*self.args, **self.kwargs) - - class DollarMessage(object): - def __init__(self, fmt, **kwargs): - self.fmt = fmt - self.kwargs = kwargs - - def __str__(self): - from string import Template - return Template(self.fmt).substitute(**self.kwargs) - -Either of these can be used in place of a format string, to allow {}- or -$-formatting to be used to build the actual "message" part which appears in the -formatted log output in place of “%(message)s” or “{message}” or “$message”. -If you find it a little unwieldy to use the class names whenever you want to log -something, you can make it more palatable if you use an alias such as ``M`` or -``_`` for the message (or perhaps ``__``, if you are using ``_`` for -localization). - -Examples of this approach are given below. Firstly, formatting with -:meth:`str.format`:: - - >>> __ = BraceMessage - >>> print(__('Message with {0} {1}', 2, 'placeholders')) - Message with 2 placeholders - >>> class Point: pass - ... - >>> p = Point() - >>> p.x = 0.5 - >>> p.y = 0.5 - >>> print(__('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', point=p)) - Message with coordinates: (0.50, 0.50) - -Secondly, formatting with :class:`string.Template`:: - - >>> __ = DollarMessage - >>> print(__('Message with $num $what', num=2, what='placeholders')) - Message with 2 placeholders - >>> - -One thing to note is that you pay no significant performance penalty with this -approach: the actual formatting happens not when you make the logging call, but -when (and if) the logged message is actually about to be output to a log by a -handler. So the only slightly unusual thing which might trip you up is that the -parentheses go around the format string and the arguments, not just the format -string. That’s because the __ notation is just syntax sugar for a constructor -call to one of the ``XXXMessage`` classes shown above. - - -.. _filters-dictconfig: - -.. currentmodule:: logging.config - -Configuring filters with :func:`dictConfig` -------------------------------------------- - -You *can* configure filters using :func:`~logging.config.dictConfig`, though it -might not be obvious at first glance how to do it (hence this recipe). Since -:class:`~logging.Filter` is the only filter class included in the standard -library, and it is unlikely to cater to many requirements (it's only there as a -base class), you will typically need to define your own :class:`~logging.Filter` -subclass with an overridden :meth:`~logging.Filter.filter` method. To do this, -specify the ``()`` key in the configuration dictionary for the filter, -specifying a callable which will be used to create the filter (a class is the -most obvious, but you can provide any callable which returns a -:class:`~logging.Filter` instance). Here is a complete example:: - - import logging - import logging.config - import sys - - class MyFilter(logging.Filter): - def __init__(self, param=None): - self.param = param - - def filter(self, record): - if self.param is None: - allow = True - else: - allow = self.param not in record.msg - if allow: - record.msg = 'changed: ' + record.msg - return allow - - LOGGING = { - 'version': 1, - 'filters': { - 'myfilter': { - '()': MyFilter, - 'param': 'noshow', - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'filters': ['myfilter'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'] - }, - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.debug('hello') - logging.debug('hello - noshow') - -This example shows how you can pass configuration data to the callable which -constructs the instance, in the form of keyword parameters. When run, the above -script will print: - -.. code-block:: none - - changed: hello - -which shows that the filter is working as configured. - -A couple of extra points to note: - -* If you can't refer to the callable directly in the configuration (e.g. if it - lives in a different module, and you can't import it directly where the - configuration dictionary is), you can use the form ``ext://...`` as described - in :ref:`logging-config-dict-externalobj`. For example, you could have used - the text ``'ext://__main__.MyFilter'`` instead of ``MyFilter`` in the above - example. - -* As well as for filters, this technique can also be used to configure custom - handlers and formatters. See :ref:`logging-config-dict-userdef` for more - information on how logging supports using user-defined objects in its - configuration, and see the other cookbook recipe :ref:`custom-handlers` above. - - -.. _custom-format-exception: - -Customized exception formatting -------------------------------- - -There might be times when you want to do customized exception formatting - for -argument's sake, let's say you want exactly one line per logged event, even -when exception information is present. You can do this with a custom formatter -class, as shown in the following example:: - - import logging - - class OneLineExceptionFormatter(logging.Formatter): - def formatException(self, exc_info): - """ - Format an exception so that it prints on a single line. - """ - result = super(OneLineExceptionFormatter, self).formatException(exc_info) - return repr(result) # or format into one line however you want to - - def format(self, record): - s = super(OneLineExceptionFormatter, self).format(record) - if record.exc_text: - s = s.replace('\n', '') + '|' - return s - - def configure_logging(): - fh = logging.FileHandler('output.txt', 'w') - f = OneLineExceptionFormatter('%(asctime)s|%(levelname)s|%(message)s|', - '%d/%m/%Y %H:%M:%S') - fh.setFormatter(f) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(fh) - - def main(): - configure_logging() - logging.info('Sample message') - try: - x = 1 / 0 - except ZeroDivisionError as e: - logging.exception('ZeroDivisionError: %s', e) - - if __name__ == '__main__': - main() - -When run, this produces a file with exactly two lines: - -.. code-block:: none - - 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| - -While the above treatment is simplistic, it points the way to how exception -information can be formatted to your liking. The :mod:`traceback` module may be -helpful for more specialized needs. - -.. _spoken-messages: - -Speaking logging messages -------------------------- - -There might be situations when it is desirable to have logging messages rendered -in an audible rather than a visible format. This is easy to do if you have -text-to-speech (TTS) functionality available in your system, even if it doesn't have -a Python binding. Most TTS systems have a command line program you can run, and -this can be invoked from a handler using :mod:`subprocess`. It's assumed here -that TTS command line programs won't expect to interact with users or take a -long time to complete, and that the frequency of logged messages will be not so -high as to swamp the user with messages, and that it's acceptable to have the -messages spoken one at a time rather than concurrently, The example implementation -below waits for one message to be spoken before the next is processed, and this -might cause other handlers to be kept waiting. Here is a short example showing -the approach, which assumes that the ``espeak`` TTS package is available:: - - import logging - import subprocess - import sys - - class TTSHandler(logging.Handler): - def emit(self, record): - msg = self.format(record) - # Speak slowly in a female English voice - cmd = ['espeak', '-s150', '-ven+f3', msg] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # wait for the program to finish - p.communicate() - - def configure_logging(): - h = TTSHandler() - root = logging.getLogger() - root.addHandler(h) - # the default formatter just returns the message - root.setLevel(logging.DEBUG) - - def main(): - logging.info('Hello') - logging.debug('Goodbye') - - if __name__ == '__main__': - configure_logging() - sys.exit(main()) - -When run, this script should say "Hello" and then "Goodbye" in a female voice. - -The above approach can, of course, be adapted to other TTS systems and even -other systems altogether which can process messages via external programs run -from a command line. - - -.. _buffered-logging: - -Buffering logging messages and outputting them conditionally ------------------------------------------------------------- - -There might be situations where you want to log messages in a temporary area -and only output them if a certain condition occurs. For example, you may want to -start logging debug events in a function, and if the function completes without -errors, you don't want to clutter the log with the collected debug information, -but if there is an error, you want all the debug information to be output as well -as the error. - -Here is an example which shows how you could do this using a decorator for your -functions where you want logging to behave this way. It makes use of the -:class:`logging.handlers.MemoryHandler`, which allows buffering of logged events -until some condition occurs, at which point the buffered events are ``flushed`` -- passed to another handler (the ``target`` handler) for processing. By default, -the ``MemoryHandler`` flushed when its buffer gets filled up or an event whose -level is greater than or equal to a specified threshold is seen. You can use this -recipe with a more specialised subclass of ``MemoryHandler`` if you want custom -flushing behavior. - -The example script has a simple function, ``foo``, which just cycles through -all the logging levels, writing to ``sys.stderr`` to say what level it's about -to log at, and then actually logging a message at that level. You can pass a -parameter to ``foo`` which, if true, will log at ERROR and CRITICAL levels - -otherwise, it only logs at DEBUG, INFO and WARNING levels. - -The script just arranges to decorate ``foo`` with a decorator which will do the -conditional logging that's required. The decorator takes a logger as a parameter -and attaches a memory handler for the duration of the call to the decorated -function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. - -Here's the script:: - - import logging - from logging.handlers import MemoryHandler - import sys - - logger = logging.getLogger(__name__) - logger.addHandler(logging.NullHandler()) - - def log_if_errors(logger, target_handler=None, flush_level=None, capacity=None): - if target_handler is None: - target_handler = logging.StreamHandler() - if flush_level is None: - flush_level = logging.ERROR - if capacity is None: - capacity = 100 - handler = MemoryHandler(capacity, flushLevel=flush_level, target=target_handler) - - def decorator(fn): - def wrapper(*args, **kwargs): - logger.addHandler(handler) - try: - return fn(*args, **kwargs) - except Exception: - logger.exception('call failed') - raise - finally: - super(MemoryHandler, handler).flush() - logger.removeHandler(handler) - return wrapper - - return decorator - - def write_line(s): - sys.stderr.write('%s\n' % s) - - def foo(fail=False): - write_line('about to log at DEBUG ...') - logger.debug('Actually logged at DEBUG') - write_line('about to log at INFO ...') - logger.info('Actually logged at INFO') - write_line('about to log at WARNING ...') - logger.warning('Actually logged at WARNING') - if fail: - write_line('about to log at ERROR ...') - logger.error('Actually logged at ERROR') - write_line('about to log at CRITICAL ...') - logger.critical('Actually logged at CRITICAL') - return fail - - decorated_foo = log_if_errors(logger)(foo) - - if __name__ == '__main__': - logger.setLevel(logging.DEBUG) - write_line('Calling undecorated foo with False') - assert not foo(False) - write_line('Calling undecorated foo with True') - assert foo(True) - write_line('Calling decorated foo with False') - assert not decorated_foo(False) - write_line('Calling decorated foo with True') - assert decorated_foo(True) - -When this script is run, the following output should be observed: - -.. code-block:: none - - Calling undecorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling undecorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - about to log at CRITICAL ... - Calling decorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling decorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - Actually logged at DEBUG - Actually logged at INFO - Actually logged at WARNING - Actually logged at ERROR - about to log at CRITICAL ... - Actually logged at CRITICAL - -As you can see, actual logging output only occurs when an event is logged whose -severity is ERROR or greater, but in that case, any previous events at lower -severities are also logged. - -You can of course use the conventional means of decoration:: - - @log_if_errors(logger) - def foo(fail=False): - ... - - -.. _utc-formatting: - -Formatting times using UTC (GMT) via configuration --------------------------------------------------- - -Sometimes you want to format times using UTC, which can be done using a class -such as `UTCFormatter`, shown below:: - - import logging - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - -and you can then use the ``UTCFormatter`` in your code instead of -:class:`~logging.Formatter`. If you want to do that via configuration, you can -use the :func:`~logging.config.dictConfig` API with an approach illustrated by -the following complete example:: - - import logging - import logging.config - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'utc': { - '()': UTCFormatter, - 'format': '%(asctime)s %(message)s', - }, - 'local': { - 'format': '%(asctime)s %(message)s', - } - }, - 'handlers': { - 'console1': { - 'class': 'logging.StreamHandler', - 'formatter': 'utc', - }, - 'console2': { - 'class': 'logging.StreamHandler', - 'formatter': 'local', - }, - }, - 'root': { - 'handlers': ['console1', 'console2'], - } - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.warning('The local time is %s', time.asctime()) - -When this script is run, it should print something like: - -.. code-block:: none - - 2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - 2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - -showing how the time is formatted both as local time and UTC, one for each -handler. - - -.. _context-manager: - -Using a context manager for selective logging ---------------------------------------------- - -There are times when it would be useful to temporarily change the logging -configuration and revert it back after doing something. For this, a context -manager is the most obvious way of saving and restoring the logging context. -Here is a simple example of such a context manager, which allows you to -optionally change the logging level and add a logging handler purely in the -scope of the context manager:: - - import logging - import sys - - class LoggingContext(object): - def __init__(self, logger, level=None, handler=None, close=True): - self.logger = logger - self.level = level - self.handler = handler - self.close = close - - def __enter__(self): - if self.level is not None: - self.old_level = self.logger.level - self.logger.setLevel(self.level) - if self.handler: - self.logger.addHandler(self.handler) - - def __exit__(self, et, ev, tb): - if self.level is not None: - self.logger.setLevel(self.old_level) - if self.handler: - self.logger.removeHandler(self.handler) - if self.handler and self.close: - self.handler.close() - # implicit return of None => don't swallow exceptions - -If you specify a level value, the logger's level is set to that value in the -scope of the with block covered by the context manager. If you specify a -handler, it is added to the logger on entry to the block and removed on exit -from the block. You can also ask the manager to close the handler for you on -block exit - you could do this if you don't need the handler any more. - -To illustrate how it works, we can add the following block of code to the -above:: - - if __name__ == '__main__': - logger = logging.getLogger('foo') - logger.addHandler(logging.StreamHandler()) - logger.setLevel(logging.INFO) - logger.info('1. This should appear just once on stderr.') - logger.debug('2. This should not appear.') - with LoggingContext(logger, level=logging.DEBUG): - logger.debug('3. This should appear once on stderr.') - logger.debug('4. This should not appear.') - h = logging.StreamHandler(sys.stdout) - with LoggingContext(logger, level=logging.DEBUG, handler=h, close=True): - logger.debug('5. This should appear twice - once on stderr and once on stdout.') - logger.info('6. This should appear just once on stderr.') - logger.debug('7. This should not appear.') - -We initially set the logger's level to ``INFO``, so message #1 appears and -message #2 doesn't. We then change the level to ``DEBUG`` temporarily in the -following ``with`` block, and so message #3 appears. After the block exits, the -logger's level is restored to ``INFO`` and so message #4 doesn't appear. In the -next ``with`` block, we set the level to ``DEBUG`` again but also add a handler -writing to ``sys.stdout``. Thus, message #5 appears twice on the console (once -via ``stderr`` and once via ``stdout``). After the ``with`` statement's -completion, the status is as it was before so message #6 appears (like message -#1) whereas message #7 doesn't (just like message #2). - -If we run the resulting script, the result is as follows: - -.. code-block:: shell-session - - $ python logctx.py - 1. This should appear just once on stderr. - 3. This should appear once on stderr. - 5. This should appear twice - once on stderr and once on stdout. - 5. This should appear twice - once on stderr and once on stdout. - 6. This should appear just once on stderr. - -If we run it again, but pipe ``stderr`` to ``/dev/null``, we see the following, -which is the only message written to ``stdout``: - -.. code-block:: shell-session - - $ python logctx.py 2>/dev/null - 5. This should appear twice - once on stderr and once on stdout. - -Once again, but piping ``stdout`` to ``/dev/null``, we get: - -.. code-block:: shell-session - - $ python logctx.py >/dev/null - 1. This should appear just once on stderr. - 3. This should appear once on stderr. - 5. This should appear twice - once on stderr and once on stdout. - 6. This should appear just once on stderr. - -In this case, the message #5 printed to ``stdout`` doesn't appear, as expected. - -Of course, the approach described here can be generalised, for example to attach -logging filters temporarily. Note that the above code works in Python 2 as well -as Python 3. diff --git a/third_party/python/Doc/howto/logging.rst b/third_party/python/Doc/howto/logging.rst deleted file mode 100644 index af8c43275..000000000 --- a/third_party/python/Doc/howto/logging.rst +++ /dev/null @@ -1,1103 +0,0 @@ -============= -Logging HOWTO -============= - -:Author: Vinay Sajip <vinay_sajip at red-dove dot com> - -.. _logging-basic-tutorial: - -.. currentmodule:: logging - -Basic Logging Tutorial ----------------------- - -Logging is a means of tracking events that happen when some software runs. The -software's developer adds logging calls to their code to indicate that certain -events have occurred. An event is described by a descriptive message which can -optionally contain variable data (i.e. data that is potentially different for -each occurrence of the event). Events also have an importance which the -developer ascribes to the event; the importance can also be called the *level* -or *severity*. - -When to use logging -^^^^^^^^^^^^^^^^^^^ - -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. - -+-------------------------------------+--------------------------------------+ -| Task you want to perform | The best tool for the task | -+=====================================+======================================+ -| Display console output for ordinary | :func:`print` | -| usage of a command line script or | | -| program | | -+-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | -| for status monitoring or fault | detailed output for diagnostic | -| investigation) | purposes) | -+-------------------------------------+--------------------------------------+ -| Issue a warning regarding a | :func:`warnings.warn` in library | -| particular runtime event | code if the issue is avoidable and | -| | the client application should be | -| | modified to eliminate the warning | -| | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | -+-------------------------------------+--------------------------------------+ -| Report an error regarding a | Raise an exception | -| particular runtime event | | -+-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | -| server process) | appropriate for the specific error | -| | and application domain | -+-------------------------------------+--------------------------------------+ - -The logging functions are named after the level or severity of the events -they are used to track. The standard levels and their applicability are -described below (in increasing order of severity): - -.. tabularcolumns:: |l|L| - -+--------------+---------------------------------------------+ -| Level | When it's used | -+==============+=============================================+ -| ``DEBUG`` | Detailed information, typically of interest | -| | only when diagnosing problems. | -+--------------+---------------------------------------------+ -| ``INFO`` | Confirmation that things are working as | -| | expected. | -+--------------+---------------------------------------------+ -| ``WARNING`` | An indication that something unexpected | -| | happened, or indicative of some problem in | -| | the near future (e.g. 'disk space low'). | -| | The software is still working as expected. | -+--------------+---------------------------------------------+ -| ``ERROR`` | Due to a more serious problem, the software | -| | has not been able to perform some function. | -+--------------+---------------------------------------------+ -| ``CRITICAL`` | A serious error, indicating that the program| -| | itself may be unable to continue running. | -+--------------+---------------------------------------------+ - -The default level is ``WARNING``, which means that only events of this level -and above will be tracked, unless the logging package is configured to do -otherwise. - -Events that are tracked can be handled in different ways. The simplest way of -handling tracked events is to print them to the console. Another common way -is to write them to a disk file. - - -.. _howto-minimal-example: - -A simple example -^^^^^^^^^^^^^^^^ - -A very simple example is:: - - import logging - logging.warning('Watch out!') # will print a message to the console - logging.info('I told you so') # will not print anything - -If you type these lines into a script and run it, you'll see: - -.. code-block:: none - - WARNING:root:Watch out! - -printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - - -Logging to a file -^^^^^^^^^^^^^^^^^ - -A very common situation is that of recording logging events in a file, so let's -look at that next. Be sure to try the following in a newly-started Python -interpreter, and don't just continue from the session described above:: - - import logging - logging.basicConfig(filename='example.log',level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - -And now if we open the file and look at what we have, we should find the log -messages: - -.. code-block:: none - - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - -This example also shows how you can set the logging level which acts as the -threshold for tracking. In this case, because we set the threshold to -``DEBUG``, all of the messages were printed. - -If you want to set the logging level from a command-line option such as: - -.. code-block:: none - - --log=INFO - -and you have the value of the parameter passed for ``--log`` in some variable -*loglevel*, you can use:: - - getattr(logging, loglevel.upper()) - -to get the value which you'll pass to :func:`basicConfig` via the *level* -argument. You may want to error check any user input value, perhaps as in the -following example:: - - # assuming loglevel is bound to the string value obtained from the - # command line argument. Convert to upper case to allow the user to - # specify --log=DEBUG or --log=debug - numeric_level = getattr(logging, loglevel.upper(), None) - if not isinstance(numeric_level, int): - raise ValueError('Invalid log level: %s' % loglevel) - logging.basicConfig(level=numeric_level, ...) - -The call to :func:`basicConfig` should come *before* any calls to :func:`debug`, -:func:`info` etc. As it's intended as a one-off simple configuration facility, -only the first call will actually do anything: subsequent calls are effectively -no-ops. - -If you run the above script several times, the messages from successive runs -are appended to the file *example.log*. If you want each run to start afresh, -not remembering the messages from earlier runs, you can specify the *filemode* -argument, by changing the call in the above example to:: - - logging.basicConfig(filename='example.log', filemode='w', level=logging.DEBUG) - -The output will be the same as before, but the log file is no longer appended -to, so the messages from earlier runs are lost. - - -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - -Logging variable data -^^^^^^^^^^^^^^^^^^^^^ - -To log variable data, use a format string for the event description message and -append the variable data as arguments. For example:: - - import logging - logging.warning('%s before you %s', 'Look', 'leap!') - -will display: - -.. code-block:: none - - WARNING:root:Look before you leap! - -As you can see, merging of variable data into the event description message -uses the old, %-style of string formatting. This is for backwards -compatibility: the logging package pre-dates newer formatting options such as -:meth:`str.format` and :class:`string.Template`. These newer formatting -options *are* supported, but exploring them is outside the scope of this -tutorial: see :ref:`formatting-styles` for more information. - - -Changing the format of displayed messages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To change the format which is used to display messages, you need to -specify the format you want to use:: - - import logging - logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG) - logging.debug('This message should appear on the console') - logging.info('So should this') - logging.warning('And this, too') - -which would print: - -.. code-block:: none - - DEBUG:This message should appear on the console - INFO:So should this - WARNING:And this, too - -Notice that the 'root' which appeared in earlier examples has disappeared. For -a full set of things that can appear in format strings, you can refer to the -documentation for :ref:`logrecord-attributes`, but for simple usage, you just -need the *levelname* (severity), *message* (event description, including -variable data) and perhaps to display when the event occurred. This is -described in the next section. - - -Displaying the date/time in messages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To display the date and time of an event, you would place '%(asctime)s' in -your format string:: - - import logging - logging.basicConfig(format='%(asctime)s %(message)s') - logging.warning('is when this event was logged.') - -which should print something like this: - -.. code-block:: none - - 2010-12-12 11:41:42,612 is when this event was logged. - -The default format for date/time display (shown above) is like ISO8601 or -:rfc:`3339`. If you need more control over the formatting of the date/time, provide -a *datefmt* argument to ``basicConfig``, as in this example:: - - import logging - logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') - logging.warning('is when this event was logged.') - -which would display something like this: - -.. code-block:: none - - 12/12/2010 11:46:36 AM is when this event was logged. - -The format of the *datefmt* argument is the same as supported by -:func:`time.strftime`. - - -Next Steps -^^^^^^^^^^ - -That concludes the basic tutorial. It should be enough to get you up and -running with logging. There's a lot more that the logging package offers, but -to get the best out of it, you'll need to invest a little more of your time in -reading the following sections. If you're ready for that, grab some of your -favourite beverage and carry on. - -If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/group/comp.lang.python) and you -should receive help before too long. - -Still here? You can carry on reading the next few sections, which provide a -slightly more advanced/in-depth tutorial than the basic one above. After that, -you can take a look at the :ref:`logging-cookbook`. - -.. _logging-advanced-tutorial: - - -Advanced Logging Tutorial -------------------------- - -The logging library takes a modular approach and offers several categories -of components: loggers, handlers, filters, and formatters. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - -Log event information is passed between loggers, handlers, filters and -formatters in a :class:`LogRecord` instance. - -Logging is performed by calling methods on instances of the :class:`Logger` -class (hereafter called :dfn:`loggers`). Each instance has a name, and they are -conceptually arranged in a namespace hierarchy using dots (periods) as -separators. For example, a logger named 'scan' is the parent of loggers -'scan.text', 'scan.html' and 'scan.pdf'. Logger names can be anything you want, -and indicate the area of an application in which a logged message originates. - -A good convention to use when naming loggers is to use a module-level logger, -in each module which uses logging, named as follows:: - - logger = logging.getLogger(__name__) - -This means that logger names track the package/module hierarchy, and it's -intuitively obvious where events are logged just from the logger name. - -The root of the hierarchy of loggers is called the root logger. That's the -logger used by the functions :func:`debug`, :func:`info`, :func:`warning`, -:func:`error` and :func:`critical`, which just call the same-named method of -the root logger. The functions and the methods have the same signatures. The -root logger's name is printed as 'root' in the logged output. - -It is, of course, possible to log messages to different destinations. Support -is included in the package for writing log messages to files, HTTP GET/POST -locations, email via SMTP, generic sockets, queues, or OS-specific logging -mechanisms such as syslog or the Windows NT event log. Destinations are served -by :dfn:`handler` classes. You can create your own log destination class if -you have special requirements not met by any of the built-in handler classes. - -By default, no destination is set for any logging messages. You can specify -a destination (such as console or file) by using :func:`basicConfig` as in the -tutorial examples. If you call the functions :func:`debug`, :func:`info`, -:func:`warning`, :func:`error` and :func:`critical`, they will check to see -if no destination is set; and if one is not set, they will set a destination -of the console (``sys.stderr``) and a default format for the displayed -message before delegating to the root logger to do the actual message output. - -The default format set by :func:`basicConfig` for messages is: - -.. code-block:: none - - severity:logger name:message - -You can change this by passing a format string to :func:`basicConfig` with the -*format* keyword argument. For all options regarding how a format string is -constructed, see :ref:`formatter-objects`. - -Logging Flow -^^^^^^^^^^^^ - -The flow of log event information in loggers and handlers is illustrated in the -following diagram. - -.. image:: logging_flow.png - -Loggers -^^^^^^^ - -:class:`Logger` objects have a threefold job. First, they expose several -methods to application code so that applications can log messages at runtime. -Second, logger objects determine which log messages to act upon based upon -severity (the default filtering facility) or filter objects. Third, logger -objects pass along relevant log messages to all interested log handlers. - -The most widely used methods on logger objects fall into two categories: -configuration and message sending. - -These are the most common configuration methods: - -* :meth:`Logger.setLevel` specifies the lowest-severity log message a logger - will handle, where debug is the lowest built-in severity level and critical - is the highest built-in severity. For example, if the severity level is - INFO, the logger will handle only INFO, WARNING, ERROR, and CRITICAL messages - and will ignore DEBUG messages. - -* :meth:`Logger.addHandler` and :meth:`Logger.removeHandler` add and remove - handler objects from the logger object. Handlers are covered in more detail - in :ref:`handler-basic`. - -* :meth:`Logger.addFilter` and :meth:`Logger.removeFilter` add and remove filter - objects from the logger object. Filters are covered in more detail in - :ref:`filter`. - -You don't need to always call these methods on every logger you create. See the -last two paragraphs in this section. - -With the logger object configured, the following methods create log messages: - -* :meth:`Logger.debug`, :meth:`Logger.info`, :meth:`Logger.warning`, - :meth:`Logger.error`, and :meth:`Logger.critical` all create log records with - a message and a level that corresponds to their respective method names. The - message is actually a format string, which may contain the standard string - substitution syntax of ``%s``, ``%d``, ``%f``, and so on. The - rest of their arguments is a list of objects that correspond with the - substitution fields in the message. With regard to ``**kwargs``, the - logging methods care only about a keyword of ``exc_info`` and use it to - determine whether to log exception information. - -* :meth:`Logger.exception` creates a log message similar to - :meth:`Logger.error`. The difference is that :meth:`Logger.exception` dumps a - stack trace along with it. Call this method only from an exception handler. - -* :meth:`Logger.log` takes a log level as an explicit argument. This is a - little more verbose for logging messages than using the log level convenience - methods listed above, but this is how to log at custom log levels. - -:func:`getLogger` returns a reference to a logger instance with the specified -name if it is provided, or ``root`` if not. The names are period-separated -hierarchical structures. Multiple calls to :func:`getLogger` with the same name -will return a reference to the same logger object. Loggers that are further -down in the hierarchical list are children of loggers higher up in the list. -For example, given a logger with a name of ``foo``, loggers with names of -``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all descendants of ``foo``. - -Loggers have a concept of *effective level*. If a level is not explicitly set -on a logger, the level of its parent is used instead as its effective level. -If the parent has no explicit level set, *its* parent is examined, and so on - -all ancestors are searched until an explicitly set level is found. The root -logger always has an explicit level set (``WARNING`` by default). When deciding -whether to process an event, the effective level of the logger is used to -determine whether the event is passed to the logger's handlers. - -Child loggers propagate messages up to the handlers associated with their -ancestor loggers. Because of this, it is unnecessary to define and configure -handlers for all the loggers an application uses. It is sufficient to -configure handlers for a top-level logger and create child loggers as needed. -(You can, however, turn off propagation by setting the *propagate* -attribute of a logger to ``False``.) - - -.. _handler-basic: - -Handlers -^^^^^^^^ - -:class:`~logging.Handler` objects are responsible for dispatching the -appropriate log messages (based on the log messages' severity) to the handler's -specified destination. :class:`Logger` objects can add zero or more handler -objects to themselves with an :meth:`~Logger.addHandler` method. As an example -scenario, an application may want to send all log messages to a log file, all -log messages of error or higher to stdout, and all messages of critical to an -email address. This scenario requires three individual handlers where each -handler is responsible for sending messages of a specific severity to a specific -location. - -The standard library includes quite a few handler types (see -:ref:`useful-handlers`); the tutorials use mainly :class:`StreamHandler` and -:class:`FileHandler` in its examples. - -There are very few methods in a handler for application developers to concern -themselves with. The only handler methods that seem relevant for application -developers who are using the built-in handler objects (that is, not creating -custom handlers) are the following configuration methods: - -* The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the - lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger - determines which severity of messages it will pass to its handlers. The level - set in each handler determines which messages that handler will send on. - -* :meth:`~Handler.setFormatter` selects a Formatter object for this handler to - use. - -* :meth:`~Handler.addFilter` and :meth:`~Handler.removeFilter` respectively - configure and deconfigure filter objects on handlers. - -Application code should not directly instantiate and use instances of -:class:`Handler`. Instead, the :class:`Handler` class is a base class that -defines the interface that all handlers should have and establishes some -default behavior that child classes can use (or override). - - -Formatters -^^^^^^^^^^ - -Formatter objects configure the final order, structure, and contents of the log -message. Unlike the base :class:`logging.Handler` class, application code may -instantiate formatter classes, although you could likely subclass the formatter -if your application needs special behavior. The constructor takes three -optional arguments -- a message format string, a date format string and a style -indicator. - -.. method:: logging.Formatter.__init__(fmt=None, datefmt=None, style='%') - -If there is no message format string, the default is to use the -raw message. If there is no date format string, the default date format is: - -.. code-block:: none - - %Y-%m-%d %H:%M:%S - -with the milliseconds tacked on at the end. The ``style`` is one of `%`, '{' -or '$'. If one of these is not specified, then '%' will be used. - -If the ``style`` is '%', the message format string uses -``%(<dictionary key>)s`` styled string substitution; the possible keys are -documented in :ref:`logrecord-attributes`. If the style is '{', the message -format string is assumed to be compatible with :meth:`str.format` (using -keyword arguments), while if the style is '$' then the message format string -should conform to what is expected by :meth:`string.Template.substitute`. - -.. versionchanged:: 3.2 - Added the ``style`` parameter. - -The following message format string will log the time in a human-readable -format, the severity of the message, and the contents of the message, in that -order:: - - '%(asctime)s - %(levelname)s - %(message)s' - -Formatters use a user-configurable function to convert the creation time of a -record to a tuple. By default, :func:`time.localtime` is used; to change this -for a particular formatter instance, set the ``converter`` attribute of the -instance to a function with the same signature as :func:`time.localtime` or -:func:`time.gmtime`. To change it for all formatters, for example if you want -all logging times to be shown in GMT, set the ``converter`` attribute in the -Formatter class (to ``time.gmtime`` for GMT display). - - -Configuring Logging -^^^^^^^^^^^^^^^^^^^ - -.. currentmodule:: logging.config - -Programmers can configure logging in three ways: - -1. Creating loggers, handlers, and formatters explicitly using Python - code that calls the configuration methods listed above. -2. Creating a logging config file and reading it using the :func:`fileConfig` - function. -3. Creating a dictionary of configuration information and passing it - to the :func:`dictConfig` function. - -For the reference documentation on the last two options, see -:ref:`logging-config-api`. The following example configures a very simple -logger, a console handler, and a simple formatter using Python code:: - - import logging - - # create logger - logger = logging.getLogger('simple_example') - logger.setLevel(logging.DEBUG) - - # create console handler and set level to debug - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - # create formatter - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - # add formatter to ch - ch.setFormatter(formatter) - - # add ch to logger - logger.addHandler(ch) - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warn('warn message') - logger.error('error message') - logger.critical('critical message') - -Running this module from the command line produces the following output: - -.. code-block:: shell-session - - $ python simple_logging_module.py - 2005-03-19 15:10:26,618 - simple_example - DEBUG - debug message - 2005-03-19 15:10:26,620 - simple_example - INFO - info message - 2005-03-19 15:10:26,695 - simple_example - WARNING - warn message - 2005-03-19 15:10:26,697 - simple_example - ERROR - error message - 2005-03-19 15:10:26,773 - simple_example - CRITICAL - critical message - -The following Python module creates a logger, handler, and formatter nearly -identical to those in the example listed above, with the only difference being -the names of the objects:: - - import logging - import logging.config - - logging.config.fileConfig('logging.conf') - - # create logger - logger = logging.getLogger('simpleExample') - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warn('warn message') - logger.error('error message') - logger.critical('critical message') - -Here is the logging.conf file: - -.. code-block:: ini - - [loggers] - keys=root,simpleExample - - [handlers] - keys=consoleHandler - - [formatters] - keys=simpleFormatter - - [logger_root] - level=DEBUG - handlers=consoleHandler - - [logger_simpleExample] - level=DEBUG - handlers=consoleHandler - qualname=simpleExample - propagate=0 - - [handler_consoleHandler] - class=StreamHandler - level=DEBUG - formatter=simpleFormatter - args=(sys.stdout,) - - [formatter_simpleFormatter] - format=%(asctime)s - %(name)s - %(levelname)s - %(message)s - datefmt= - -The output is nearly identical to that of the non-config-file-based example: - -.. code-block:: shell-session - - $ python simple_logging_config.py - 2005-03-19 15:38:55,977 - simpleExample - DEBUG - debug message - 2005-03-19 15:38:55,979 - simpleExample - INFO - info message - 2005-03-19 15:38:56,054 - simpleExample - WARNING - warn message - 2005-03-19 15:38:56,055 - simpleExample - ERROR - error message - 2005-03-19 15:38:56,130 - simpleExample - CRITICAL - critical message - -You can see that the config file approach has a few advantages over the Python -code approach, mainly separation of configuration and code and the ability of -noncoders to easily modify the logging properties. - -.. warning:: The :func:`fileConfig` function takes a default parameter, - ``disable_existing_loggers``, which defaults to ``True`` for reasons of - backward compatibility. This may or may not be what you want, since it - will cause any loggers existing before the :func:`fileConfig` call to - be disabled unless they (or an ancestor) are explicitly named in the - configuration. Please refer to the reference documentation for more - information, and specify ``False`` for this parameter if you wish. - - The dictionary passed to :func:`dictConfig` can also specify a Boolean - value with key ``disable_existing_loggers``, which if not specified - explicitly in the dictionary also defaults to being interpreted as - ``True``. This leads to the logger-disabling behaviour described above, - which may not be what you want - in which case, provide the key - explicitly with a value of ``False``. - - -.. currentmodule:: logging - -Note that the class names referenced in config files need to be either relative -to the logging module, or absolute values which can be resolved using normal -import mechanisms. Thus, you could use either -:class:`~logging.handlers.WatchedFileHandler` (relative to the logging module) or -``mypackage.mymodule.MyHandler`` (for a class defined in package ``mypackage`` -and module ``mymodule``, where ``mypackage`` is available on the Python import -path). - -In Python 3.2, a new means of configuring logging has been introduced, using -dictionaries to hold configuration information. This provides a superset of the -functionality of the config-file-based approach outlined above, and is the -recommended configuration method for new applications and deployments. Because -a Python dictionary is used to hold configuration information, and since you -can populate that dictionary using different means, you have more options for -configuration. For example, you can use a configuration file in JSON format, -or, if you have access to YAML processing functionality, a file in YAML -format, to populate the configuration dictionary. Or, of course, you can -construct the dictionary in Python code, receive it in pickled form over a -socket, or use whatever approach makes sense for your application. - -Here's an example of the same configuration as above, in YAML format for -the new dictionary-based approach: - -.. code-block:: yaml - - version: 1 - formatters: - simple: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - handlers: - console: - class: logging.StreamHandler - level: DEBUG - formatter: simple - stream: ext://sys.stdout - loggers: - simpleExample: - level: DEBUG - handlers: [console] - propagate: no - root: - level: DEBUG - handlers: [console] - -For more information about logging using a dictionary, see -:ref:`logging-config-api`. - -What happens if no configuration is provided -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If no logging configuration is provided, it is possible to have a situation -where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. - -For versions of Python prior to 3.2, the behaviour is as follows: - -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. - -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. - -In Python 3.2 and later, the behaviour is as follows: - -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. - -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. - -.. _library-config: - -Configuring Logging for a Library -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When developing a library which uses logging, you should take care to -document how the library uses logging - for example, the names of loggers -used. Some consideration also needs to be given to its logging configuration. -If the using application does not use logging, and library code makes logging -calls, then (as described in the previous section) events of severity -``WARNING`` and greater will be printed to ``sys.stderr``. This is regarded as -the best default behaviour. - -If for some reason you *don't* want these messages printed in the absence of -any logging configuration, you can attach a do-nothing handler to the top-level -logger for your library. This avoids the message being printed, since a handler -will be always be found for the library's events: it just doesn't produce any -output. If the library user configures logging for application use, presumably -that configuration will add some handlers, and if levels are suitably -configured then logging calls made in library code will send output to those -handlers, as normal. - -A do-nothing handler is included in the logging package: -:class:`~logging.NullHandler` (since Python 3.1). An instance of this handler -could be added to the top-level logger of the logging namespace used by the -library (*if* you want to prevent your library's logged events being output to -``sys.stderr`` in the absence of logging configuration). If all logging by a -library *foo* is done using loggers with names matching 'foo.x', 'foo.x.y', -etc. then the code:: - - import logging - logging.getLogger('foo').addHandler(logging.NullHandler()) - -should have the desired effect. If an organisation produces a number of -libraries, then the logger name specified can be 'orgname.foo' rather than -just 'foo'. - -.. note:: It is strongly advised that you *do not add any handlers other - than* :class:`~logging.NullHandler` *to your library's loggers*. This is - because the configuration of handlers is the prerogative of the application - developer who uses your library. The application developer knows their - target audience and what handlers are most appropriate for their - application: if you add handlers 'under the hood', you might well interfere - with their ability to carry out unit tests and deliver logs which suit their - requirements. - - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - -Levels can also be associated with loggers, being set either by the developer or -through loading a saved logging configuration. When a logging method is called -on a logger, the logger compares its own level with the level associated with -the method call. If the logger's level is higher than the method call's, no -logging message is actually generated. This is the basic mechanism controlling -the verbosity of logging output. - -Logging messages are encoded as instances of the :class:`~logging.LogRecord` -class. When a logger decides to actually log an event, a -:class:`~logging.LogRecord` instance is created from the logging message. - -Logging messages are subjected to a dispatch mechanism through the use of -:dfn:`handlers`, which are instances of subclasses of the :class:`Handler` -class. Handlers are responsible for ensuring that a logged message (in the form -of a :class:`LogRecord`) ends up in a particular location (or set of locations) -which is useful for the target audience for that message (such as end users, -support desk staff, system administrators, developers). Handlers are passed -:class:`LogRecord` instances intended for particular destinations. Each logger -can have zero, one or more handlers associated with it (via the -:meth:`~Logger.addHandler` method of :class:`Logger`). In addition to any -handlers directly associated with a logger, *all handlers associated with all -ancestors of the logger* are called to dispatch the message (unless the -*propagate* flag for a logger is set to a false value, at which point the -passing to ancestor handlers stops). - -Just as for loggers, handlers can have levels associated with them. A handler's -level acts as a filter in the same way as a logger's level does. If a handler -decides to actually dispatch an event, the :meth:`~Handler.emit` method is used -to send the message to its destination. Most user-defined subclasses of -:class:`Handler` will need to override this :meth:`~Handler.emit`. - -.. _custom-levels: - -Custom Levels -^^^^^^^^^^^^^ - -Defining your own levels is possible, but should not be necessary, as the -existing levels have been chosen on the basis of practical experience. -However, if you are convinced that you need custom levels, great care should -be exercised when doing this, and it is possibly *a very bad idea to define -custom levels if you are developing a library*. That's because if multiple -library authors all define their own custom levels, there is a chance that -the logging output from such multiple libraries used together will be -difficult for the using developer to control and/or interpret, because a -given numeric value might mean different things for different libraries. - -.. _useful-handlers: - -Useful Handlers ---------------- - -In addition to the base :class:`Handler` class, many useful subclasses are -provided: - -#. :class:`StreamHandler` instances send messages to streams (file-like - objects). - -#. :class:`FileHandler` instances send messages to disk files. - -#. :class:`~handlers.BaseRotatingHandler` is the base class for handlers that - rotate log files at a certain point. It is not meant to be instantiated - directly. Instead, use :class:`~handlers.RotatingFileHandler` or - :class:`~handlers.TimedRotatingFileHandler`. - -#. :class:`~handlers.RotatingFileHandler` instances send messages to disk - files, with support for maximum log file sizes and log file rotation. - -#. :class:`~handlers.TimedRotatingFileHandler` instances send messages to - disk files, rotating the log file at certain timed intervals. - -#. :class:`~handlers.SocketHandler` instances send messages to TCP/IP - sockets. Since 3.4, Unix domain sockets are also supported. - -#. :class:`~handlers.DatagramHandler` instances send messages to UDP - sockets. Since 3.4, Unix domain sockets are also supported. - -#. :class:`~handlers.SMTPHandler` instances send messages to a designated - email address. - -#. :class:`~handlers.SysLogHandler` instances send messages to a Unix - syslog daemon, possibly on a remote machine. - -#. :class:`~handlers.NTEventLogHandler` instances send messages to a - Windows NT/2000/XP event log. - -#. :class:`~handlers.MemoryHandler` instances send messages to a buffer - in memory, which is flushed whenever specific criteria are met. - -#. :class:`~handlers.HTTPHandler` instances send messages to an HTTP - server using either ``GET`` or ``POST`` semantics. - -#. :class:`~handlers.WatchedFileHandler` instances watch the file they are - logging to. If the file changes, it is closed and reopened using the file - name. This handler is only useful on Unix-like systems; Windows does not - support the underlying mechanism used. - -#. :class:`~handlers.QueueHandler` instances send messages to a queue, such as - those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. - -#. :class:`NullHandler` instances do nothing with error messages. They are used - by library developers who want to use logging, but want to avoid the 'No - handlers could be found for logger XXX' message which can be displayed if - the library user has not configured logging. See :ref:`library-config` for - more information. - -.. versionadded:: 3.1 - The :class:`NullHandler` class. - -.. versionadded:: 3.2 - The :class:`~handlers.QueueHandler` class. - -The :class:`NullHandler`, :class:`StreamHandler` and :class:`FileHandler` -classes are defined in the core logging package. The other handlers are -defined in a sub-module, :mod:`logging.handlers`. (There is also another -sub-module, :mod:`logging.config`, for configuration functionality.) - -Logged messages are formatted for presentation through instances of the -:class:`Formatter` class. They are initialized with a format string suitable for -use with the % operator and a dictionary. - -For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format -string (which is applied to each message in the batch), there is provision for -header and trailer format strings. - -When filtering based on logger level and/or handler level is not enough, -instances of :class:`Filter` can be added to both :class:`Logger` and -:class:`Handler` instances (through their :meth:`~Handler.addFilter` method). -Before deciding to process a message further, both loggers and handlers consult -all their filters for permission. If any filter returns a false value, the -message is not processed further. - -The basic :class:`Filter` functionality allows filtering by specific logger -name. If this feature is used, messages sent to the named logger and its -children are allowed through the filter, and all others dropped. - - -.. _logging-exceptions: - -Exceptions raised during logging --------------------------------- - -The logging package is designed to swallow exceptions which occur while logging -in production. This is so that errors which occur while handling logging events -- such as logging misconfiguration, network or other similar errors - do not -cause the application using logging to terminate prematurely. - -:class:`SystemExit` and :class:`KeyboardInterrupt` exceptions are never -swallowed. Other exceptions which occur during the :meth:`~Handler.emit` method -of a :class:`Handler` subclass are passed to its :meth:`~Handler.handleError` -method. - -The default implementation of :meth:`~Handler.handleError` in :class:`Handler` -checks to see if a module-level variable, :data:`raiseExceptions`, is set. If -set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is -swallowed. - -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is - because during development, you typically want to be notified of any - exceptions that occur. It's advised that you set :data:`raiseExceptions` to - ``False`` for production usage. - -.. currentmodule:: logging - -.. _arbitrary-object-messages: - -Using arbitrary objects as messages ------------------------------------ - -In the preceding sections and examples, it has been assumed that the message -passed when logging the event is a string. However, this is not the only -possibility. You can pass an arbitrary object as a message, and its -:meth:`~object.__str__` method will be called when the logging system needs to -convert it to a string representation. In fact, if you want to, you can avoid -computing a string representation altogether - for example, the -:class:`~handlers.SocketHandler` emits an event by pickling it and sending it -over the wire. - - -Optimization ------------- - -Formatting of message arguments is deferred until it cannot be avoided. -However, computing the arguments passed to the logging method can also be -expensive, and you may want to avoid doing it if the logger will just throw -away your event. To decide what to do, you can call the -:meth:`~Logger.isEnabledFor` method which takes a level argument and returns -true if the event would be created by the Logger for that level of call. -You can write code like this:: - - if logger.isEnabledFor(logging.DEBUG): - logger.debug('Message with %s, %s', expensive_func1(), - expensive_func2()) - -so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. - -.. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more - expensive than you'd like (e.g. for deeply nested loggers where an explicit - level is only set high up in the logger hierarchy). In such cases (or if you - want to avoid calling a method in tight loops), you can cache the result of a - call to :meth:`~Logger.isEnabledFor` in a local or instance variable, and use - that instead of calling the method each time. Such a cached value would only - need to be recomputed when the logging configuration changes dynamically - while the application is running (which is not all that common). - -There are other optimizations which can be made for specific applications which -need more precise control over what logging information is collected. Here's a -list of things you can do to avoid processing during logging which you don't -need: - -+-----------------------------------------------+----------------------------------------+ -| What you don't want to collect | How to avoid collecting it | -+===============================================+========================================+ -| Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | -| | This avoids calling | -| | :func:`sys._getframe`, which may help | -| | to speed up your code in environments | -| | like PyPy (which can't speed up code | -| | that uses :func:`sys._getframe`), if | -| | and when PyPy supports Python 3.x. | -+-----------------------------------------------+----------------------------------------+ -| Threading information. | Set ``logging.logThreads`` to ``0``. | -+-----------------------------------------------+----------------------------------------+ -| Process information. | Set ``logging.logProcesses`` to ``0``. | -+-----------------------------------------------+----------------------------------------+ - -Also note that the core logging module only includes the basic handlers. If -you don't import :mod:`logging.handlers` and :mod:`logging.config`, they won't -take up any memory. - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :ref:`A logging cookbook <logging-cookbook>` diff --git a/third_party/python/Doc/howto/logging_flow.png b/third_party/python/Doc/howto/logging_flow.png deleted file mode 100755 index a88382309..000000000 Binary files a/third_party/python/Doc/howto/logging_flow.png and /dev/null differ diff --git a/third_party/python/Doc/howto/pyporting.rst b/third_party/python/Doc/howto/pyporting.rst deleted file mode 100644 index 88b01779d..000000000 --- a/third_party/python/Doc/howto/pyporting.rst +++ /dev/null @@ -1,452 +0,0 @@ -.. _pyporting-howto: - -********************************* -Porting Python 2 Code to Python 3 -********************************* - -:author: Brett Cannon - -.. topic:: Abstract - - With Python 3 being the future of Python while Python 2 is still in active - use, it is good to have your project available for both major releases of - Python. This guide is meant to help you figure out how best to support both - Python 2 & 3 simultaneously. - - If you are looking to port an extension module instead of pure Python code, - please see :ref:`cporting-howto`. - - If you would like to read one core Python developer's take on why Python 3 - came into existence, you can read Nick Coghlan's `Python 3 Q & A`_ or - Brett Cannon's `Why Python 3 exists`_. - - For help with porting, you can email the python-porting_ mailing list with - questions. - -The Short Explanation -===================== - -To make your project be single-source Python 2/3 compatible, the basic steps -are: - -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``pip install coverage``) -#. Learn the differences between Python 2 & 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``pip install future``) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 & 3 (tox_ can help test - against multiple versions of Python; ``pip install tox``) -#. Consider using optional static type checking to make sure your type usage - works in both Python 2 & 3 (e.g. use mypy_ to check your typing under both - Python 2 & Python 3). - - -Details -======= - -A key point about supporting Python 2 & 3 simultaneously is that you can start -**today**! Even if your dependencies are not supporting Python 3 yet that does -not mean you can't modernize your code **now** to support Python 3. Most changes -required to support Python 3 lead to cleaner code using newer practices even in -Python 2 code. - -Another key point is that modernizing your Python 2 code to also support -Python 3 is largely automated for you. While you might have to make some API -decisions thanks to Python 3 clarifying text data versus binary data, the -lower-level work is now mostly done for you and thus can at least benefit from -the automated changes immediately. - -Keep those key points in mind while you read on about the details of porting -your code to support Python 2 & 3 simultaneously. - - -Drop support for Python 2.6 and older -------------------------------------- - -While you can make Python 2.5 work with Python 3, it is **much** easier if you -only have to work with Python 2.7. If dropping Python 2.5 is not an -option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``pip install six``). Do realize, though, that nearly all the projects listed -in this HOWTO will not be available to you. - -If you are able to skip Python 2.5 and older, then the required changes -to your code should continue to look and feel like idiomatic Python code. At -worst you will have to use a function instead of a method in some instances or -have to import a function instead of using a built-in one, but otherwise the -overall transformation should not feel foreign to you. - -But you should aim for only supporting Python 2.7. Python 2.6 is no longer -freely supported and thus is not receiving bugfixes. This means **you** will have -to work around any issues you come across with Python 2.6. There are also some -tools mentioned in this HOWTO which do not support Python 2.6 (e.g., Pylint_), -and this will become more commonplace as time goes on. It will simply be easier -for you if you only support the versions of Python that you have to support. - - -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- - -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. - - -Have good test coverage ------------------------ - -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you find it hard to get better than 90% -coverage). If you don't already have a tool to measure test coverage then -coverage.py_ is recommended. - - -Learn the differences between Python 2 & 3 -------------------------------------------- - -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the `"What's New"`_ doc for each release of Python 3 and the -`Porting to Python 3`_ book (which is free online). There is also a handy -`cheat sheet`_ from the Python-Future project. - - -Update your code ----------------- - -Once you feel like you know what is different in Python 3 compared to Python 2, -it's time to update your code! You have a choice between two tools in porting -your code automatically: Futurize_ and Modernize_. Which tool you choose will -depend on how much like Python 3 you want your code to be. Futurize_ does its -best to make Python 3 idioms and practices exist in Python 2, e.g. backporting -the ``bytes`` type from Python 3 so that you have semantic parity between the -major versions of Python. Modernize_, -on the other hand, is more conservative and targets a Python 2/3 subset of -Python, directly relying on six_ to help provide compatibility. As Python 3 is -the future, it might be best to consider Futurize to begin adjusting to any new -practices that Python 3 introduces which you are not accustomed to yet. - -Regardless of which tool you choose, they will update your code to run under -Python 3 while staying compatible with the version of Python 2 you started with. -Depending on how conservative you want to be, you may want to run the tool over -your test suite first and visually inspect the diff to make sure the -transformation is accurate. After you have transformed your test suite and -verified that all the tests still pass as expected, then you can transform your -application code knowing that any tests which fail is a translation failure. - -Unfortunately the tools can't automate everything to make your code work under -Python 3 and so there are a handful of things you will need to update manually -to get full Python 3 support (which of these steps are necessary vary between -the tools). Read the documentation for the tool you choose to use to see what it -fixes by default and what it can do optionally to know what will (not) be fixed -for you and what you may have to fix on your own (e.g. using ``io.open()`` over -the built-in ``open()`` function is off by default in Modernize). Luckily, -though, there are only a couple of things to watch out for which can be -considered large issues that may be hard to debug if not watched for. - - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2``; all division between ``int`` values -result in a ``float``. This change has actually been planned since Python 2.2 -which was released in 2002. Since then users have been encouraged to add -``from __future__ import division`` to any and all files which use the ``/`` and -``//`` operators or to be running the interpreter with the ``-Q`` flag. If you -have not been doing this then you will need to go through your code and do two -things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). - - -Text versus binary data -+++++++++++++++++++++++ - -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. - -To make the distinction between text and binary data clearer and more -pronounced, Python 3 did what most languages created in the age of the internet -have done and made text and binary data distinct types that cannot blindly be -mixed together (Python predates widespread access to the internet). For any code -that deals only with text or only binary data, this separation doesn't pose an -issue. But for code that has to deal with both, it does mean you might have to -now care about when you are using text compared to binary data, which is why -this cannot be entirely automated. - -To start, you will need to decide which APIs take text and which take binary -(it is **highly** recommended you don't design APIs that can take both due to -the difficulty of keeping the code working; as stated earlier it is difficult to -do well). In Python 2 this means making sure the APIs that take text can work -with ``unicode`` and those that work with binary data work with the -``bytes`` type from Python 3 (which is a subset of ``str`` in Python 2 and acts -as an alias for ``bytes`` type in Python 2). Usually the biggest issue is -realizing which methods exist on which types in Python 2 & 3 simultaneously -(for text that's ``unicode`` in Python 2 and ``str`` in Python 3, for binary -that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following -table lists the **unique** methods of each data type across Python 2 & 3 -(e.g., the ``decode()`` method is usable on the equivalent binary data type in -either Python 2 or 3, but it can't be used by the textual data type consistently -between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do -note that as of Python 3.5 the ``__mod__`` method was added to the bytes type. - -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== - -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. - -The next issue is making sure you know whether the string literals in your code -represent text or binary data. You should add a ``b`` prefix to any -literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (there is a :mod:`__future__` import to force all unspecified -literals to be Unicode, but usage has shown it isn't as effective as adding a -``b`` or ``u`` prefix to all literals explicitly) - -As part of this dichotomy you also need to be careful about opening files. -Unless you have been working on Windows, there is a chance you have not always -bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for -binary reading). Under Python 3, binary files and text files are clearly -distinct and mutually incompatible; see the :mod:`io` module for details. -Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing binary data to be read and/or written) or textual access -(allowing text data to be read and/or written). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). Do not bother with the -outdated practice of using :func:`codecs.open` as that's only necessary for -keeping compatibility with Python 2.5. - -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 & 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. - -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. - -To summarize: - -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, textual literals with a ``u`` - prefix -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing into binary data - - -Use feature detection instead of version detection -++++++++++++++++++++++++++++++++++++++++++++++++++ - -Inevitably you will have code that has to choose what to do based on what -version of Python is running. The best way to do this is with feature detection -of whether the version of Python you're running under supports what you need. -If for some reason that doesn't work then you should make the version check be -against Python 2 and not Python 3. To help explain this, let's look at an -example. - -Let's pretend that you need access to a feature of importlib_ that -is available in Python's standard library since Python 3.3 and available for -Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the ``importlib.abc`` module by doing the following:: - - import sys - - if sys.version_info[0] == 3: - from importlib import abc - else: - from importlib2 import abc - -The problem with this code is what happens when Python 4 comes out? It would -be better to treat Python 2 as the exceptional case instead of Python 3 and -assume that future Python versions will be more compatible with Python 3 than -Python 2:: - - import sys - - if sys.version_info[0] > 2: - from importlib import abc - else: - from importlib2 import abc - -The best solution, though, is to do no version detection at all and instead rely -on feature detection. That avoids any potential issues of getting the version -detection wrong and helps keep you future-compatible:: - - try: - from importlib import abc - except ImportError: - from importlib2 import abc - - -Prevent compatibility regressions ---------------------------------- - -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. - -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. - -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. - - -Check which dependencies block your transition ----------------------------------------------- - -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com. - -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. - - -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- - -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell anyone using your code that you -support Python 2 **and** 3. Ideally you will also want to add classifiers for -each major/minor version of Python you now support. - - -Use continuous integration to stay compatible ---------------------------------------------- - -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 & 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. - -You may also want to use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. - -And that's mostly it! At this point your code base is compatible with both -Python 2 and 3 simultaneously. Your testing will also be set up so that you -don't accidentally break Python 2 or 3 compatibility regardless of which version -you typically run your tests under while developing. - - -Consider using optional static type checking --------------------------------------------- - -Another way to help port your code is to use a static type checker like -mypy_ or pytype_ on your code. These tools can be used to analyze your code as -if it's being run under Python 2, then you can run the tool a second time as if -your code is running under Python 3. By running a static type checker twice like -this you can discover if you're e.g. misusing binary data type in one version -of Python compared to another. If you add optional type hints to your code you -can also explicitly state whether your APIs use textual or binary data, helping -to make sure everything functions as expected in both versions of Python. - - -.. _2to3: https://docs.python.org/3/library/2to3.html -.. _caniusepython3: https://pypi.org/project/caniusepython3 -.. _cheat sheet: http://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.org/project/coverage -.. _Futurize: http://python-future.org/automatic_conversion.html -.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib -.. _importlib2: https://pypi.org/project/importlib2 -.. _Modernize: https://python-modernize.readthedocs.org/en/latest/ -.. _mypy: http://mypy-lang.org/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.org/project/pylint - -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.org/en/latest/python3/questions_and_answers.html - -.. _pytype: https://github.com/google/pytype -.. _python-future: http://python-future.org/ -.. _python-porting: https://mail.python.org/mailman/listinfo/python-porting -.. _six: https://pypi.org/project/six -.. _tox: https://pypi.org/project/tox -.. _trove classifier: https://pypi.org/classifiers - -.. _"What's New": https://docs.python.org/3/whatsnew/index.html - -.. _Why Python 3 exists: http://www.snarky.ca/why-python-3-exists diff --git a/third_party/python/Doc/howto/regex.rst b/third_party/python/Doc/howto/regex.rst deleted file mode 100644 index 65b094593..000000000 --- a/third_party/python/Doc/howto/regex.rst +++ /dev/null @@ -1,1385 +0,0 @@ -.. _regex-howto: - -**************************** - Regular Expression HOWTO -**************************** - -:Author: A.M. Kuchling <amk@amk.ca> - -.. TODO: - Document lookbehind assertions - Better way of displaying a RE, a string, and what it matches - Mention optional argument to match.groups() - Unicode (at least a reference) - - -.. topic:: Abstract - - This document is an introductory tutorial to using regular expressions in Python - with the :mod:`re` module. It provides a gentler introduction than the - corresponding section in the Library Reference. - - -Introduction -============ - -Regular expressions (called REs, or regexes, or regex patterns) are essentially -a tiny, highly specialized programming language embedded inside Python and made -available through the :mod:`re` module. Using this little language, you specify -the rules for the set of possible strings that you want to match; this set might -contain English sentences, or e-mail addresses, or TeX commands, or anything you -like. You can then ask questions such as "Does this string match the pattern?", -or "Is there a match for the pattern anywhere in this string?". You can also -use REs to modify a string or to split it apart in various ways. - -Regular expression patterns are compiled into a series of bytecodes which are -then executed by a matching engine written in C. For advanced use, it may be -necessary to pay careful attention to how the engine will execute a given RE, -and write the RE in a certain way in order to produce bytecode that runs faster. -Optimization isn't covered in this document, because it requires that you have a -good understanding of the matching engine's internals. - -The regular expression language is relatively small and restricted, so not all -possible string processing tasks can be done using regular expressions. There -are also tasks that *can* be done with regular expressions, but the expressions -turn out to be very complicated. In these cases, you may be better off writing -Python code to do the processing; while Python code will be slower than an -elaborate regular expression, it will also probably be more understandable. - - -Simple Patterns -=============== - -We'll start by learning about the simplest possible regular expressions. Since -regular expressions are used to operate on strings, we'll begin with the most -common task: matching characters. - -For a detailed explanation of the computer science underlying regular -expressions (deterministic and non-deterministic finite automata), you can refer -to almost any textbook on writing compilers. - - -Matching Characters -------------------- - -Most letters and characters will simply match themselves. For example, the -regular expression ``test`` will match the string ``test`` exactly. (You can -enable a case-insensitive mode that would let this RE match ``Test`` or ``TEST`` -as well; more about this later.) - -There are exceptions to this rule; some characters are special -:dfn:`metacharacters`, and don't match themselves. Instead, they signal that -some out-of-the-ordinary thing should be matched, or they affect other portions -of the RE by repeating them or changing their meaning. Much of this document is -devoted to discussing various metacharacters and what they do. - -Here's a complete list of the metacharacters; their meanings will be discussed -in the rest of this HOWTO. - -.. code-block:: none - - . ^ $ * + ? { } [ ] \ | ( ) - -The first metacharacters we'll look at are ``[`` and ``]``. They're used for -specifying a character class, which is a set of characters that you wish to -match. Characters can be listed individually, or a range of characters can be -indicated by giving two characters and separating them by a ``'-'``. For -example, ``[abc]`` will match any of the characters ``a``, ``b``, or ``c``; this -is the same as ``[a-c]``, which uses a range to express the same set of -characters. If you wanted to match only lowercase letters, your RE would be -``[a-z]``. - -Metacharacters are not active inside classes. For example, ``[akm$]`` will -match any of the characters ``'a'``, ``'k'``, ``'m'``, or ``'$'``; ``'$'`` is -usually a metacharacter, but inside a character class it's stripped of its -special nature. - -You can match the characters not listed within the class by :dfn:`complementing` -the set. This is indicated by including a ``'^'`` as the first character of the -class; ``'^'`` outside a character class will simply match the ``'^'`` -character. For example, ``[^5]`` will match any character except ``'5'``. - -Perhaps the most important metacharacter is the backslash, ``\``. As in Python -string literals, the backslash can be followed by various characters to signal -various special sequences. It's also used to escape all the metacharacters so -you can still match them in patterns; for example, if you need to match a ``[`` -or ``\``, you can precede them with a backslash to remove their special -meaning: ``\[`` or ``\\``. - -Some of the special sequences beginning with ``'\'`` represent -predefined sets of characters that are often useful, such as the set -of digits, the set of letters, or the set of anything that isn't -whitespace. - -Let's take an example: ``\w`` matches any alphanumeric character. If -the regex pattern is expressed in bytes, this is equivalent to the -class ``[a-zA-Z0-9_]``. If the regex pattern is a string, ``\w`` will -match all the characters marked as letters in the Unicode database -provided by the :mod:`unicodedata` module. You can use the more -restricted definition of ``\w`` in a string pattern by supplying the -:const:`re.ASCII` flag when compiling the regular expression. - -The following list of special sequences isn't complete. For a complete -list of sequences and expanded class definitions for Unicode string -patterns, see the last part of :ref:`Regular Expression Syntax -<re-syntax>` in the Standard Library reference. In general, the -Unicode versions match any character that's in the appropriate -category in the Unicode database. - -``\d`` - Matches any decimal digit; this is equivalent to the class ``[0-9]``. - -``\D`` - Matches any non-digit character; this is equivalent to the class ``[^0-9]``. - -``\s`` - Matches any whitespace character; this is equivalent to the class ``[ - \t\n\r\f\v]``. - -``\S`` - Matches any non-whitespace character; this is equivalent to the class ``[^ - \t\n\r\f\v]``. - -``\w`` - Matches any alphanumeric character; this is equivalent to the class - ``[a-zA-Z0-9_]``. - -``\W`` - Matches any non-alphanumeric character; this is equivalent to the class - ``[^a-zA-Z0-9_]``. - -These sequences can be included inside a character class. For example, -``[\s,.]`` is a character class that will match any whitespace character, or -``','`` or ``'.'``. - -The final metacharacter in this section is ``.``. It matches anything except a -newline character, and there's an alternate mode (:const:`re.DOTALL`) where it will -match even a newline. ``.`` is often used where you want to match "any -character". - - -Repeating Things ----------------- - -Being able to match varying sets of characters is the first thing regular -expressions can do that isn't already possible with the methods available on -strings. However, if that was the only additional capability of regexes, they -wouldn't be much of an advance. Another capability is that you can specify that -portions of the RE must be repeated a certain number of times. - -The first metacharacter for repeating things that we'll look at is ``*``. ``*`` -doesn't match the literal character ``'*'``; instead, it specifies that the -previous character can be matched zero or more times, instead of exactly once. - -For example, ``ca*t`` will match ``'ct'`` (0 ``'a'`` characters), ``'cat'`` (1 ``'a'``), -``'caaat'`` (3 ``'a'`` characters), and so forth. - -Repetitions such as ``*`` are :dfn:`greedy`; when repeating a RE, the matching -engine will try to repeat it as many times as possible. If later portions of the -pattern don't match, the matching engine will then back up and try again with -fewer repetitions. - -A step-by-step example will make this more obvious. Let's consider the -expression ``a[bcd]*b``. This matches the letter ``'a'``, zero or more letters -from the class ``[bcd]``, and finally ends with a ``'b'``. Now imagine matching -this RE against the string ``'abcbd'``. - -+------+-----------+---------------------------------+ -| Step | Matched | Explanation | -+======+===========+=================================+ -| 1 | ``a`` | The ``a`` in the RE matches. | -+------+-----------+---------------------------------+ -| 2 | ``abcbd`` | The engine matches ``[bcd]*``, | -| | | going as far as it can, which | -| | | is to the end of the string. | -+------+-----------+---------------------------------+ -| 3 | *Failure* | The engine tries to match | -| | | ``b``, but the current position | -| | | is at the end of the string, so | -| | | it fails. | -+------+-----------+---------------------------------+ -| 4 | ``abcb`` | Back up, so that ``[bcd]*`` | -| | | matches one less character. | -+------+-----------+---------------------------------+ -| 5 | *Failure* | Try ``b`` again, but the | -| | | current position is at the last | -| | | character, which is a ``'d'``. | -+------+-----------+---------------------------------+ -| 6 | ``abc`` | Back up again, so that | -| | | ``[bcd]*`` is only matching | -| | | ``bc``. | -+------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | -| | | the character at the | -| | | current position is ``'b'``, so | -| | | it succeeds. | -+------+-----------+---------------------------------+ - -The end of the RE has now been reached, and it has matched ``'abcb'``. This -demonstrates how the matching engine goes as far as it can at first, and if no -match is found it will then progressively back up and retry the rest of the RE -again and again. It will back up until it has tried zero matches for -``[bcd]*``, and if that subsequently fails, the engine will conclude that the -string doesn't match the RE at all. - -Another repeating metacharacter is ``+``, which matches one or more times. Pay -careful attention to the difference between ``*`` and ``+``; ``*`` matches -*zero* or more times, so whatever's being repeated may not be present at all, -while ``+`` requires at least *one* occurrence. To use a similar example, -``ca+t`` will match ``'cat'`` (1 ``'a'``), ``'caaat'`` (3 ``'a'``\ s), but won't -match ``'ct'``. - -There are two more repeating qualifiers. The question mark character, ``?``, -matches either once or zero times; you can think of it as marking something as -being optional. For example, ``home-?brew`` matches either ``'homebrew'`` or -``'home-brew'``. - -The most complicated repeated qualifier is ``{m,n}``, where *m* and *n* are -decimal integers. This qualifier means there must be at least *m* repetitions, -and at most *n*. For example, ``a/{1,3}b`` will match ``'a/b'``, ``'a//b'``, and -``'a///b'``. It won't match ``'ab'``, which has no slashes, or ``'a////b'``, which -has four. - -You can omit either *m* or *n*; in that case, a reasonable value is assumed for -the missing value. Omitting *m* is interpreted as a lower limit of 0, while -omitting *n* results in an upper bound of infinity. - -Readers of a reductionist bent may notice that the three other qualifiers can -all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` -is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use -``*``, ``+``, or ``?`` when you can, simply because they're shorter and easier -to read. - - -Using Regular Expressions -========================= - -Now that we've looked at some simple regular expressions, how do we actually use -them in Python? The :mod:`re` module provides an interface to the regular -expression engine, allowing you to compile REs into objects and then perform -matches with them. - - -Compiling Regular Expressions ------------------------------ - -Regular expressions are compiled into pattern objects, which have -methods for various operations such as searching for pattern matches or -performing string substitutions. :: - - >>> import re - >>> p = re.compile('ab*') - >>> p - re.compile('ab*') - -:func:`re.compile` also accepts an optional *flags* argument, used to enable -various special features and syntax variations. We'll go over the available -settings later, but for now a single example will do:: - - >>> p = re.compile('ab*', re.IGNORECASE) - -The RE is passed to :func:`re.compile` as a string. REs are handled as strings -because regular expressions aren't part of the core Python language, and no -special syntax was created for expressing them. (There are applications that -don't need REs at all, so there's no need to bloat the language specification by -including them.) Instead, the :mod:`re` module is simply a C extension module -included with Python, just like the :mod:`socket` or :mod:`zlib` modules. - -Putting REs in strings keeps the Python language simpler, but has one -disadvantage which is the topic of the next section. - - -.. _the-backslash-plague: - -The Backslash Plague --------------------- - -As stated earlier, regular expressions use the backslash character (``'\'``) to -indicate special forms or to allow special characters to be used without -invoking their special meaning. This conflicts with Python's usage of the same -character for the same purpose in string literals. - -Let's say you want to write a RE that matches the string ``\section``, which -might be found in a LaTeX file. To figure out what to write in the program -code, start with the desired string to be matched. Next, you must escape any -backslashes and other metacharacters by preceding them with a backslash, -resulting in the string ``\\section``. The resulting string that must be passed -to :func:`re.compile` must be ``\\section``. However, to express this as a -Python string literal, both backslashes must be escaped *again*. - -+-------------------+------------------------------------------+ -| Characters | Stage | -+===================+==========================================+ -| ``\section`` | Text string to be matched | -+-------------------+------------------------------------------+ -| ``\\section`` | Escaped backslash for :func:`re.compile` | -+-------------------+------------------------------------------+ -| ``"\\\\section"`` | Escaped backslashes for a string literal | -+-------------------+------------------------------------------+ - -In short, to match a literal backslash, one has to write ``'\\\\'`` as the RE -string, because the regular expression must be ``\\``, and each backslash must -be expressed as ``\\`` inside a regular Python string literal. In REs that -feature backslashes repeatedly, this leads to lots of repeated backslashes and -makes the resulting strings difficult to understand. - -The solution is to use Python's raw string notation for regular expressions; -backslashes are not handled in any special way in a string literal prefixed with -``'r'``, so ``r"\n"`` is a two-character string containing ``'\'`` and ``'n'``, -while ``"\n"`` is a one-character string containing a newline. Regular -expressions will often be written in Python code using this raw string notation. - -In addition, special escape sequences that are valid in regular expressions, -but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, -which means the sequences will be invalid if raw string notation or escaping -the backslashes isn't used. - - -+-------------------+------------------+ -| Regular String | Raw string | -+===================+==================+ -| ``"ab*"`` | ``r"ab*"`` | -+-------------------+------------------+ -| ``"\\\\section"`` | ``r"\\section"`` | -+-------------------+------------------+ -| ``"\\w+\\s+\\1"`` | ``r"\w+\s+\1"`` | -+-------------------+------------------+ - - -Performing Matches ------------------- - -Once you have an object representing a compiled regular expression, what do you -do with it? Pattern objects have several methods and attributes. -Only the most significant ones will be covered here; consult the :mod:`re` docs -for a complete listing. - -+------------------+-----------------------------------------------+ -| Method/Attribute | Purpose | -+==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ -| ``search()`` | Scan through a string, looking for any | -| | location where this RE matches. | -+------------------+-----------------------------------------------+ -| ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | -+------------------+-----------------------------------------------+ -| ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | -+------------------+-----------------------------------------------+ - -:meth:`~re.pattern.match` and :meth:`~re.pattern.search` return ``None`` if no match can be found. If -they're successful, a :ref:`match object <match-objects>` instance is returned, -containing information about the match: where it starts and ends, the substring -it matched, and more. - -You can learn about this by interactively experimenting with the :mod:`re` -module. If you have :mod:`tkinter` available, you may also want to look at -:source:`Tools/demo/redemo.py`, a demonstration program included with the -Python distribution. It allows you to enter REs and strings, and displays -whether the RE matches or fails. :file:`redemo.py` can be quite useful when -trying to debug a complicated RE. - -This HOWTO uses the standard Python interpreter for its examples. First, run the -Python interpreter, import the :mod:`re` module, and compile a RE:: - - >>> import re - >>> p = re.compile('[a-z]+') - >>> p - re.compile('[a-z]+') - -Now, you can try matching various strings against the RE ``[a-z]+``. An empty -string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.pattern.match` should return ``None`` in this case, which will cause the -interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: - - >>> p.match("") - >>> print(p.match("")) - None - -Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.pattern.match` will return a :ref:`match object <match-objects>`, so you -should store the result in a variable for later use. :: - - >>> m = p.match('tempo') - >>> m - <_sre.SRE_Match object; span=(0, 5), match='tempo'> - -Now you can query the :ref:`match object <match-objects>` for information -about the matching string. Match object instances -also have several methods and attributes; the most important ones are: - -+------------------+--------------------------------------------+ -| Method/Attribute | Purpose | -+==================+============================================+ -| ``group()`` | Return the string matched by the RE | -+------------------+--------------------------------------------+ -| ``start()`` | Return the starting position of the match | -+------------------+--------------------------------------------+ -| ``end()`` | Return the ending position of the match | -+------------------+--------------------------------------------+ -| ``span()`` | Return a tuple containing the (start, end) | -| | positions of the match | -+------------------+--------------------------------------------+ - -Trying these methods will soon clarify their meaning:: - - >>> m.group() - 'tempo' - >>> m.start(), m.end() - (0, 5) - >>> m.span() - (0, 5) - -:meth:`~re.match.group` returns the substring that was matched by the RE. :meth:`~re.match.start` -and :meth:`~re.match.end` return the starting and ending index of the match. :meth:`~re.match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: - - >>> print(p.match('::: message')) - None - >>> m = p.search('::: message'); print(m) - <_sre.SRE_Match object; span=(4, 11), match='message'> - >>> m.group() - 'message' - >>> m.span() - (4, 11) - -In actual programs, the most common style is to store the -:ref:`match object <match-objects>` in a variable, and then check if it was -``None``. This usually looks like:: - - p = re.compile( ... ) - m = p.match( 'string goes here' ) - if m: - print('Match found: ', m.group()) - else: - print('No match') - -Two pattern methods return all of the matches for a pattern. -:meth:`~re.pattern.findall` returns a list of matching strings:: - - >>> p = re.compile(r'\d+') - >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') - ['12', '11', '10'] - -The ``r`` prefix, making the literal a raw string literal, is needed in this -example because escape sequences in a normal "cooked" string literal that are -not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See -:ref:`the-backslash-plague`. - -:meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the -result. The :meth:`~re.Pattern.finditer` method returns a sequence of -:ref:`match object <match-objects>` instances as an :term:`iterator`:: - - >>> iterator = p.finditer('12 drummers drumming, 11 ... 10 ...') - >>> iterator #doctest: +ELLIPSIS - <callable_iterator object at 0x...> - >>> for match in iterator: - ... print(match.span()) - ... - (0, 2) - (22, 24) - (29, 31) - - -Module-Level Functions ----------------------- - -You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions -take the same arguments as the corresponding pattern method with -the RE string added as the first argument, and still return either ``None`` or a -:ref:`match object <match-objects>` instance. :: - - >>> print(re.match(r'From\s+', 'Fromage amk')) - None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 5), match='From '> - -Under the hood, these functions simply create a pattern object for you -and call the appropriate method on it. They also store the compiled -object in a cache, so future calls using the same RE won't need to -parse the pattern again and again. - -Should you use these module-level functions, or should you get the -pattern and call its methods yourself? If you're accessing a regex -within a loop, pre-compiling it will save a few function calls. -Outside of loops, there's not much difference thanks to the internal -cache. - - -Compilation Flags ------------------ - -Compilation flags let you modify some aspects of how regular expressions work. -Flags are available in the :mod:`re` module under two names, a long name such as -:const:`IGNORECASE` and a short, one-letter form such as :const:`I`. (If you're -familiar with Perl's pattern modifiers, the one-letter forms use the same -letters; the short form of :const:`re.VERBOSE` is :const:`re.X`, for example.) -Multiple flags can be specified by bitwise OR-ing them; ``re.I | re.M`` sets -both the :const:`I` and :const:`M` flags, for example. - -Here's a table of the available flags, followed by a more detailed explanation -of each one. - -+---------------------------------+--------------------------------------------+ -| Flag | Meaning | -+=================================+============================================+ -| :const:`ASCII`, :const:`A` | Makes several escapes like ``\w``, ``\b``, | -| | ``\s`` and ``\d`` match only on ASCII | -| | characters with the respective property. | -+---------------------------------+--------------------------------------------+ -| :const:`DOTALL`, :const:`S` | Make ``.`` match any character, including | -| | newlines. | -+---------------------------------+--------------------------------------------+ -| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches. | -+---------------------------------+--------------------------------------------+ -| :const:`LOCALE`, :const:`L` | Do a locale-aware match. | -+---------------------------------+--------------------------------------------+ -| :const:`MULTILINE`, :const:`M` | Multi-line matching, affecting ``^`` and | -| | ``$``. | -+---------------------------------+--------------------------------------------+ -| :const:`VERBOSE`, :const:`X` | Enable verbose REs, which can be organized | -| (for 'extended') | more cleanly and understandably. | -+---------------------------------+--------------------------------------------+ - - -.. data:: I - IGNORECASE - :noindex: - - Perform case-insensitive matching; character class and literal strings will - match letters by ignoring case. For example, ``[A-Z]`` will match lowercase - letters, too. Full Unicode matching also works unless the :const:`ASCII` - flag is used to disable non-ASCII matches. When the Unicode patterns - ``[a-z]`` or ``[A-Z]`` are used in combination with the :const:`IGNORECASE` - flag, they will match the 52 ASCII letters and 4 additional non-ASCII - letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131, - Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and - 'K' (U+212A, Kelvin sign). ``Spam`` will match ``'Spam'``, ``'spam'``, - ``'spAM'``, or ``'ſpam'`` (the latter is matched only in Unicode mode). - This lowercasing doesn't take the current locale into account; - it will if you also set the :const:`LOCALE` flag. - - -.. data:: L - LOCALE - :noindex: - - Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching dependent - on the current locale instead of the Unicode database. - - Locales are a feature of the C library intended to help in writing programs - that take account of language differences. For example, if you're - processing encoded French text, you'd want to be able to write ``\w+`` to - match words, but ``\w`` only matches the character class ``[A-Za-z]`` in - bytes patterns; it won't match bytes corresponding to ``é`` or ``ç``. - If your system is configured properly and a French locale is selected, - certain C functions will tell the program that the byte corresponding to - ``é`` should also be considered a letter. - Setting the :const:`LOCALE` flag when compiling a regular expression will cause - the resulting compiled object to use these C functions for ``\w``; this is - slower, but also enables ``\w+`` to match French words as you'd expect. - The use of this flag is discouraged in Python 3 as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. - - -.. data:: M - MULTILINE - :noindex: - - (``^`` and ``$`` haven't been explained yet; they'll be introduced in section - :ref:`more-metacharacters`.) - - Usually ``^`` matches only at the beginning of the string, and ``$`` matches - only at the end of the string and immediately before the newline (if any) at the - end of the string. When this flag is specified, ``^`` matches at the beginning - of the string and at the beginning of each line within the string, immediately - following each newline. Similarly, the ``$`` metacharacter matches either at - the end of the string and at the end of each line (immediately preceding each - newline). - - -.. data:: S - DOTALL - :noindex: - - Makes the ``'.'`` special character match any character at all, including a - newline; without this flag, ``'.'`` will match anything *except* a newline. - - -.. data:: A - ASCII - :noindex: - - Make ``\w``, ``\W``, ``\b``, ``\B``, ``\s`` and ``\S`` perform ASCII-only - matching instead of full Unicode matching. This is only meaningful for - Unicode patterns, and is ignored for byte patterns. - - -.. data:: X - VERBOSE - :noindex: - - This flag allows you to write regular expressions that are more readable by - granting you more flexibility in how you can format them. When this flag has - been specified, whitespace within the RE string is ignored, except when the - whitespace is in a character class or preceded by an unescaped backslash; this - lets you organize and indent the RE more clearly. This flag also lets you put - comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped - backslash. - - For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it - is to read? :: - - charref = re.compile(r""" - &[#] # Start of a numeric entity reference - ( - 0[0-7]+ # Octal form - | [0-9]+ # Decimal form - | x[0-9a-fA-F]+ # Hexadecimal form - ) - ; # Trailing semicolon - """, re.VERBOSE) - - Without the verbose setting, the RE would look like this:: - - charref = re.compile("&#(0[0-7]+" - "|[0-9]+" - "|x[0-9a-fA-F]+);") - - In the above example, Python's automatic concatenation of string literals has - been used to break up the RE into smaller pieces, but it's still more difficult - to understand than the version using :const:`re.VERBOSE`. - - -More Pattern Power -================== - -So far we've only covered a part of the features of regular expressions. In -this section, we'll cover some new metacharacters, and how to use groups to -retrieve portions of the text that was matched. - - -.. _more-metacharacters: - -More Metacharacters -------------------- - -There are some metacharacters that we haven't covered yet. Most of them will be -covered in this section. - -Some of the remaining metacharacters to be discussed are :dfn:`zero-width -assertions`. They don't cause the engine to advance through the string; -instead, they consume no characters at all, and simply succeed or fail. For -example, ``\b`` is an assertion that the current position is located at a word -boundary; the position isn't changed by the ``\b`` at all. This means that -zero-width assertions should never be repeated, because if they match once at a -given location, they can obviously be matched an infinite number of times. - -``|`` - Alternation, or the "or" operator. If *A* and *B* are regular expressions, - ``A|B`` will match any string that matches either *A* or *B*. ``|`` has very - low precedence in order to make it work reasonably when you're alternating - multi-character strings. ``Crow|Servo`` will match either ``'Crow'`` or ``'Servo'``, - not ``'Cro'``, a ``'w'`` or an ``'S'``, and ``'ervo'``. - - To match a literal ``'|'``, use ``\|``, or enclose it inside a character class, - as in ``[|]``. - -``^`` - Matches at the beginning of lines. Unless the :const:`MULTILINE` flag has been - set, this will only match at the beginning of the string. In :const:`MULTILINE` - mode, this also matches immediately after each newline within the string. - - For example, if you wish to match the word ``From`` only at the beginning of a - line, the RE to use is ``^From``. :: - - >>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 4), match='From'> - >>> print(re.search('^From', 'Reciting From Memory')) - None - - To match a literal ``'^'``, use ``\^``. - -``$`` - Matches at the end of a line, which is defined as either the end of the string, - or any location followed by a newline character. :: - - >>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(6, 7), match='}'> - >>> print(re.search('}$', '{block} ')) - None - >>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(6, 7), match='}'> - - To match a literal ``'$'``, use ``\$`` or enclose it inside a character class, - as in ``[$]``. - -``\A`` - Matches only at the start of the string. When not in :const:`MULTILINE` mode, - ``\A`` and ``^`` are effectively the same. In :const:`MULTILINE` mode, they're - different: ``\A`` still matches only at the beginning of the string, but ``^`` - may match at any location inside the string that follows a newline character. - -``\Z`` - Matches only at the end of the string. - -``\b`` - Word boundary. This is a zero-width assertion that matches only at the - beginning or end of a word. A word is defined as a sequence of alphanumeric - characters, so the end of a word is indicated by whitespace or a - non-alphanumeric character. - - The following example matches ``class`` only when it's a complete word; it won't - match when it's contained inside another word. :: - - >>> p = re.compile(r'\bclass\b') - >>> print(p.search('no class at all')) - <_sre.SRE_Match object; span=(3, 8), match='class'> - >>> print(p.search('the declassified algorithm')) - None - >>> print(p.search('one subclass is')) - None - - There are two subtleties you should remember when using this special sequence. - First, this is the worst collision between Python's string literals and regular - expression sequences. In Python's string literals, ``\b`` is the backspace - character, ASCII value 8. If you're not using raw strings, then Python will - convert the ``\b`` to a backspace, and your RE won't match as you expect it to. - The following example looks the same as our previous RE, but omits the ``'r'`` - in front of the RE string. :: - - >>> p = re.compile('\bclass\b') - >>> print(p.search('no class at all')) - None - >>> print(p.search('\b' + 'class' + '\b')) - <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'> - - Second, inside a character class, where there's no use for this assertion, - ``\b`` represents the backspace character, for compatibility with Python's - string literals. - -``\B`` - Another zero-width assertion, this is the opposite of ``\b``, only matching when - the current position is not at a word boundary. - - -Grouping --------- - -Frequently you need to obtain more information than just whether the RE matched -or not. Regular expressions are often used to dissect strings by writing a RE -divided into several subgroups which match different components of interest. -For example, an RFC-822 header line is divided into a header name and a value, -separated by a ``':'``, like this: - -.. code-block:: none - - From: author@example.com - User-Agent: Thunderbird 1.5.0.9 (X11/20061227) - MIME-Version: 1.0 - To: editor@example.com - -This can be handled by writing a regular expression which matches an entire -header line, and has one group which matches the header name, and another group -which matches the header's value. - -Groups are marked by the ``'('``, ``')'`` metacharacters. ``'('`` and ``')'`` -have much the same meaning as they do in mathematical expressions; they group -together the expressions contained inside them, and you can repeat the contents -of a group with a repeating qualifier, such as ``*``, ``+``, ``?``, or -``{m,n}``. For example, ``(ab)*`` will match zero or more repetitions of -``ab``. :: - - >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) - (0, 10) - -Groups indicated with ``'('``, ``')'`` also capture the starting and ending -index of the text that they match; this can be retrieved by passing an argument -to :meth:`~re.match.group`, :meth:`~re.match.start`, :meth:`~re.match.end`, and -:meth:`~re.match.span`. Groups are -numbered starting with 0. Group 0 is always present; it's the whole RE, so -:ref:`match object <match-objects>` methods all have group 0 as their default -argument. Later we'll see how to express groups that don't capture the span -of text that they match. :: - - >>> p = re.compile('(a)b') - >>> m = p.match('ab') - >>> m.group() - 'ab' - >>> m.group(0) - 'ab' - -Subgroups are numbered from left to right, from 1 upward. Groups can be nested; -to determine the number, just count the opening parenthesis characters, going -from left to right. :: - - >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') - >>> m.group(0) - 'abcd' - >>> m.group(1) - 'abc' - >>> m.group(2) - 'b' - -:meth:`~re.match.group` can be passed multiple group numbers at a time, in which case it -will return a tuple containing the corresponding values for those groups. :: - - >>> m.group(2,1,2) - ('b', 'abc', 'b') - -The :meth:`~re.match.groups` method returns a tuple containing the strings for all the -subgroups, from 1 up to however many there are. :: - - >>> m.groups() - ('abc', 'b') - -Backreferences in a pattern allow you to specify that the contents of an earlier -capturing group must also be found at the current location in the string. For -example, ``\1`` will succeed if the exact contents of group 1 can be found at -the current position, and fails otherwise. Remember that Python's string -literals also use a backslash followed by numbers to allow including arbitrary -characters in a string, so be sure to use a raw string when incorporating -backreferences in a RE. - -For example, the following RE detects doubled words in a string. :: - - >>> p = re.compile(r'\b(\w+)\s+\1\b') - >>> p.search('Paris in the the spring').group() - 'the the' - -Backreferences like this aren't often useful for just searching through a string ---- there are few text formats which repeat data in this way --- but you'll soon -find out that they're *very* useful when performing string substitutions. - - -Non-capturing and Named Groups ------------------------------- - -Elaborate REs may use many groups, both to capture substrings of interest, and -to group and structure the RE itself. In complex REs, it becomes difficult to -keep track of the group numbers. There are two features which help with this -problem. Both of them use a common syntax for regular expression extensions, so -we'll look at that first. - -Perl 5 is well known for its powerful additions to standard regular expressions. -For these new features the Perl developers couldn't choose new single-keystroke metacharacters -or new special sequences beginning with ``\`` without making Perl's regular -expressions confusingly different from standard REs. If they chose ``&`` as a -new metacharacter, for example, old expressions would be assuming that ``&`` was -a regular character and wouldn't have escaped it by writing ``\&`` or ``[&]``. - -The solution chosen by the Perl developers was to use ``(?...)`` as the -extension syntax. ``?`` immediately after a parenthesis was a syntax error -because the ``?`` would have nothing to repeat, so this didn't introduce any -compatibility problems. The characters immediately after the ``?`` indicate -what extension is being used, so ``(?=foo)`` is one thing (a positive lookahead -assertion) and ``(?:foo)`` is something else (a non-capturing group containing -the subexpression ``foo``). - -Python supports several of Perl's extensions and adds an extension -syntax to Perl's extension syntax. If the first character after the -question mark is a ``P``, you know that it's an extension that's -specific to Python. - -Now that we've looked at the general extension syntax, we can return -to the features that simplify working with groups in complex REs. - -Sometimes you'll want to use a group to denote a part of a regular expression, -but aren't interested in retrieving the group's contents. You can make this fact -explicit by using a non-capturing group: ``(?:...)``, where you can replace the -``...`` with any other regular expression. :: - - >>> m = re.match("([abc])+", "abc") - >>> m.groups() - ('c',) - >>> m = re.match("(?:[abc])+", "abc") - >>> m.groups() - () - -Except for the fact that you can't retrieve the contents of what the group -matched, a non-capturing group behaves exactly the same as a capturing group; -you can put anything inside it, repeat it with a repetition metacharacter such -as ``*``, and nest it within other groups (capturing or non-capturing). -``(?:...)`` is particularly useful when modifying an existing pattern, since you -can add new groups without changing how all the other groups are numbered. It -should be mentioned that there's no performance difference in searching between -capturing and non-capturing groups; neither form is any faster than the other. - -A more significant feature is named groups: instead of referring to them by -numbers, groups can be referenced by a name. - -The syntax for a named group is one of the Python-specific extensions: -``(?P<name>...)``. *name* is, obviously, the name of the group. Named groups -behave exactly like capturing groups, and additionally associate a name -with a group. The :ref:`match object <match-objects>` methods that deal with -capturing groups all accept either integers that refer to the group by number -or strings that contain the desired group's name. Named groups are still -given numbers, so you can retrieve information about a group in two ways:: - - >>> p = re.compile(r'(?P<word>\b\w+\b)') - >>> m = p.search( '(((( Lots of punctuation )))' ) - >>> m.group('word') - 'Lots' - >>> m.group(1) - 'Lots' - -Named groups are handy because they let you use easily-remembered names, instead -of having to remember numbers. Here's an example RE from the :mod:`imaplib` -module:: - - InternalDate = re.compile(r'INTERNALDATE "' - r'(?P<day>[ 123][0-9])-(?P<mon>[A-Z][a-z][a-z])-' - r'(?P<year>[0-9][0-9][0-9][0-9])' - r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])' - r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])' - r'"') - -It's obviously much easier to retrieve ``m.group('zonem')``, instead of having -to remember to retrieve group 9. - -The syntax for backreferences in an expression such as ``(...)\1`` refers to the -number of the group. There's naturally a variant that uses the group name -instead of the number. This is another Python extension: ``(?P=name)`` indicates -that the contents of the group called *name* should again be matched at the -current point. The regular expression for finding doubled words, -``\b(\w+)\s+\1\b`` can also be written as ``\b(?P<word>\w+)\s+(?P=word)\b``:: - - >>> p = re.compile(r'\b(?P<word>\w+)\s+(?P=word)\b') - >>> p.search('Paris in the the spring').group() - 'the the' - - -Lookahead Assertions --------------------- - -Another zero-width assertion is the lookahead assertion. Lookahead assertions -are available in both positive and negative form, and look like this: - -``(?=...)`` - Positive lookahead assertion. This succeeds if the contained regular - expression, represented here by ``...``, successfully matches at the current - location, and fails otherwise. But, once the contained expression has been - tried, the matching engine doesn't advance at all; the rest of the pattern is - tried right where the assertion started. - -``(?!...)`` - Negative lookahead assertion. This is the opposite of the positive assertion; - it succeeds if the contained expression *doesn't* match at the current position - in the string. - -To make this concrete, let's look at a case where a lookahead is useful. -Consider a simple pattern to match a filename and split it apart into a base -name and an extension, separated by a ``.``. For example, in ``news.rc``, -``news`` is the base name, and ``rc`` is the filename's extension. - -The pattern to match this is quite simple: - -``.*[.].*$`` - -Notice that the ``.`` needs to be treated specially because it's a -metacharacter, so it's inside a character class to only match that -specific character. Also notice the trailing ``$``; this is added to -ensure that all the rest of the string must be included in the -extension. This regular expression matches ``foo.bar`` and -``autoexec.bat`` and ``sendmail.cf`` and ``printers.conf``. - -Now, consider complicating the problem a bit; what if you want to match -filenames where the extension is not ``bat``? Some incorrect attempts: - -``.*[.][^b].*$`` The first attempt above tries to exclude ``bat`` by requiring -that the first character of the extension is not a ``b``. This is wrong, -because the pattern also doesn't match ``foo.bar``. - -``.*[.]([^b]..|.[^a].|..[^t])$`` - -The expression gets messier when you try to patch up the first solution by -requiring one of the following cases to match: the first character of the -extension isn't ``b``; the second character isn't ``a``; or the third character -isn't ``t``. This accepts ``foo.bar`` and rejects ``autoexec.bat``, but it -requires a three-letter extension and won't accept a filename with a two-letter -extension such as ``sendmail.cf``. We'll complicate the pattern again in an -effort to fix it. - -``.*[.]([^b].?.?|.[^a]?.?|..?[^t]?)$`` - -In the third attempt, the second and third letters are all made optional in -order to allow matching extensions shorter than three characters, such as -``sendmail.cf``. - -The pattern's getting really complicated now, which makes it hard to read and -understand. Worse, if the problem changes and you want to exclude both ``bat`` -and ``exe`` as extensions, the pattern would get even more complicated and -confusing. - -A negative lookahead cuts through all this confusion: - -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` -doesn't match at this point, try the rest of the pattern; if ``bat$`` does -match, the whole pattern will fail. The trailing ``$`` is required to ensure -that something like ``sample.batch``, where the extension only starts with -``bat``, will be allowed. The ``[^.]*`` makes sure that the pattern works -when there are multiple dots in the filename. - -Excluding another filename extension is now easy; simply add it as an -alternative inside the assertion. The following pattern excludes filenames that -end in either ``bat`` or ``exe``: - -``.*[.](?!bat$|exe$)[^.]*$`` - - -Modifying Strings -================= - -Up to this point, we've simply performed searches against a static string. -Regular expressions are also commonly used to modify strings in various ways, -using the following pattern methods: - -+------------------+-----------------------------------------------+ -| Method/Attribute | Purpose | -+==================+===============================================+ -| ``split()`` | Split the string into a list, splitting it | -| | wherever the RE matches | -+------------------+-----------------------------------------------+ -| ``sub()`` | Find all substrings where the RE matches, and | -| | replace them with a different string | -+------------------+-----------------------------------------------+ -| ``subn()`` | Does the same thing as :meth:`!sub`, but | -| | returns the new string and the number of | -| | replacements | -+------------------+-----------------------------------------------+ - - -Splitting Strings ------------------ - -The :meth:`~re.pattern.split` method of a pattern splits a string apart -wherever the RE matches, returning a list of the pieces. It's similar to the -:meth:`~str.split` method of strings but provides much more generality in the -delimiters that you can split by; string :meth:`!split` only supports splitting by -whitespace or by a fixed string. As you'd expect, there's a module-level -:func:`re.split` function, too. - - -.. method:: .split(string [, maxsplit=0]) - :noindex: - - Split *string* by the matches of the regular expression. If capturing - parentheses are used in the RE, then their contents will also be returned as - part of the resulting list. If *maxsplit* is nonzero, at most *maxsplit* splits - are performed. - -You can limit the number of splits made, by passing a value for *maxsplit*. -When *maxsplit* is nonzero, at most *maxsplit* splits will be made, and the -remainder of the string is returned as the final element of the list. In the -following example, the delimiter is any sequence of non-alphanumeric characters. -:: - - >>> p = re.compile(r'\W+') - >>> p.split('This is a test, short and sweet, of split().') - ['This', 'is', 'a', 'test', 'short', 'and', 'sweet', 'of', 'split', ''] - >>> p.split('This is a test, short and sweet, of split().', 3) - ['This', 'is', 'a', 'test, short and sweet, of split().'] - -Sometimes you're not only interested in what the text between delimiters is, but -also need to know what the delimiter was. If capturing parentheses are used in -the RE, then their values are also returned as part of the list. Compare the -following calls:: - - >>> p = re.compile(r'\W+') - >>> p2 = re.compile(r'(\W+)') - >>> p.split('This... is a test.') - ['This', 'is', 'a', 'test', ''] - >>> p2.split('This... is a test.') - ['This', '... ', 'is', ' ', 'a', ' ', 'test', '.', ''] - -The module-level function :func:`re.split` adds the RE to be used as the first -argument, but is otherwise the same. :: - - >>> re.split(r'[\W]+', 'Words, words, words.') - ['Words', 'words', 'words', ''] - >>> re.split(r'([\W]+)', 'Words, words, words.') - ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'[\W]+', 'Words, words, words.', 1) - ['Words', 'words, words.'] - - -Search and Replace ------------------- - -Another common task is to find all the matches for a pattern, and replace them -with a different string. The :meth:`~re.pattern.sub` method takes a replacement value, -which can be either a string or a function, and the string to be processed. - -.. method:: .sub(replacement, string[, count=0]) - :noindex: - - Returns the string obtained by replacing the leftmost non-overlapping - occurrences of the RE in *string* by the replacement *replacement*. If the - pattern isn't found, *string* is returned unchanged. - - The optional argument *count* is the maximum number of pattern occurrences to be - replaced; *count* must be a non-negative integer. The default value of 0 means - to replace all occurrences. - -Here's a simple example of using the :meth:`~re.pattern.sub` method. It replaces colour -names with the word ``colour``:: - - >>> p = re.compile('(blue|white|red)') - >>> p.sub('colour', 'blue socks and red shoes') - 'colour socks and colour shoes' - >>> p.sub('colour', 'blue socks and red shoes', count=1) - 'colour socks and red shoes' - -The :meth:`~re.pattern.subn` method does the same work, but returns a 2-tuple containing the -new string value and the number of replacements that were performed:: - - >>> p = re.compile('(blue|white|red)') - >>> p.subn('colour', 'blue socks and red shoes') - ('colour socks and colour shoes', 2) - >>> p.subn('colour', 'no colours at all') - ('no colours at all', 0) - -Empty matches are replaced only when they're not adjacent to a previous match. -:: - - >>> p = re.compile('x*') - >>> p.sub('-', 'abxd') - '-a-b-d-' - -If *replacement* is a string, any backslash escapes in it are processed. That -is, ``\n`` is converted to a single newline character, ``\r`` is converted to a -carriage return, and so forth. Unknown escapes such as ``\&`` are left alone. -Backreferences, such as ``\6``, are replaced with the substring matched by the -corresponding group in the RE. This lets you incorporate portions of the -original text in the resulting replacement string. - -This example matches the word ``section`` followed by a string enclosed in -``{``, ``}``, and changes ``section`` to ``subsection``:: - - >>> p = re.compile('section{ ( [^}]* ) }', re.VERBOSE) - >>> p.sub(r'subsection{\1}','section{First} section{second}') - 'subsection{First} subsection{second}' - -There's also a syntax for referring to named groups as defined by the -``(?P<name>...)`` syntax. ``\g<name>`` will use the substring matched by the -group named ``name``, and ``\g<number>`` uses the corresponding group number. -``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous in a -replacement string such as ``\g<2>0``. (``\20`` would be interpreted as a -reference to group 20, not a reference to group 2 followed by the literal -character ``'0'``.) The following substitutions are all equivalent, but use all -three variations of the replacement string. :: - - >>> p = re.compile('section{ (?P<name> [^}]* ) }', re.VERBOSE) - >>> p.sub(r'subsection{\1}','section{First}') - 'subsection{First}' - >>> p.sub(r'subsection{\g<1>}','section{First}') - 'subsection{First}' - >>> p.sub(r'subsection{\g<name>}','section{First}') - 'subsection{First}' - -*replacement* can also be a function, which gives you even more control. If -*replacement* is a function, the function is called for every non-overlapping -occurrence of *pattern*. On each call, the function is passed a -:ref:`match object <match-objects>` argument for the match and can use this -information to compute the desired replacement string and return it. - -In the following example, the replacement function translates decimals into -hexadecimal:: - - >>> def hexrepl(match): - ... "Return the hex string for a decimal number" - ... value = int(match.group()) - ... return hex(value) - ... - >>> p = re.compile(r'\d+') - >>> p.sub(hexrepl, 'Call 65490 for printing, 49152 for user code.') - 'Call 0xffd2 for printing, 0xc000 for user code.' - -When using the module-level :func:`re.sub` function, the pattern is passed as -the first argument. The pattern may be provided as an object or as a string; if -you need to specify regular expression flags, you must either use a -pattern object as the first parameter, or use embedded modifiers in the -pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. - - -Common Problems -=============== - -Regular expressions are a powerful tool for some applications, but in some ways -their behaviour isn't intuitive and at times they don't behave the way you may -expect them to. This section will point out some of the most common pitfalls. - - -Use String Methods ------------------- - -Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed -string, or a single character class, and you're not using any :mod:`re` features -such as the :const:`~re.IGNORECASE` flag, then the full power of regular expressions -may not be required. Strings have several methods for performing operations with -fixed strings and they're usually much faster, because the implementation is a -single small C loop that's been optimized for the purpose, instead of the large, -more generalized regular expression engine. - -One example might be replacing a single fixed string with another one; for -example, you might replace ``word`` with ``deed``. :func:`re.sub` seems like the -function to use for this, but consider the :meth:`~str.replace` method. Note that -:meth:`!replace` will also replace ``word`` inside words, turning ``swordfish`` -into ``sdeedfish``, but the naive RE ``word`` would have done that, too. (To -avoid performing the substitution on parts of words, the pattern would have to -be ``\bword\b``, in order to require that ``word`` have a word boundary on -either side. This takes the job beyond :meth:`!replace`'s abilities.) - -Another common task is deleting every occurrence of a single character from a -string or replacing it with another single character. You might do this with -something like ``re.sub('\n', ' ', S)``, but :meth:`~str.translate` is capable of -doing both tasks and will be faster than any regular expression operation can -be. - -In short, before turning to the :mod:`re` module, consider whether your problem -can be solved with a faster and simpler string method. - - -match() versus search() ------------------------ - -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: - - >>> print(re.match('super', 'superstition').span()) - (0, 5) - >>> print(re.match('super', 'insuperable')) - None - -On the other hand, :func:`~re.search` will scan forward through the string, -reporting the first match it finds. :: - - >>> print(re.search('super', 'superstition').span()) - (0, 5) - >>> print(re.search('super', 'insuperable').span()) - (2, 7) - -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. - - -Greedy versus Non-Greedy ------------------------- - -When repeating a regular expression, as in ``a*``, the resulting action is to -consume as much of the pattern as possible. This fact often bites you when -you're trying to match a pair of balanced delimiters, such as the angle brackets -surrounding an HTML tag. The naive pattern for matching a single HTML tag -doesn't work because of the greedy nature of ``.*``. :: - - >>> s = '<html><head><title>Title' - >>> len(s) - 32 - >>> print(re.match('<.*>', s).span()) - (0, 32) - >>> print(re.match('<.*>', s).group()) - Title - -The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of -the string. There's still more left in the RE, though, and the ``>`` can't -match at the end of the string, so the regular expression engine has to -backtrack character by character until it finds a match for the ``>``. The -final match extends from the ``'<'`` in ``''`` to the ``'>'`` in -``''``, which isn't what you want. - -In this case, the solution is to use the non-greedy qualifiers ``*?``, ``+?``, -``??``, or ``{m,n}?``, which match as *little* text as possible. In the above -example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and -when it fails, the engine advances a character at a time, retrying the ``'>'`` -at every step. This produces just the right result:: - - >>> print(re.match('<.*?>', s).group()) - - -(Note that parsing HTML or XML with regular expressions is painful. -Quick-and-dirty patterns will handle common cases, but HTML and XML have special -cases that will break the obvious regular expression; by the time you've written -a regular expression that handles all of the possible cases, the patterns will -be *very* complicated. Use an HTML or XML parser module for such tasks.) - - -Using re.VERBOSE ----------------- - -By now you've probably noticed that regular expressions are a very compact -notation, but they're not terribly readable. REs of moderate complexity can -become lengthy collections of backslashes, parentheses, and metacharacters, -making them difficult to read and understand. - -For such REs, specifying the :const:`re.VERBOSE` flag when compiling the regular -expression can be helpful, because it allows you to format the regular -expression more clearly. - -The ``re.VERBOSE`` flag has several effects. Whitespace in the regular -expression that *isn't* inside a character class is ignored. This means that an -expression such as ``dog | cat`` is equivalent to the less readable ``dog|cat``, -but ``[a b]`` will still match the characters ``'a'``, ``'b'``, or a space. In -addition, you can also put comments inside a RE; comments extend from a ``#`` -character to the next newline. When used with triple-quoted strings, this -enables REs to be formatted more neatly:: - - pat = re.compile(r""" - \s* # Skip leading whitespace - (?P
[^:]+) # Header name - \s* : # Whitespace, and a colon - (?P.*?) # The header's value -- *? used to - # lose the following trailing whitespace - \s*$ # Trailing whitespace to end-of-line - """, re.VERBOSE) - -This is far more readable than:: - - pat = re.compile(r"\s*(?P
[^:]+)\s*:(?P.*?)\s*$") - - -Feedback -======== - -Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you -encountered that weren't covered here? If so, please send suggestions for -improvements to the author. - -The most complete book on regular expressions is almost certainly Jeffrey -Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, -it exclusively concentrates on Perl and Java's flavours of regular expressions, -and doesn't contain any Python material at all, so it won't be useful as a -reference for programming in Python. (The first edition covered Python's -now-removed :mod:`!regex` module, which won't help you much.) Consider checking -it out from your library. diff --git a/third_party/python/Doc/howto/sockets.rst b/third_party/python/Doc/howto/sockets.rst deleted file mode 100644 index bc71d85a8..000000000 --- a/third_party/python/Doc/howto/sockets.rst +++ /dev/null @@ -1,383 +0,0 @@ -.. _socket-howto: - -**************************** - Socket Programming HOWTO -**************************** - -:Author: Gordon McMillan - - -.. topic:: Abstract - - Sockets are used nearly everywhere, but are one of the most severely - misunderstood technologies around. This is a 10,000 foot overview of sockets. - It's not really a tutorial - you'll still have work to do in getting things - operational. It doesn't cover the fine points (and there are a lot of them), but - I hope it will give you enough background to begin using them decently. - - -Sockets -======= - -I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of -the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really -know what you're doing (in which case this HOWTO isn't for you!), you'll get -better behavior and performance from a STREAM socket than anything else. I will -try to clear up the mystery of what a socket is, as well as some hints on how to -work with blocking and non-blocking sockets. But I'll start by talking about -blocking sockets. You'll need to know how they work before dealing with -non-blocking sockets. - -Part of the trouble with understanding these things is that "socket" can mean a -number of subtly different things, depending on context. So first, let's make a -distinction between a "client" socket - an endpoint of a conversation, and a -"server" socket, which is more like a switchboard operator. The client -application (your browser, for example) uses "client" sockets exclusively; the -web server it's talking to uses both "server" sockets and "client" sockets. - - -History -------- - -Of the various forms of :abbr:`IPC (Inter Process Communication)`, -sockets are by far the most popular. On any given platform, there are -likely to be other forms of IPC that are faster, but for -cross-platform communication, sockets are about the only game in town. - -They were invented in Berkeley as part of the BSD flavor of Unix. They spread -like wildfire with the Internet. With good reason --- the combination of sockets -with INET makes talking to arbitrary machines around the world unbelievably easy -(at least compared to other schemes). - - -Creating a Socket -================= - -Roughly speaking, when you clicked on the link that brought you to this page, -your browser did something like the following:: - - # create an INET, STREAMing socket - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # now connect to the web server on port 80 - the normal http port - s.connect(("www.python.org", 80)) - -When the ``connect`` completes, the socket ``s`` can be used to send -in a request for the text of the page. The same socket will read the -reply, and then be destroyed. That's right, destroyed. Client sockets -are normally only used for one exchange (or a small set of sequential -exchanges). - -What happens in the web server is a bit more complex. First, the web server -creates a "server socket":: - - # create an INET, STREAMing socket - serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # bind the socket to a public host, and a well-known port - serversocket.bind((socket.gethostname(), 80)) - # become a server socket - serversocket.listen(5) - -A couple things to notice: we used ``socket.gethostname()`` so that the socket -would be visible to the outside world. If we had used ``s.bind(('localhost', -80))`` or ``s.bind(('127.0.0.1', 80))`` we would still have a "server" socket, -but one that was only visible within the same machine. ``s.bind(('', 80))`` -specifies that the socket is reachable by any address the machine happens to -have. - -A second thing to note: low number ports are usually reserved for "well known" -services (HTTP, SNMP etc). If you're playing around, use a nice high number (4 -digits). - -Finally, the argument to ``listen`` tells the socket library that we want it to -queue up as many as 5 connect requests (the normal max) before refusing outside -connections. If the rest of the code is written properly, that should be plenty. - -Now that we have a "server" socket, listening on port 80, we can enter the -mainloop of the web server:: - - while True: - # accept connections from outside - (clientsocket, address) = serversocket.accept() - # now do something with the clientsocket - # in this case, we'll pretend this is a threaded server - ct = client_thread(clientsocket) - ct.run() - -There's actually 3 general ways in which this loop could work - dispatching a -thread to handle ``clientsocket``, create a new process to handle -``clientsocket``, or restructure this app to use non-blocking sockets, and -multiplex between our "server" socket and any active ``clientsocket``\ s using -``select``. More about that later. The important thing to understand now is -this: this is *all* a "server" socket does. It doesn't send any data. It doesn't -receive any data. It just produces "client" sockets. Each ``clientsocket`` is -created in response to some *other* "client" socket doing a ``connect()`` to the -host and port we're bound to. As soon as we've created that ``clientsocket``, we -go back to listening for more connections. The two "clients" are free to chat it -up - they are using some dynamically allocated port which will be recycled when -the conversation ends. - - -IPC ---- - -If you need fast IPC between two processes on one machine, you should look into -pipes or shared memory. If you do decide to use AF_INET sockets, bind the -"server" socket to ``'localhost'``. On most platforms, this will take a -shortcut around a couple of layers of network code and be quite a bit faster. - -.. seealso:: - The :mod:`multiprocessing` integrates cross-platform IPC into a higher-level - API. - - -Using a Socket -============== - -The first thing to note, is that the web browser's "client" socket and the web -server's "client" socket are identical beasts. That is, this is a "peer to peer" -conversation. Or to put it another way, *as the designer, you will have to -decide what the rules of etiquette are for a conversation*. Normally, the -``connect``\ ing socket starts the conversation, by sending in a request, or -perhaps a signon. But that's a design decision - it's not a rule of sockets. - -Now there are two sets of verbs to use for communication. You can use ``send`` -and ``recv``, or you can transform your client socket into a file-like beast and -use ``read`` and ``write``. The latter is the way Java presents its sockets. -I'm not going to talk about it here, except to warn you that you need to use -``flush`` on sockets. These are buffered "files", and a common mistake is to -``write`` something, and then ``read`` for a reply. Without a ``flush`` in -there, you may wait forever for the reply, because the request may still be in -your output buffer. - -Now we come to the major stumbling block of sockets - ``send`` and ``recv`` operate -on the network buffers. They do not necessarily handle all the bytes you hand -them (or expect from them), because their major focus is handling the network -buffers. In general, they return when the associated network buffers have been -filled (``send``) or emptied (``recv``). They then tell you how many bytes they -handled. It is *your* responsibility to call them again until your message has -been completely dealt with. - -When a ``recv`` returns 0 bytes, it means the other side has closed (or is in -the process of closing) the connection. You will not receive any more data on -this connection. Ever. You may be able to send data successfully; I'll talk -more about this later. - -A protocol like HTTP uses a socket for only one transfer. The client sends a -request, then reads a reply. That's it. The socket is discarded. This means that -a client can detect the end of the reply by receiving 0 bytes. - -But if you plan to reuse your socket for further transfers, you need to realize -that *there is no* :abbr:`EOT (End of Transfer)` *on a socket.* I repeat: if a socket -``send`` or ``recv`` returns after handling 0 bytes, the connection has been -broken. If the connection has *not* been broken, you may wait on a ``recv`` -forever, because the socket will *not* tell you that there's nothing more to -read (for now). Now if you think about that a bit, you'll come to realize a -fundamental truth of sockets: *messages must either be fixed length* (yuck), *or -be delimited* (shrug), *or indicate how long they are* (much better), *or end by -shutting down the connection*. The choice is entirely yours, (but some ways are -righter than others). - -Assuming you don't want to end the connection, the simplest solution is a fixed -length message:: - - class MySocket: - """demonstration class only - - coded for clarity, not efficiency - """ - - def __init__(self, sock=None): - if sock is None: - self.sock = socket.socket( - socket.AF_INET, socket.SOCK_STREAM) - else: - self.sock = sock - - def connect(self, host, port): - self.sock.connect((host, port)) - - def mysend(self, msg): - totalsent = 0 - while totalsent < MSGLEN: - sent = self.sock.send(msg[totalsent:]) - if sent == 0: - raise RuntimeError("socket connection broken") - totalsent = totalsent + sent - - def myreceive(self): - chunks = [] - bytes_recd = 0 - while bytes_recd < MSGLEN: - chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048)) - if chunk == b'': - raise RuntimeError("socket connection broken") - chunks.append(chunk) - bytes_recd = bytes_recd + len(chunk) - return b''.join(chunks) - -The sending code here is usable for almost any messaging scheme - in Python you -send strings, and you can use ``len()`` to determine its length (even if it has -embedded ``\0`` characters). It's mostly the receiving code that gets more -complex. (And in C, it's not much worse, except you can't use ``strlen`` if the -message has embedded ``\0``\ s.) - -The easiest enhancement is to make the first character of the message an -indicator of message type, and have the type determine the length. Now you have -two ``recv``\ s - the first to get (at least) that first character so you can -look up the length, and the second in a loop to get the rest. If you decide to -go the delimited route, you'll be receiving in some arbitrary chunk size, (4096 -or 8192 is frequently a good match for network buffer sizes), and scanning what -you've received for a delimiter. - -One complication to be aware of: if your conversational protocol allows multiple -messages to be sent back to back (without some kind of reply), and you pass -``recv`` an arbitrary chunk size, you may end up reading the start of a -following message. You'll need to put that aside and hold onto it, until it's -needed. - -Prefixing the message with its length (say, as 5 numeric characters) gets more -complex, because (believe it or not), you may not get all 5 characters in one -``recv``. In playing around, you'll get away with it; but in high network loads, -your code will very quickly break unless you use two ``recv`` loops - the first -to determine the length, the second to get the data part of the message. Nasty. -This is also when you'll discover that ``send`` does not always manage to get -rid of everything in one pass. And despite having read this, you will eventually -get bit by it! - -In the interests of space, building your character, (and preserving my -competitive position), these enhancements are left as an exercise for the -reader. Lets move on to cleaning up. - - -Binary Data ------------ - -It is perfectly possible to send binary data over a socket. The major problem is -that not all machines use the same formats for binary data. For example, a -Motorola chip will represent a 16 bit integer with the value 1 as the two hex -bytes 00 01. Intel and DEC, however, are byte-reversed - that same 1 is 01 00. -Socket libraries have calls for converting 16 and 32 bit integers - ``ntohl, -htonl, ntohs, htons`` where "n" means *network* and "h" means *host*, "s" means -*short* and "l" means *long*. Where network order is host order, these do -nothing, but where the machine is byte-reversed, these swap the bytes around -appropriately. - -In these days of 32 bit machines, the ascii representation of binary data is -frequently smaller than the binary representation. That's because a surprising -amount of the time, all those longs have the value 0, or maybe 1. The string "0" -would be two bytes, while binary is four. Of course, this doesn't fit well with -fixed-length messages. Decisions, decisions. - - -Disconnecting -============= - -Strictly speaking, you're supposed to use ``shutdown`` on a socket before you -``close`` it. The ``shutdown`` is an advisory to the socket at the other end. -Depending on the argument you pass it, it can mean "I'm not going to send -anymore, but I'll still listen", or "I'm not listening, good riddance!". Most -socket libraries, however, are so used to programmers neglecting to use this -piece of etiquette that normally a ``close`` is the same as ``shutdown(); -close()``. So in most situations, an explicit ``shutdown`` is not needed. - -One way to use ``shutdown`` effectively is in an HTTP-like exchange. The client -sends a request and then does a ``shutdown(1)``. This tells the server "This -client is done sending, but can still receive." The server can detect "EOF" by -a receive of 0 bytes. It can assume it has the complete request. The server -sends a reply. If the ``send`` completes successfully then, indeed, the client -was still receiving. - -Python takes the automatic shutdown a step further, and says that when a socket -is garbage collected, it will automatically do a ``close`` if it's needed. But -relying on this is a very bad habit. If your socket just disappears without -doing a ``close``, the socket at the other end may hang indefinitely, thinking -you're just being slow. *Please* ``close`` your sockets when you're done. - - -When Sockets Die ----------------- - -Probably the worst thing about using blocking sockets is what happens when the -other side comes down hard (without doing a ``close``). Your socket is likely to -hang. TCP is a reliable protocol, and it will wait a long, long time -before giving up on a connection. If you're using threads, the entire thread is -essentially dead. There's not much you can do about it. As long as you aren't -doing something dumb, like holding a lock while doing a blocking read, the -thread isn't really consuming much in the way of resources. Do *not* try to kill -the thread - part of the reason that threads are more efficient than processes -is that they avoid the overhead associated with the automatic recycling of -resources. In other words, if you do manage to kill the thread, your whole -process is likely to be screwed up. - - -Non-blocking Sockets -==================== - -If you've understood the preceding, you already know most of what you need to -know about the mechanics of using sockets. You'll still use the same calls, in -much the same ways. It's just that, if you do it right, your app will be almost -inside-out. - -In Python, you use ``socket.setblocking(0)`` to make it non-blocking. In C, it's -more complex, (for one thing, you'll need to choose between the BSD flavor -``O_NONBLOCK`` and the almost indistinguishable Posix flavor ``O_NDELAY``, which -is completely different from ``TCP_NODELAY``), but it's the exact same idea. You -do this after creating the socket, but before using it. (Actually, if you're -nuts, you can switch back and forth.) - -The major mechanical difference is that ``send``, ``recv``, ``connect`` and -``accept`` can return without having done anything. You have (of course) a -number of choices. You can check return code and error codes and generally drive -yourself crazy. If you don't believe me, try it sometime. Your app will grow -large, buggy and suck CPU. So let's skip the brain-dead solutions and do it -right. - -Use ``select``. - -In C, coding ``select`` is fairly complex. In Python, it's a piece of cake, but -it's close enough to the C version that if you understand ``select`` in Python, -you'll have little trouble with it in C:: - - ready_to_read, ready_to_write, in_error = \ - select.select( - potential_readers, - potential_writers, - potential_errs, - timeout) - -You pass ``select`` three lists: the first contains all sockets that you might -want to try reading; the second all the sockets you might want to try writing -to, and the last (normally left empty) those that you want to check for errors. -You should note that a socket can go into more than one list. The ``select`` -call is blocking, but you can give it a timeout. This is generally a sensible -thing to do - give it a nice long timeout (say a minute) unless you have good -reason to do otherwise. - -In return, you will get three lists. They contain the sockets that are actually -readable, writable and in error. Each of these lists is a subset (possibly -empty) of the corresponding list you passed in. - -If a socket is in the output readable list, you can be -as-close-to-certain-as-we-ever-get-in-this-business that a ``recv`` on that -socket will return *something*. Same idea for the writable list. You'll be able -to send *something*. Maybe not all you want to, but *something* is better than -nothing. (Actually, any reasonably healthy socket will return as writable - it -just means outbound network buffer space is available.) - -If you have a "server" socket, put it in the potential_readers list. If it comes -out in the readable list, your ``accept`` will (almost certainly) work. If you -have created a new socket to ``connect`` to someone else, put it in the -potential_writers list. If it shows up in the writable list, you have a decent -chance that it has connected. - -Actually, ``select`` can be handy even with blocking sockets. It's one way of -determining whether you will block - the socket returns as readable when there's -something in the buffers. However, this still doesn't help with the problem of -determining whether the other end is done, or just busy with something else. - -**Portability alert**: On Unix, ``select`` works both with the sockets and -files. Don't try this on Windows. On Windows, ``select`` works with sockets -only. Also note that in C, many of the more advanced socket options are done -differently on Windows. In fact, on Windows I usually use threads (which work -very, very well) with my sockets. - - diff --git a/third_party/python/Doc/howto/sorting.rst b/third_party/python/Doc/howto/sorting.rst deleted file mode 100644 index 128044662..000000000 --- a/third_party/python/Doc/howto/sorting.rst +++ /dev/null @@ -1,293 +0,0 @@ -.. _sortinghowto: - -Sorting HOW TO -************** - -:Author: Andrew Dalke and Raymond Hettinger -:Release: 0.1 - - -Python lists have a built-in :meth:`list.sort` method that modifies the list -in-place. There is also a :func:`sorted` built-in function that builds a new -sorted list from an iterable. - -In this document, we explore the various techniques for sorting data using Python. - - -Sorting Basics -============== - -A simple ascending sort is very easy: just call the :func:`sorted` function. It -returns a new sorted list:: - - >>> sorted([5, 2, 3, 1, 4]) - [1, 2, 3, 4, 5] - -You can also use the :meth:`list.sort` method. It modifies the list -in-place (and returns ``None`` to avoid confusion). Usually it's less convenient -than :func:`sorted` - but if you don't need the original list, it's slightly -more efficient. - - >>> a = [5, 2, 3, 1, 4] - >>> a.sort() - >>> a - [1, 2, 3, 4, 5] - -Another difference is that the :meth:`list.sort` method is only defined for -lists. In contrast, the :func:`sorted` function accepts any iterable. - - >>> sorted({1: 'D', 2: 'B', 3: 'B', 4: 'E', 5: 'A'}) - [1, 2, 3, 4, 5] - -Key Functions -============= - -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function to be called on each list element prior to making comparisons. - -For example, here's a case-insensitive string comparison: - - >>> sorted("This is a test string from Andrew".split(), key=str.lower) - ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] - -The value of the *key* parameter should be a function that takes a single argument -and returns a key to use for sorting purposes. This technique is fast because -the key function is called exactly once for each input record. - -A common pattern is to sort complex objects using some of the object's indices -as keys. For example: - - >>> student_tuples = [ - ... ('john', 'A', 15), - ... ('jane', 'B', 12), - ... ('dave', 'B', 10), - ... ] - >>> sorted(student_tuples, key=lambda student: student[2]) # sort by age - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The same technique works for objects with named attributes. For example: - - >>> class Student: - ... def __init__(self, name, grade, age): - ... self.name = name - ... self.grade = grade - ... self.age = age - ... def __repr__(self): - ... return repr((self.name, self.grade, self.age)) - - >>> student_objects = [ - ... Student('john', 'A', 15), - ... Student('jane', 'B', 12), - ... Student('dave', 'B', 10), - ... ] - >>> sorted(student_objects, key=lambda student: student.age) # sort by age - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -Operator Module Functions -========================= - -The key-function patterns shown above are very common, so Python provides -convenience functions to make accessor functions easier and faster. The -:mod:`operator` module has :func:`~operator.itemgetter`, -:func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. - -Using those functions, the above examples become simpler and faster: - - >>> from operator import itemgetter, attrgetter - - >>> sorted(student_tuples, key=itemgetter(2)) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - - >>> sorted(student_objects, key=attrgetter('age')) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The operator module functions allow multiple levels of sorting. For example, to -sort by *grade* then by *age*: - - >>> sorted(student_tuples, key=itemgetter(1,2)) - [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] - - >>> sorted(student_objects, key=attrgetter('grade', 'age')) - [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] - -Ascending and Descending -======================== - -Both :meth:`list.sort` and :func:`sorted` accept a *reverse* parameter with a -boolean value. This is used to flag descending sorts. For example, to get the -student data in reverse *age* order: - - >>> sorted(student_tuples, key=itemgetter(2), reverse=True) - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - - >>> sorted(student_objects, key=attrgetter('age'), reverse=True) - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - -Sort Stability and Complex Sorts -================================ - -Sorts are guaranteed to be `stable -`_\. That means that -when multiple records have the same key, their original order is preserved. - - >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] - >>> sorted(data, key=itemgetter(0)) - [('blue', 1), ('blue', 2), ('red', 1), ('red', 2)] - -Notice how the two records for *blue* retain their original order so that -``('blue', 1)`` is guaranteed to precede ``('blue', 2)``. - -This wonderful property lets you build complex sorts in a series of sorting -steps. For example, to sort the student data by descending *grade* and then -ascending *age*, do the *age* sort first and then sort again using *grade*: - - >>> s = sorted(student_objects, key=attrgetter('age')) # sort on secondary key - >>> sorted(s, key=attrgetter('grade'), reverse=True) # now sort on primary key, descending - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The `Timsort `_ algorithm used in Python -does multiple sorts efficiently because it can take advantage of any ordering -already present in a dataset. - -The Old Way Using Decorate-Sort-Undecorate -========================================== - -This idiom is called Decorate-Sort-Undecorate after its three steps: - -* First, the initial list is decorated with new values that control the sort order. - -* Second, the decorated list is sorted. - -* Finally, the decorations are removed, creating a list that contains only the - initial values in the new order. - -For example, to sort the student data by *grade* using the DSU approach: - - >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] - >>> decorated.sort() - >>> [student for grade, i, student in decorated] # undecorate - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - -This idiom works because tuples are compared lexicographically; the first items -are compared; if they are the same then the second items are compared, and so -on. - -It is not strictly necessary in all cases to include the index *i* in the -decorated list, but including it gives two benefits: - -* The sort is stable -- if two items have the same key, their order will be - preserved in the sorted list. - -* The original items do not have to be comparable because the ordering of the - decorated tuples will be determined by at most the first two items. So for - example the original list could contain complex numbers which cannot be sorted - directly. - -Another name for this idiom is -`Schwartzian transform `_\, -after Randal L. Schwartz, who popularized it among Perl programmers. - -Now that Python sorting provides key-functions, this technique is not often needed. - - -The Old Way Using the *cmp* Parameter -===================================== - -Many constructs given in this HOWTO assume Python 2.4 or later. Before that, -there was no :func:`sorted` builtin and :meth:`list.sort` took no keyword -arguments. Instead, all of the Py2.x versions supported a *cmp* parameter to -handle user specified comparison functions. - -In Py3.0, the *cmp* parameter was removed entirely (as part of a larger effort to -simplify and unify the language, eliminating the conflict between rich -comparisons and the :meth:`__cmp__` magic method). - -In Py2.x, sort allowed an optional function which can be called for doing the -comparisons. That function should take two arguments to be compared and then -return a negative value for less-than, return zero if they are equal, or return -a positive value for greater-than. For example, we can do: - - >>> def numeric_compare(x, y): - ... return x - y - >>> sorted([5, 2, 4, 1, 3], cmp=numeric_compare) # doctest: +SKIP - [1, 2, 3, 4, 5] - -Or you can reverse the order of comparison with: - - >>> def reverse_numeric(x, y): - ... return y - x - >>> sorted([5, 2, 4, 1, 3], cmp=reverse_numeric) # doctest: +SKIP - [5, 4, 3, 2, 1] - -When porting code from Python 2.x to 3.x, the situation can arise when you have -the user supplying a comparison function and you need to convert that to a key -function. The following wrapper makes that easy to do:: - - def cmp_to_key(mycmp): - 'Convert a cmp= function into a key= function' - class K: - def __init__(self, obj, *args): - self.obj = obj - def __lt__(self, other): - return mycmp(self.obj, other.obj) < 0 - def __gt__(self, other): - return mycmp(self.obj, other.obj) > 0 - def __eq__(self, other): - return mycmp(self.obj, other.obj) == 0 - def __le__(self, other): - return mycmp(self.obj, other.obj) <= 0 - def __ge__(self, other): - return mycmp(self.obj, other.obj) >= 0 - def __ne__(self, other): - return mycmp(self.obj, other.obj) != 0 - return K - -To convert to a key function, just wrap the old comparison function: - -.. testsetup:: - - from functools import cmp_to_key - -.. doctest:: - - >>> sorted([5, 2, 4, 1, 3], key=cmp_to_key(reverse_numeric)) - [5, 4, 3, 2, 1] - -In Python 3.2, the :func:`functools.cmp_to_key` function was added to the -:mod:`functools` module in the standard library. - -Odd and Ends -============ - -* For locale aware sorting, use :func:`locale.strxfrm` for a key function or - :func:`locale.strcoll` for a comparison function. - -* The *reverse* parameter still maintains sort stability (so that records with - equal keys retain the original order). Interestingly, that effect can be - simulated without the parameter by using the builtin :func:`reversed` function - twice: - - >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] - >>> standard_way = sorted(data, key=itemgetter(0), reverse=True) - >>> double_reversed = list(reversed(sorted(reversed(data), key=itemgetter(0)))) - >>> assert standard_way == double_reversed - >>> standard_way - [('red', 1), ('red', 2), ('blue', 1), ('blue', 2)] - -* The sort routines are guaranteed to use :meth:`__lt__` when making comparisons - between two objects. So, it is easy to add a standard sort order to a class by - defining an :meth:`__lt__` method:: - - >>> Student.__lt__ = lambda self, other: self.age < other.age - >>> sorted(student_objects) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -* Key functions need not depend directly on the objects being sorted. A key - function can also access external resources. For instance, if the student grades - are stored in a dictionary, they can be used to sort a separate list of student - names: - - >>> students = ['dave', 'john', 'jane'] - >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} - >>> sorted(students, key=newgrades.__getitem__) - ['jane', 'dave', 'john'] diff --git a/third_party/python/Doc/howto/unicode.rst b/third_party/python/Doc/howto/unicode.rst deleted file mode 100644 index ec5d48e64..000000000 --- a/third_party/python/Doc/howto/unicode.rst +++ /dev/null @@ -1,733 +0,0 @@ -.. _unicode-howto: - -***************** - Unicode HOWTO -***************** - -:Release: 1.12 - -This HOWTO discusses Python support for Unicode, and explains -various problems that people commonly encounter when trying to work -with Unicode. - -Introduction to Unicode -======================= - -History of Character Codes --------------------------- - -In 1968, the American Standard Code for Information Interchange, better known by -its acronym ASCII, was standardized. ASCII defined numeric codes for various -characters, with the numeric values running from 0 to 127. For example, the -lowercase letter 'a' is assigned 97 as its code value. - -ASCII was an American-developed standard, so it only defined unaccented -characters. There was an 'e', but no 'é' or 'Í'. This meant that languages -which required accented characters couldn't be faithfully represented in ASCII. -(Actually the missing accents matter for English, too, which contains words such -as 'naïve' and 'café', and some publications have house styles which require -spellings such as 'coöperate'.) - -For a while people just wrote programs that didn't display accents. -In the mid-1980s an Apple II BASIC program written by a French speaker -might have lines like these: - -.. code-block:: basic - - PRINT "MISE A JOUR TERMINEE" - PRINT "PARAMETRES ENREGISTRES" - -Those messages should contain accents (terminée, paramètre, enregistrés) and -they just look wrong to someone who can read French. - -In the 1980s, almost all personal computers were 8-bit, meaning that bytes could -hold values ranging from 0 to 255. ASCII codes only went up to 127, so some -machines assigned values between 128 and 255 to accented characters. Different -machines had different codes, however, which led to problems exchanging files. -Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Organization for -Standardization, and some were *de facto* conventions that were invented by one -company or another and managed to catch on. - -255 characters aren't very many. For example, you can't fit both the accented -characters used in Western Europe and the Cyrillic alphabet used for Russian -into the 128--255 range because there are more than 128 such characters. - -You could write files using different codes (all your Russian files in a coding -system called KOI8, all your French files in a different coding system called -Latin1), but what if you wanted to write a French document that quotes some -Russian text? In the 1980s people began to want to solve this problem, and the -Unicode standardization effort began. - -Unicode started out using 16-bit characters instead of 8-bit characters. 16 -bits means you have 2^16 = 65,536 distinct values available, making it possible -to represent many different characters from many different alphabets; an initial -goal was to have Unicode contain the alphabets for every single human language. -It turns out that even 16 bits isn't enough to meet that goal, and the modern -Unicode specification uses a wider range of codes, 0 through 1,114,111 ( -``0x10FFFF`` in base 16). - -There's a related ISO standard, ISO 10646. Unicode and ISO 10646 were -originally separate efforts, but the specifications were merged with the 1.1 -revision of Unicode. - -(This discussion of Unicode's history is highly simplified. The -precise historical details aren't necessary for understanding how to -use Unicode effectively, but if you're curious, consult the Unicode -consortium site listed in the References or -the `Wikipedia entry for Unicode `_ -for more information.) - - -Definitions ------------ - -A **character** is the smallest possible component of a text. 'A', 'B', 'C', -etc., are all different characters. So are 'È' and 'Í'. Characters are -abstractions, and vary depending on the language or context you're talking -about. For example, the symbol for ohms (Ω) is usually drawn much like the -capital letter omega (Ω) in the Greek alphabet (they may even be the same in -some fonts), but these are two different characters that have different -meanings. - -The Unicode standard describes how characters are represented by **code -points**. A code point is an integer value, usually denoted in base 16. In the -standard, a code point is written using the notation ``U+12CA`` to mean the -character with value ``0x12ca`` (4,810 decimal). The Unicode standard contains -a lot of tables listing characters and their corresponding code points: - -.. code-block:: none - - 0061 'a'; LATIN SMALL LETTER A - 0062 'b'; LATIN SMALL LETTER B - 0063 'c'; LATIN SMALL LETTER C - ... - 007B '{'; LEFT CURLY BRACKET - -Strictly, these definitions imply that it's meaningless to say 'this is -character ``U+12CA``'. ``U+12CA`` is a code point, which represents some particular -character; in this case, it represents the character 'ETHIOPIC SYLLABLE WI'. In -informal contexts, this distinction between code points and characters will -sometimes be forgotten. - -A character is represented on a screen or on paper by a set of graphical -elements that's called a **glyph**. The glyph for an uppercase A, for example, -is two diagonal strokes and a horizontal stroke, though the exact details will -depend on the font being used. Most Python code doesn't need to worry about -glyphs; figuring out the correct glyph to display is generally the job of a GUI -toolkit or a terminal's font renderer. - - -Encodings ---------- - -To summarize the previous section: a Unicode string is a sequence of code -points, which are numbers from 0 through ``0x10FFFF`` (1,114,111 decimal). This -sequence needs to be represented as a set of bytes (meaning, values -from 0 through 255) in memory. The rules for translating a Unicode string -into a sequence of bytes are called an **encoding**. - -The first encoding you might think of is an array of 32-bit integers. In this -representation, the string "Python" would look like this: - -.. code-block:: none - - P y t h o n - 0x50 00 00 00 79 00 00 00 74 00 00 00 68 00 00 00 6f 00 00 00 6e 00 00 00 - 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 - -This representation is straightforward but using it presents a number of -problems. - -1. It's not portable; different processors order the bytes differently. - -2. It's very wasteful of space. In most texts, the majority of the code points - are less than 127, or less than 255, so a lot of space is occupied by ``0x00`` - bytes. The above string takes 24 bytes compared to the 6 bytes needed for an - ASCII representation. Increased RAM usage doesn't matter too much (desktop - computers have gigabytes of RAM, and strings aren't usually that large), but - expanding our usage of disk and network bandwidth by a factor of 4 is - intolerable. - -3. It's not compatible with existing C functions such as ``strlen()``, so a new - family of wide string functions would need to be used. - -4. Many Internet standards are defined in terms of textual data, and can't - handle content with embedded zero bytes. - -Generally people don't use this encoding, instead choosing other -encodings that are more efficient and convenient. UTF-8 is probably -the most commonly supported encoding; it will be discussed below. - -Encodings don't have to handle every possible Unicode character, and most -encodings don't. The rules for converting a Unicode string into the ASCII -encoding, for example, are simple; for each code point: - -1. If the code point is < 128, each byte is the same as the value of the code - point. - -2. If the code point is 128 or greater, the Unicode string can't be represented - in this encoding. (Python raises a :exc:`UnicodeEncodeError` exception in this - case.) - -Latin-1, also known as ISO-8859-1, is a similar encoding. Unicode code points -0--255 are identical to the Latin-1 values, so converting to this encoding simply -requires converting code points to byte values; if a code point larger than 255 -is encountered, the string can't be encoded into Latin-1. - -Encodings don't have to be simple one-to-one mappings like Latin-1. Consider -IBM's EBCDIC, which was used on IBM mainframes. Letter values weren't in one -block: 'a' through 'i' had values from 129 to 137, but 'j' through 'r' were 145 -through 153. If you wanted to use EBCDIC as an encoding, you'd probably use -some sort of lookup table to perform the conversion, but this is largely an -internal detail. - -UTF-8 is one of the most commonly used encodings. UTF stands for "Unicode -Transformation Format", and the '8' means that 8-bit numbers are used in the -encoding. (There are also a UTF-16 and UTF-32 encodings, but they are less -frequently used than UTF-8.) UTF-8 uses the following rules: - -1. If the code point is < 128, it's represented by the corresponding byte value. -2. If the code point is >= 128, it's turned into a sequence of two, three, or - four bytes, where each byte of the sequence is between 128 and 255. - -UTF-8 has several convenient properties: - -1. It can handle any Unicode code point. -2. A Unicode string is turned into a sequence of bytes containing no embedded zero - bytes. This avoids byte-ordering issues, and means UTF-8 strings can be - processed by C functions such as ``strcpy()`` and sent through protocols that - can't handle zero bytes. -3. A string of ASCII text is also valid UTF-8 text. -4. UTF-8 is fairly compact; the majority of commonly used characters can be - represented with one or two bytes. -5. If bytes are corrupted or lost, it's possible to determine the start of the - next UTF-8-encoded code point and resynchronize. It's also unlikely that - random 8-bit data will look like valid UTF-8. - - - -References ----------- - -The `Unicode Consortium site `_ has character charts, a -glossary, and PDF versions of the Unicode specification. Be prepared for some -difficult reading. `A chronology `_ of the -origin and development of Unicode is also available on the site. - -To help understand the standard, Jukka Korpela has written `an introductory -guide `_ to reading the -Unicode character tables. - -Another `good introductory article `_ -was written by Joel Spolsky. -If this introduction didn't make things clear to you, you should try -reading this alternate article before continuing. - -Wikipedia entries are often helpful; see the entries for "`character encoding -`_" and `UTF-8 -`_, for example. - - -Python's Unicode Support -======================== - -Now that you've learned the rudiments of Unicode, we can look at Python's -Unicode features. - -The String Type ---------------- - -Since Python 3.0, the language features a :class:`str` type that contain Unicode -characters, meaning any string created using ``"unicode rocks!"``, ``'unicode -rocks!'``, or the triple-quoted string syntax is stored as Unicode. - -The default encoding for Python source code is UTF-8, so you can simply -include a Unicode character in a string literal:: - - try: - with open('/tmp/input.txt', 'r') as f: - ... - except OSError: - # 'File not found' error message. - print("Fichier non trouvé") - -You can use a different encoding from UTF-8 by putting a specially-formatted -comment as the first or second line of the source code:: - - # -*- coding: -*- - -Side note: Python 3 also supports using Unicode characters in identifiers:: - - répertoire = "/tmp/records.log" - with open(répertoire, "w") as f: - f.write("test\n") - -If you can't enter a particular character in your editor or want to -keep the source code ASCII-only for some reason, you can also use -escape sequences in string literals. (Depending on your system, -you may see the actual capital-delta glyph instead of a \u escape.) :: - - >>> "\N{GREEK CAPITAL LETTER DELTA}" # Using the character name - '\u0394' - >>> "\u0394" # Using a 16-bit hex value - '\u0394' - >>> "\U00000394" # Using a 32-bit hex value - '\u0394' - -In addition, one can create a string using the :func:`~bytes.decode` method of -:class:`bytes`. This method takes an *encoding* argument, such as ``UTF-8``, -and optionally an *errors* argument. - -The *errors* argument specifies the response when the input string can't be -converted according to the encoding's rules. Legal values for this argument are -``'strict'`` (raise a :exc:`UnicodeDecodeError` exception), ``'replace'`` (use -``U+FFFD``, ``REPLACEMENT CHARACTER``), ``'ignore'`` (just leave the -character out of the Unicode result), or ``'backslashreplace'`` (inserts a -``\xNN`` escape sequence). -The following examples show the differences:: - - >>> b'\x80abc'.decode("utf-8", "strict") #doctest: +NORMALIZE_WHITESPACE - Traceback (most recent call last): - ... - UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: - invalid start byte - >>> b'\x80abc'.decode("utf-8", "replace") - '\ufffdabc' - >>> b'\x80abc'.decode("utf-8", "backslashreplace") - '\\x80abc' - >>> b'\x80abc'.decode("utf-8", "ignore") - 'abc' - -Encodings are specified as strings containing the encoding's name. Python 3.2 -comes with roughly 100 different encodings; see the Python Library Reference at -:ref:`standard-encodings` for a list. Some encodings have multiple names; for -example, ``'latin-1'``, ``'iso_8859_1'`` and ``'8859``' are all synonyms for -the same encoding. - -One-character Unicode strings can also be created with the :func:`chr` -built-in function, which takes integers and returns a Unicode string of length 1 -that contains the corresponding code point. The reverse operation is the -built-in :func:`ord` function that takes a one-character Unicode string and -returns the code point value:: - - >>> chr(57344) - '\ue000' - >>> ord('\ue000') - 57344 - -Converting to Bytes -------------------- - -The opposite method of :meth:`bytes.decode` is :meth:`str.encode`, -which returns a :class:`bytes` representation of the Unicode string, encoded in the -requested *encoding*. - -The *errors* parameter is the same as the parameter of the -:meth:`~bytes.decode` method but supports a few more possible handlers. As well as -``'strict'``, ``'ignore'``, and ``'replace'`` (which in this case -inserts a question mark instead of the unencodable character), there is -also ``'xmlcharrefreplace'`` (inserts an XML character reference), -``backslashreplace`` (inserts a ``\uNNNN`` escape sequence) and -``namereplace`` (inserts a ``\N{...}`` escape sequence). - -The following example shows the different results:: - - >>> u = chr(40960) + 'abcd' + chr(1972) - >>> u.encode('utf-8') - b'\xea\x80\x80abcd\xde\xb4' - >>> u.encode('ascii') #doctest: +NORMALIZE_WHITESPACE - Traceback (most recent call last): - ... - UnicodeEncodeError: 'ascii' codec can't encode character '\ua000' in - position 0: ordinal not in range(128) - >>> u.encode('ascii', 'ignore') - b'abcd' - >>> u.encode('ascii', 'replace') - b'?abcd?' - >>> u.encode('ascii', 'xmlcharrefreplace') - b'ꀀabcd޴' - >>> u.encode('ascii', 'backslashreplace') - b'\\ua000abcd\\u07b4' - >>> u.encode('ascii', 'namereplace') - b'\\N{YI SYLLABLE IT}abcd\\u07b4' - -The low-level routines for registering and accessing the available -encodings are found in the :mod:`codecs` module. Implementing new -encodings also requires understanding the :mod:`codecs` module. -However, the encoding and decoding functions returned by this module -are usually more low-level than is comfortable, and writing new encodings -is a specialized task, so the module won't be covered in this HOWTO. - - -Unicode Literals in Python Source Code --------------------------------------- - -In Python source code, specific Unicode code points can be written using the -``\u`` escape sequence, which is followed by four hex digits giving the code -point. The ``\U`` escape sequence is similar, but expects eight hex digits, -not four:: - - >>> s = "a\xac\u1234\u20ac\U00008000" - ... # ^^^^ two-digit hex escape - ... # ^^^^^^ four-digit Unicode escape - ... # ^^^^^^^^^^ eight-digit Unicode escape - >>> [ord(c) for c in s] - [97, 172, 4660, 8364, 32768] - -Using escape sequences for code points greater than 127 is fine in small doses, -but becomes an annoyance if you're using many accented characters, as you would -in a program with messages in French or some other accent-using language. You -can also assemble strings using the :func:`chr` built-in function, but this is -even more tedious. - -Ideally, you'd want to be able to write literals in your language's natural -encoding. You could then edit Python source code with your favorite editor -which would display the accented characters naturally, and have the right -characters used at runtime. - -Python supports writing source code in UTF-8 by default, but you can use almost -any encoding if you declare the encoding being used. This is done by including -a special comment as either the first or second line of the source file:: - - #!/usr/bin/env python - # -*- coding: latin-1 -*- - - u = 'abcdé' - print(ord(u[-1])) - -The syntax is inspired by Emacs's notation for specifying variables local to a -file. Emacs supports many different variables, but Python only supports -'coding'. The ``-*-`` symbols indicate to Emacs that the comment is special; -they have no significance to Python but are a convention. Python looks for -``coding: name`` or ``coding=name`` in the comment. - -If you don't include such a comment, the default encoding used will be UTF-8 as -already mentioned. See also :pep:`263` for more information. - - -Unicode Properties ------------------- - -The Unicode specification includes a database of information about code points. -For each defined code point, the information includes the character's -name, its category, the numeric value if applicable (Unicode has characters -representing the Roman numerals and fractions such as one-third and -four-fifths). There are also properties related to the code point's use in -bidirectional text and other display-related properties. - -The following program displays some information about several characters, and -prints the numeric value of one particular character:: - - import unicodedata - - u = chr(233) + chr(0x0bf2) + chr(3972) + chr(6000) + chr(13231) - - for i, c in enumerate(u): - print(i, '%04x' % ord(c), unicodedata.category(c), end=" ") - print(unicodedata.name(c)) - - # Get numeric value of second character - print(unicodedata.numeric(u[1])) - -When run, this prints: - -.. code-block:: none - - 0 00e9 Ll LATIN SMALL LETTER E WITH ACUTE - 1 0bf2 No TAMIL NUMBER ONE THOUSAND - 2 0f84 Mn TIBETAN MARK HALANTA - 3 1770 Lo TAGBANWA LETTER SA - 4 33af So SQUARE RAD OVER S SQUARED - 1000.0 - -The category codes are abbreviations describing the nature of the character. -These are grouped into categories such as "Letter", "Number", "Punctuation", or -"Symbol", which in turn are broken up into subcategories. To take the codes -from the above output, ``'Ll'`` means 'Letter, lowercase', ``'No'`` means -"Number, other", ``'Mn'`` is "Mark, nonspacing", and ``'So'`` is "Symbol, -other". See -`the General Category Values section of the Unicode Character Database documentation `_ for a -list of category codes. - - -Unicode Regular Expressions ---------------------------- - -The regular expressions supported by the :mod:`re` module can be provided -either as bytes or strings. Some of the special character sequences such as -``\d`` and ``\w`` have different meanings depending on whether -the pattern is supplied as bytes or a string. For example, -``\d`` will match the characters ``[0-9]`` in bytes but -in strings will match any character that's in the ``'Nd'`` category. - -The string in this example has the number 57 written in both Thai and -Arabic numerals:: - - import re - p = re.compile(r'\d+') - - s = "Over \u0e55\u0e57 57 flavours" - m = p.search(s) - print(repr(m.group())) - -When executed, ``\d+`` will match the Thai numerals and print them -out. If you supply the :const:`re.ASCII` flag to -:func:`~re.compile`, ``\d+`` will match the substring "57" instead. - -Similarly, ``\w`` matches a wide variety of Unicode characters but -only ``[a-zA-Z0-9_]`` in bytes or if :const:`re.ASCII` is supplied, -and ``\s`` will match either Unicode whitespace characters or -``[ \t\n\r\f\v]``. - - -References ----------- - -.. comment should these be mentioned earlier, e.g. at the start of the "introduction to Unicode" first section? - -Some good alternative discussions of Python's Unicode support are: - -* `Processing Text Files in Python 3 `_, by Nick Coghlan. -* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder. - -The :class:`str` type is described in the Python library reference at -:ref:`textseq`. - -The documentation for the :mod:`unicodedata` module. - -The documentation for the :mod:`codecs` module. - -Marc-André Lemburg gave `a presentation titled "Python and Unicode" (PDF slides) -`_ at -EuroPython 2002. The slides are an excellent overview of the design of Python -2's Unicode features (where the Unicode string type is called ``unicode`` and -literals start with ``u``). - - -Reading and Writing Unicode Data -================================ - -Once you've written some code that works with Unicode data, the next problem is -input/output. How do you get Unicode strings into your program, and how do you -convert Unicode into a form suitable for storage or transmission? - -It's possible that you may not need to do anything depending on your input -sources and output destinations; you should check whether the libraries used in -your application support Unicode natively. XML parsers often return Unicode -data, for example. Many relational databases also support Unicode-valued -columns and can return Unicode values from an SQL query. - -Unicode data is usually converted to a particular encoding before it gets -written to disk or sent over a socket. It's possible to do all the work -yourself: open a file, read an 8-bit bytes object from it, and convert the bytes -with ``bytes.decode(encoding)``. However, the manual approach is not recommended. - -One problem is the multi-byte nature of encodings; one Unicode character can be -represented by several bytes. If you want to read the file in arbitrary-sized -chunks (say, 1024 or 4096 bytes), you need to write error-handling code to catch the case -where only part of the bytes encoding a single Unicode character are read at the -end of a chunk. One solution would be to read the entire file into memory and -then perform the decoding, but that prevents you from working with files that -are extremely large; if you need to read a 2 GiB file, you need 2 GiB of RAM. -(More, really, since for at least a moment you'd need to have both the encoded -string and its Unicode version in memory.) - -The solution would be to use the low-level decoding interface to catch the case -of partial coding sequences. The work of implementing this has already been -done for you: the built-in :func:`open` function can return a file-like object -that assumes the file's contents are in a specified encoding and accepts Unicode -parameters for methods such as :meth:`~io.TextIOBase.read` and -:meth:`~io.TextIOBase.write`. This works through :func:`open`\'s *encoding* and -*errors* parameters which are interpreted just like those in :meth:`str.encode` -and :meth:`bytes.decode`. - -Reading Unicode from a file is therefore simple:: - - with open('unicode.txt', encoding='utf-8') as f: - for line in f: - print(repr(line)) - -It's also possible to open files in update mode, allowing both reading and -writing:: - - with open('test', encoding='utf-8', mode='w+') as f: - f.write('\u4500 blah blah blah\n') - f.seek(0) - print(repr(f.readline()[:1])) - -The Unicode character ``U+FEFF`` is used as a byte-order mark (BOM), and is often -written as the first character of a file in order to assist with autodetection -of the file's byte ordering. Some encodings, such as UTF-16, expect a BOM to be -present at the start of a file; when such an encoding is used, the BOM will be -automatically written as the first character and will be silently dropped when -the file is read. There are variants of these encodings, such as 'utf-16-le' -and 'utf-16-be' for little-endian and big-endian encodings, that specify one -particular byte ordering and don't skip the BOM. - -In some areas, it is also convention to use a "BOM" at the start of UTF-8 -encoded files; the name is misleading since UTF-8 is not byte-order dependent. -The mark simply announces that the file is encoded in UTF-8. Use the -'utf-8-sig' codec to automatically skip the mark if present for reading such -files. - - -Unicode filenames ------------------ - -Most of the operating systems in common use today support filenames that contain -arbitrary Unicode characters. Usually this is implemented by converting the -Unicode string into some encoding that varies depending on the system. For -example, Mac OS X uses UTF-8 while Windows uses a configurable encoding; on -Windows, Python uses the name "mbcs" to refer to whatever the currently -configured encoding is. On Unix systems, there will only be a filesystem -encoding if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if -you haven't, the default encoding is UTF-8. - -The :func:`sys.getfilesystemencoding` function returns the encoding to use on -your current system, in case you want to do the encoding manually, but there's -not much reason to bother. When opening a file for reading or writing, you can -usually just provide the Unicode string as the filename, and it will be -automatically converted to the right encoding for you:: - - filename = 'filename\u4500abc' - with open(filename, 'w') as f: - f.write('blah\n') - -Functions in the :mod:`os` module such as :func:`os.stat` will also accept Unicode -filenames. - -The :func:`os.listdir` function returns filenames and raises an issue: should it return -the Unicode version of filenames, or should it return bytes containing -the encoded versions? :func:`os.listdir` will do both, depending on whether you -provided the directory path as bytes or a Unicode string. If you pass a -Unicode string as the path, filenames will be decoded using the filesystem's -encoding and a list of Unicode strings will be returned, while passing a byte -path will return the filenames as bytes. For example, -assuming the default filesystem encoding is UTF-8, running the following -program:: - - fn = 'filename\u4500abc' - f = open(fn, 'w') - f.close() - - import os - print(os.listdir(b'.')) - print(os.listdir('.')) - -will produce the following output: - -.. code-block:: shell-session - - amk:~$ python t.py - [b'filename\xe4\x94\x80abc', ...] - ['filename\u4500abc', ...] - -The first list contains UTF-8-encoded filenames, and the second list contains -the Unicode versions. - -Note that on most occasions, the Unicode APIs should be used. The bytes APIs -should only be used on systems where undecodable file names can be present, -i.e. Unix systems. - - -Tips for Writing Unicode-aware Programs ---------------------------------------- - -This section provides some suggestions on writing software that deals with -Unicode. - -The most important tip is: - - Software should only work with Unicode strings internally, decoding the input - data as soon as possible and encoding the output only at the end. - -If you attempt to write processing functions that accept both Unicode and byte -strings, you will find your program vulnerable to bugs wherever you combine the -two different kinds of strings. There is no automatic encoding or decoding: if -you do e.g. ``str + bytes``, a :exc:`TypeError` will be raised. - -When using data coming from a web browser or some other untrusted source, a -common technique is to check for illegal characters in a string before using the -string in a generated command line or storing it in a database. If you're doing -this, be careful to check the decoded string, not the encoded bytes data; -some encodings may have interesting properties, such as not being bijective -or not being fully ASCII-compatible. This is especially true if the input -data also specifies the encoding, since the attacker can then choose a -clever way to hide malicious text in the encoded bytestream. - - -Converting Between File Encodings -''''''''''''''''''''''''''''''''' - -The :class:`~codecs.StreamRecoder` class can transparently convert between -encodings, taking a stream that returns data in encoding #1 -and behaving like a stream returning data in encoding #2. - -For example, if you have an input file *f* that's in Latin-1, you -can wrap it with a :class:`~codecs.StreamRecoder` to return bytes encoded in -UTF-8:: - - new_f = codecs.StreamRecoder(f, - # en/decoder: used by read() to encode its results and - # by write() to decode its input. - codecs.getencoder('utf-8'), codecs.getdecoder('utf-8'), - - # reader/writer: used to read and write to the stream. - codecs.getreader('latin-1'), codecs.getwriter('latin-1') ) - - -Files in an Unknown Encoding -'''''''''''''''''''''''''''' - -What can you do if you need to make a change to a file, but don't know -the file's encoding? If you know the encoding is ASCII-compatible and -only want to examine or modify the ASCII parts, you can open the file -with the ``surrogateescape`` error handler:: - - with open(fname, 'r', encoding="ascii", errors="surrogateescape") as f: - data = f.read() - - # make changes to the string 'data' - - with open(fname + '.new', 'w', - encoding="ascii", errors="surrogateescape") as f: - f.write(data) - -The ``surrogateescape`` error handler will decode any non-ASCII bytes -as code points in the Unicode Private Use Area ranging from U+DC80 to -U+DCFF. These private code points will then be turned back into the -same bytes when the ``surrogateescape`` error handler is used when -encoding the data and writing it back out. - - -References ----------- - -One section of `Mastering Python 3 Input/Output -`_, -a PyCon 2010 talk by David Beazley, discusses text processing and binary data handling. - -The `PDF slides for Marc-André Lemburg's presentation "Writing Unicode-aware -Applications in Python" -`_ -discuss questions of character encodings as well as how to internationalize -and localize an application. These slides cover Python 2.x only. - -`The Guts of Unicode in Python -`_ -is a PyCon 2013 talk by Benjamin Peterson that discusses the internal Unicode -representation in Python 3.3. - - -Acknowledgements -================ - -The initial draft of this document was written by Andrew Kuchling. -It has since been revised further by Alexander Belopolsky, Georg Brandl, -Andrew Kuchling, and Ezio Melotti. - -Thanks to the following people who have noted errors or offered -suggestions on this article: Éric Araujo, Nicholas Bastin, Nick -Coghlan, Marius Gedminas, Kent Johnson, Ken Krugler, Marc-André -Lemburg, Martin von Löwis, Terry J. Reedy, Chad Whitacre. diff --git a/third_party/python/Doc/howto/urllib2.rst b/third_party/python/Doc/howto/urllib2.rst deleted file mode 100644 index 7505e7564..000000000 --- a/third_party/python/Doc/howto/urllib2.rst +++ /dev/null @@ -1,605 +0,0 @@ -.. _urllib-howto: - -*********************************************************** - HOWTO Fetch Internet Resources Using The urllib Package -*********************************************************** - -:Author: `Michael Foord `_ - -.. note:: - - There is a French translation of an earlier revision of this - HOWTO, available at `urllib2 - Le Manuel manquant - `_. - - - -Introduction -============ - -.. sidebar:: Related Articles - - You may also find useful the following article on fetching web resources - with Python: - - * `Basic Authentication `_ - - A tutorial on *Basic Authentication*, with examples in Python. - -**urllib.request** is a Python module for fetching URLs -(Uniform Resource Locators). It offers a very simple interface, in the form of -the *urlopen* function. This is capable of fetching URLs using a variety of -different protocols. It also offers a slightly more complex interface for -handling common situations - like basic authentication, cookies, proxies and so -on. These are provided by objects called handlers and openers. - -urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of -``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). -This tutorial focuses on the most common case, HTTP. - -For straightforward situations *urlopen* is very easy to use. But as soon as you -encounter errors or non-trivial cases when opening HTTP URLs, you will need some -understanding of the HyperText Transfer Protocol. The most comprehensive and -authoritative reference to HTTP is :rfc:`2616`. This is a technical document and -not intended to be easy to read. This HOWTO aims to illustrate using *urllib*, -with enough detail about HTTP to help you through. It is not intended to replace -the :mod:`urllib.request` docs, but is supplementary to them. - - -Fetching URLs -============= - -The simplest way to use urllib.request is as follows:: - - import urllib.request - with urllib.request.urlopen('http://python.org/') as response: - html = response.read() - -If you wish to retrieve a resource via URL and store it in a temporary -location, you can do so via the :func:`shutil.copyfileobj` and -:func:`tempfile.NamedTemporaryFile` functions:: - - import shutil - import tempfile - import urllib.request - - with urllib.request.urlopen('http://python.org/') as response: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - shutil.copyfileobj(response, tmp_file) - - with open(tmp_file.name) as html: - pass - -Many uses of urllib will be that simple (note that instead of an 'http:' URL we -could have used a URL starting with 'ftp:', 'file:', etc.). However, it's the -purpose of this tutorial to explain the more complicated cases, concentrating on -HTTP. - -HTTP is based on requests and responses - the client makes requests and servers -send responses. urllib.request mirrors this with a ``Request`` object which represents -the HTTP request you are making. In its simplest form you create a Request -object that specifies the URL you want to fetch. Calling ``urlopen`` with this -Request object returns a response object for the URL requested. This response is -a file-like object, which means you can for example call ``.read()`` on the -response:: - - import urllib.request - - req = urllib.request.Request('http://www.voidspace.org.uk') - with urllib.request.urlopen(req) as response: - the_page = response.read() - -Note that urllib.request makes use of the same Request interface to handle all URL -schemes. For example, you can make an FTP request like so:: - - req = urllib.request.Request('ftp://example.com/') - -In the case of HTTP, there are two extra things that Request objects allow you -to do: First, you can pass data to be sent to the server. Second, you can pass -extra information ("metadata") *about* the data or the about request itself, to -the server - this information is sent as HTTP "headers". Let's look at each of -these in turn. - -Data ----- - -Sometimes you want to send data to a URL (often the URL will refer to a CGI -(Common Gateway Interface) script or other web application). With HTTP, -this is often done using what's known as a **POST** request. This is often what -your browser does when you submit a HTML form that you filled in on the web. Not -all POSTs have to come from forms: you can use a POST to transmit arbitrary data -to your own application. In the common case of HTML forms, the data needs to be -encoded in a standard way, and then passed to the Request object as the ``data`` -argument. The encoding is done using a function from the :mod:`urllib.parse` -library. :: - - import urllib.parse - import urllib.request - - url = 'http://www.someserver.com/cgi-bin/register.cgi' - values = {'name' : 'Michael Foord', - 'location' : 'Northampton', - 'language' : 'Python' } - - data = urllib.parse.urlencode(values) - data = data.encode('ascii') # data should be bytes - req = urllib.request.Request(url, data) - with urllib.request.urlopen(req) as response: - the_page = response.read() - -Note that other encodings are sometimes required (e.g. for file upload from HTML -forms - see `HTML Specification, Form Submission -`_ for more -details). - -If you do not pass the ``data`` argument, urllib uses a **GET** request. One -way in which GET and POST requests differ is that POST requests often have -"side-effects": they change the state of the system in some way (for example by -placing an order with the website for a hundredweight of tinned spam to be -delivered to your door). Though the HTTP standard makes it clear that POSTs are -intended to *always* cause side-effects, and GET requests *never* to cause -side-effects, nothing prevents a GET request from having side-effects, nor a -POST requests from having no side-effects. Data can also be passed in an HTTP -GET request by encoding it in the URL itself. - -This is done as follows:: - - >>> import urllib.request - >>> import urllib.parse - >>> data = {} - >>> data['name'] = 'Somebody Here' - >>> data['location'] = 'Northampton' - >>> data['language'] = 'Python' - >>> url_values = urllib.parse.urlencode(data) - >>> print(url_values) # The order may differ from below. #doctest: +SKIP - name=Somebody+Here&language=Python&location=Northampton - >>> url = 'http://www.example.com/example.cgi' - >>> full_url = url + '?' + url_values - >>> data = urllib.request.urlopen(full_url) - -Notice that the full URL is created by adding a ``?`` to the URL, followed by -the encoded values. - -Headers -------- - -We'll discuss here one particular HTTP header, to illustrate how to add headers -to your HTTP request. - -Some websites [#]_ dislike being browsed by programs, or send different versions -to different browsers [#]_. By default urllib identifies itself as -``Python-urllib/x.y`` (where ``x`` and ``y`` are the major and minor version -numbers of the Python release, -e.g. ``Python-urllib/2.5``), which may confuse the site, or just plain -not work. The way a browser identifies itself is through the -``User-Agent`` header [#]_. When you create a Request object you can -pass a dictionary of headers in. The following example makes the same -request as above, but identifies itself as a version of Internet -Explorer [#]_. :: - - import urllib.parse - import urllib.request - - url = 'http://www.someserver.com/cgi-bin/register.cgi' - user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)' - values = {'name': 'Michael Foord', - 'location': 'Northampton', - 'language': 'Python' } - headers = {'User-Agent': user_agent} - - data = urllib.parse.urlencode(values) - data = data.encode('ascii') - req = urllib.request.Request(url, data, headers) - with urllib.request.urlopen(req) as response: - the_page = response.read() - -The response also has two useful methods. See the section on `info and geturl`_ -which comes after we have a look at what happens when things go wrong. - - -Handling Exceptions -=================== - -*urlopen* raises :exc:`URLError` when it cannot handle a response (though as -usual with Python APIs, built-in exceptions such as :exc:`ValueError`, -:exc:`TypeError` etc. may also be raised). - -:exc:`HTTPError` is the subclass of :exc:`URLError` raised in the specific case of -HTTP URLs. - -The exception classes are exported from the :mod:`urllib.error` module. - -URLError --------- - -Often, URLError is raised because there is no network connection (no route to -the specified server), or the specified server doesn't exist. In this case, the -exception raised will have a 'reason' attribute, which is a tuple containing an -error code and a text error message. - -e.g. :: - - >>> req = urllib.request.Request('http://www.pretend_server.org') - >>> try: urllib.request.urlopen(req) - ... except urllib.error.URLError as e: - ... print(e.reason) #doctest: +SKIP - ... - (4, 'getaddrinfo failed') - - -HTTPError ---------- - -Every HTTP response from the server contains a numeric "status code". Sometimes -the status code indicates that the server is unable to fulfil the request. The -default handlers will handle some of these responses for you (for example, if -the response is a "redirection" that requests the client fetch the document from -a different URL, urllib will handle that for you). For those it can't handle, -urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not -found), '403' (request forbidden), and '401' (authentication required). - -See section 10 of :rfc:`2616` for a reference on all the HTTP error codes. - -The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which -corresponds to the error sent by the server. - -Error Codes -~~~~~~~~~~~ - -Because the default handlers handle redirects (codes in the 300 range), and -codes in the 100--299 range indicate success, you will usually only see error -codes in the 400--599 range. - -:attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: - - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. - responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), - } - -When an error is raised the server responds by returning an HTTP error code -*and* an error page. You can use the :exc:`HTTPError` instance as a response on the -page returned. This means that as well as the code attribute, it also has read, -geturl, and info, methods as returned by the ``urllib.response`` module:: - - >>> req = urllib.request.Request('http://www.python.org/fish.html') - >>> try: - ... urllib.request.urlopen(req) - ... except urllib.error.HTTPError as e: - ... print(e.code) - ... print(e.read()) #doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... - 404 - b'\n\n\nPage Not Found\n - ... - -Wrapping it Up --------------- - -So if you want to be prepared for :exc:`HTTPError` *or* :exc:`URLError` there are two -basic approaches. I prefer the second approach. - -Number 1 -~~~~~~~~ - -:: - - - from urllib.request import Request, urlopen - from urllib.error import URLError, HTTPError - req = Request(someurl) - try: - response = urlopen(req) - except HTTPError as e: - print('The server couldn\'t fulfill the request.') - print('Error code: ', e.code) - except URLError as e: - print('We failed to reach a server.') - print('Reason: ', e.reason) - else: - # everything is fine - - -.. note:: - - The ``except HTTPError`` *must* come first, otherwise ``except URLError`` - will *also* catch an :exc:`HTTPError`. - -Number 2 -~~~~~~~~ - -:: - - from urllib.request import Request, urlopen - from urllib.error import URLError - req = Request(someurl) - try: - response = urlopen(req) - except URLError as e: - if hasattr(e, 'reason'): - print('We failed to reach a server.') - print('Reason: ', e.reason) - elif hasattr(e, 'code'): - print('The server couldn\'t fulfill the request.') - print('Error code: ', e.code) - else: - # everything is fine - - -info and geturl -=============== - -The response returned by urlopen (or the :exc:`HTTPError` instance) has two -useful methods :meth:`info` and :meth:`geturl` and is defined in the module -:mod:`urllib.response`.. - -**geturl** - this returns the real URL of the page fetched. This is useful -because ``urlopen`` (or the opener object used) may have followed a -redirect. The URL of the page fetched may not be the same as the URL requested. - -**info** - this returns a dictionary-like object that describes the page -fetched, particularly the headers sent by the server. It is currently an -:class:`http.client.HTTPMessage` instance. - -Typical headers include 'Content-length', 'Content-type', and so on. See the -`Quick Reference to HTTP Headers `_ -for a useful listing of HTTP headers with brief explanations of their meaning -and use. - - -Openers and Handlers -==================== - -When you fetch a URL you use an opener (an instance of the perhaps -confusingly-named :class:`urllib.request.OpenerDirector`). Normally we have been using -the default opener - via ``urlopen`` - but you can create custom -openers. Openers use handlers. All the "heavy lifting" is done by the -handlers. Each handler knows how to open URLs for a particular URL scheme (http, -ftp, etc.), or how to handle an aspect of URL opening, for example HTTP -redirections or HTTP cookies. - -You will want to create openers if you want to fetch URLs with specific handlers -installed, for example to get an opener that handles cookies, or to get an -opener that does not handle redirections. - -To create an opener, instantiate an ``OpenerDirector``, and then call -``.add_handler(some_handler_instance)`` repeatedly. - -Alternatively, you can use ``build_opener``, which is a convenience function for -creating opener objects with a single function call. ``build_opener`` adds -several handlers by default, but provides a quick way to add more and/or -override the default handlers. - -Other sorts of handlers you might want to can handle proxies, authentication, -and other common but slightly specialised situations. - -``install_opener`` can be used to make an ``opener`` object the (global) default -opener. This means that calls to ``urlopen`` will use the opener you have -installed. - -Opener objects have an ``open`` method, which can be called directly to fetch -urls in the same way as the ``urlopen`` function: there's no need to call -``install_opener``, except as a convenience. - - -Basic Authentication -==================== - -To illustrate creating and installing a handler we will use the -``HTTPBasicAuthHandler``. For a more detailed discussion of this subject -- -including an explanation of how Basic Authentication works - see the `Basic -Authentication Tutorial -`_. - -When authentication is required, the server sends a header (as well as the 401 -error code) requesting authentication. This specifies the authentication scheme -and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME -realm="REALM"``. - -e.g. - -.. code-block:: none - - WWW-Authenticate: Basic realm="cPanel Users" - - -The client should then retry the request with the appropriate name and password -for the realm included as a header in the request. This is 'basic -authentication'. In order to simplify this process we can create an instance of -``HTTPBasicAuthHandler`` and an opener to use this handler. - -The ``HTTPBasicAuthHandler`` uses an object called a password manager to handle -the mapping of URLs and realms to passwords and usernames. If you know what the -realm is (from the authentication header sent by the server), then you can use a -``HTTPPasswordMgr``. Frequently one doesn't care what the realm is. In that -case, it is convenient to use ``HTTPPasswordMgrWithDefaultRealm``. This allows -you to specify a default username and password for a URL. This will be supplied -in the absence of you providing an alternative combination for a specific -realm. We indicate this by providing ``None`` as the realm argument to the -``add_password`` method. - -The top-level URL is the first URL that requires authentication. URLs "deeper" -than the URL you pass to .add_password() will also match. :: - - # create a password manager - password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() - - # Add the username and password. - # If we knew the realm, we could use it instead of None. - top_level_url = "http://example.com/foo/" - password_mgr.add_password(None, top_level_url, username, password) - - handler = urllib.request.HTTPBasicAuthHandler(password_mgr) - - # create "opener" (OpenerDirector instance) - opener = urllib.request.build_opener(handler) - - # use the opener to fetch a URL - opener.open(a_url) - - # Install the opener. - # Now all calls to urllib.request.urlopen use our opener. - urllib.request.install_opener(opener) - -.. note:: - - In the above example we only supplied our ``HTTPBasicAuthHandler`` to - ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` - environment variable is set), ``UnknownHandler``, ``HTTPHandler``, - ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. - -``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme -component and the hostname and optionally the port number) -e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` -(the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is -not correct. - - -Proxies -======= - -**urllib** will auto-detect your proxy settings and use those. This is through -the ``ProxyHandler``, which is part of the normal handler chain when a proxy -setting is detected. Normally that's a good thing, but there are occasions -when it may not be helpful [#]_. One way to do this is to setup our own -``ProxyHandler``, with no proxies defined. This is done using similar steps to -setting up a `Basic Authentication`_ handler: :: - - >>> proxy_support = urllib.request.ProxyHandler({}) - >>> opener = urllib.request.build_opener(proxy_support) - >>> urllib.request.install_opener(opener) - -.. note:: - - Currently ``urllib.request`` *does not* support fetching of ``https`` locations - through a proxy. However, this can be enabled by extending urllib.request as - shown in the recipe [#]_. - -.. note:: - - ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; see - the documentation on :func:`~urllib.request.getproxies`. - - -Sockets and Layers -================== - -The Python support for fetching resources from the web is layered. urllib uses -the :mod:`http.client` library, which in turn uses the socket library. - -As of Python 2.3 you can specify how long a socket should wait for a response -before timing out. This can be useful in applications which have to fetch web -pages. By default the socket module has *no timeout* and can hang. Currently, -the socket timeout is not exposed at the http.client or urllib.request levels. -However, you can set the default timeout globally for all sockets using :: - - import socket - import urllib.request - - # timeout in seconds - timeout = 10 - socket.setdefaulttimeout(timeout) - - # this call to urllib.request.urlopen now uses the default timeout - # we have set in the socket module - req = urllib.request.Request('http://www.voidspace.org.uk') - response = urllib.request.urlopen(req) - - -------- - - -Footnotes -========= - -This document was reviewed and revised by John Lee. - -.. [#] Google for example. -.. [#] Browser sniffing is a very bad practice for website design - building - sites using web standards is much more sensible. Unfortunately a lot of - sites still send different versions to different browsers. -.. [#] The user agent for MSIE 6 is - *'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'* -.. [#] For details of more HTTP request headers, see - `Quick Reference to HTTP Headers`_. -.. [#] In my case I have to use a proxy to access the internet at work. If you - attempt to fetch *localhost* URLs through this proxy it blocks them. IE - is set to use the proxy, which urllib picks up on. In order to test - scripts with a localhost server, I have to prevent urllib from using - the proxy. -.. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. - diff --git a/third_party/python/Doc/includes/capsulethunk.h b/third_party/python/Doc/includes/capsulethunk.h deleted file mode 100644 index 6b20564f1..000000000 --- a/third_party/python/Doc/includes/capsulethunk.h +++ /dev/null @@ -1,134 +0,0 @@ -#ifndef __CAPSULETHUNK_H -#define __CAPSULETHUNK_H - -#if ( (PY_VERSION_HEX < 0x02070000) \ - || ((PY_VERSION_HEX >= 0x03000000) \ - && (PY_VERSION_HEX < 0x03010000)) ) - -#define __PyCapsule_GetField(capsule, field, default_value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field) \ - : (default_value) \ - ) \ - -#define __PyCapsule_SetField(capsule, field, value) \ - ( PyCapsule_CheckExact(capsule) \ - ? (((PyCObject *)capsule)->field = value), 1 \ - : 0 \ - ) \ - - -#define PyCapsule_Type PyCObject_Type - -#define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -#define PyCapsule_IsValid(capsule, name) (PyCObject_Check(capsule)) - - -#define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, destructor)) - - -#define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) - -/* Don't call PyCObject_SetPointer here, it fails if there's a destructor */ -#define PyCapsule_SetPointer(capsule, pointer) \ - __PyCapsule_SetField(capsule, cobject, pointer) - - -#define PyCapsule_GetDestructor(capsule) \ - __PyCapsule_GetField(capsule, destructor) - -#define PyCapsule_SetDestructor(capsule, dtor) \ - __PyCapsule_SetField(capsule, destructor, dtor) - - -/* - * Sorry, there's simply no place - * to store a Capsule "name" in a CObject. - */ -#define PyCapsule_GetName(capsule) NULL - -static int -PyCapsule_SetName(PyObject *capsule, const char *unused) -{ - unused = unused; - PyErr_SetString(PyExc_NotImplementedError, - "can't use PyCapsule_SetName with CObjects"); - return 1; -} - - - -#define PyCapsule_GetContext(capsule) \ - __PyCapsule_GetField(capsule, descr) - -#define PyCapsule_SetContext(capsule, context) \ - __PyCapsule_SetField(capsule, descr, context) - - -static void * -PyCapsule_Import(const char *name, int no_block) -{ - PyObject *object = NULL; - void *return_value = NULL; - char *trace; - size_t name_length = (strlen(name) + 1) * sizeof(char); - char *name_dup = (char *)PyMem_MALLOC(name_length); - - if (!name_dup) { - return NULL; - } - - memcpy(name_dup, name, name_length); - - trace = name_dup; - while (trace) { - char *dot = strchr(trace, '.'); - if (dot) { - *dot++ = '\0'; - } - - if (object == NULL) { - if (no_block) { - object = PyImport_ImportModuleNoBlock(trace); - } else { - object = PyImport_ImportModule(trace); - if (!object) { - PyErr_Format(PyExc_ImportError, - "PyCapsule_Import could not " - "import module \"%s\"", trace); - } - } - } else { - PyObject *object2 = PyObject_GetAttrString(object, trace); - Py_DECREF(object); - object = object2; - } - if (!object) { - goto EXIT; - } - - trace = dot; - } - - if (PyCObject_Check(object)) { - PyCObject *cobject = (PyCObject *)object; - return_value = cobject->cobject; - } else { - PyErr_Format(PyExc_AttributeError, - "PyCapsule_Import \"%s\" is not valid", - name); - } - -EXIT: - Py_XDECREF(object); - if (name_dup) { - PyMem_FREE(name_dup); - } - return return_value; -} - -#endif /* #if PY_VERSION_HEX < 0x02070000 */ - -#endif /* __CAPSULETHUNK_H */ diff --git a/third_party/python/Doc/includes/custom.c b/third_party/python/Doc/includes/custom.c deleted file mode 100644 index fb2c7b2a4..000000000 --- a/third_party/python/Doc/includes/custom.c +++ /dev/null @@ -1,39 +0,0 @@ -#include - -typedef struct { - PyObject_HEAD - /* Type-specific fields go here. */ -} CustomObject; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom.Custom", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); - return m; -} diff --git a/third_party/python/Doc/includes/custom2.c b/third_party/python/Doc/includes/custom2.c deleted file mode 100644 index 51ab4b80d..000000000 --- a/third_party/python/Doc/includes/custom2.c +++ /dev/null @@ -1,132 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static void -Custom_dealloc(CustomObject *self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, - "last name"}, - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom2.Custom", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_members = Custom_members, - .tp_methods = Custom_methods, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom2", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom2(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); - return m; -} diff --git a/third_party/python/Doc/includes/custom3.c b/third_party/python/Doc/includes/custom3.c deleted file mode 100644 index 09e87355b..000000000 --- a/third_party/python/Doc/includes/custom3.c +++ /dev/null @@ -1,183 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static void -Custom_dealloc(CustomObject *self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_getfirst(CustomObject *self, void *closure) -{ - Py_INCREF(self->first); - return self->first; -} - -static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) -{ - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - tmp = self->first; - Py_INCREF(value); - self->first = value; - Py_DECREF(tmp); - return 0; -} - -static PyObject * -Custom_getlast(CustomObject *self, void *closure) -{ - Py_INCREF(self->last); - return self->last; -} - -static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) -{ - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The last attribute value must be a string"); - return -1; - } - tmp = self->last; - Py_INCREF(value); - self->last = value; - Py_DECREF(tmp); - return 0; -} - -static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom3.Custom", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_members = Custom_members, - .tp_methods = Custom_methods, - .tp_getset = Custom_getsetters, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom3", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom3(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); - return m; -} diff --git a/third_party/python/Doc/includes/custom4.c b/third_party/python/Doc/includes/custom4.c deleted file mode 100644 index 0994d8fda..000000000 --- a/third_party/python/Doc/includes/custom4.c +++ /dev/null @@ -1,197 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static int -Custom_traverse(CustomObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->first); - Py_VISIT(self->last); - return 0; -} - -static int -Custom_clear(CustomObject *self) -{ - Py_CLEAR(self->first); - Py_CLEAR(self->last); - return 0; -} - -static void -Custom_dealloc(CustomObject *self) -{ - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_getfirst(CustomObject *self, void *closure) -{ - Py_INCREF(self->first); - return self->first; -} - -static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - Py_INCREF(value); - Py_CLEAR(self->first); - self->first = value; - return 0; -} - -static PyObject * -Custom_getlast(CustomObject *self, void *closure) -{ - Py_INCREF(self->last); - return self->last; -} - -static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The last attribute value must be a string"); - return -1; - } - Py_INCREF(value); - Py_CLEAR(self->last); - self->last = value; - return 0; -} - -static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom4.Custom", - .tp_doc = "Custom objects", - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_traverse = (traverseproc) Custom_traverse, - .tp_clear = (inquiry) Custom_clear, - .tp_members = Custom_members, - .tp_methods = Custom_methods, - .tp_getset = Custom_getsetters, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom4", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom4(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); - return m; -} diff --git a/third_party/python/Doc/includes/dbpickle.py b/third_party/python/Doc/includes/dbpickle.py deleted file mode 100644 index b88ee87d8..000000000 --- a/third_party/python/Doc/includes/dbpickle.py +++ /dev/null @@ -1,87 +0,0 @@ -# Simple example presenting how persistent ID can be used to pickle -# external objects by reference. - -import pickle -import sqlite3 -from collections import namedtuple - -# Simple class representing a record in our database. -MemoRecord = namedtuple("MemoRecord", "key, task") - -class DBPickler(pickle.Pickler): - - def persistent_id(self, obj): - # Instead of pickling MemoRecord as a regular class instance, we emit a - # persistent ID. - if isinstance(obj, MemoRecord): - # Here, our persistent ID is simply a tuple, containing a tag and a - # key, which refers to a specific record in the database. - return ("MemoRecord", obj.key) - else: - # If obj does not have a persistent ID, return None. This means obj - # needs to be pickled as usual. - return None - - -class DBUnpickler(pickle.Unpickler): - - def __init__(self, file, connection): - super().__init__(file) - self.connection = connection - - def persistent_load(self, pid): - # This method is invoked whenever a persistent ID is encountered. - # Here, pid is the tuple returned by DBPickler. - cursor = self.connection.cursor() - type_tag, key_id = pid - if type_tag == "MemoRecord": - # Fetch the referenced record from the database and return it. - cursor.execute("SELECT * FROM memos WHERE key=?", (str(key_id),)) - key, task = cursor.fetchone() - return MemoRecord(key, task) - else: - # Always raises an error if you cannot return the correct object. - # Otherwise, the unpickler will think None is the object referenced - # by the persistent ID. - raise pickle.UnpicklingError("unsupported persistent object") - - -def main(): - import io - import pprint - - # Initialize and populate our database. - conn = sqlite3.connect(":memory:") - cursor = conn.cursor() - cursor.execute("CREATE TABLE memos(key INTEGER PRIMARY KEY, task TEXT)") - tasks = ( - 'give food to fish', - 'prepare group meeting', - 'fight with a zebra', - ) - for task in tasks: - cursor.execute("INSERT INTO memos VALUES(NULL, ?)", (task,)) - - # Fetch the records to be pickled. - cursor.execute("SELECT * FROM memos") - memos = [MemoRecord(key, task) for key, task in cursor] - # Save the records using our custom DBPickler. - file = io.BytesIO() - DBPickler(file).dump(memos) - - print("Pickled records:") - pprint.pprint(memos) - - # Update a record, just for good measure. - cursor.execute("UPDATE memos SET task='learn italian' WHERE key=1") - - # Load the records from the pickle data stream. - file.seek(0) - memos = DBUnpickler(file, conn).load() - - print("Unpickled records:") - pprint.pprint(memos) - - -if __name__ == '__main__': - main() diff --git a/third_party/python/Doc/includes/email-alternative.py b/third_party/python/Doc/includes/email-alternative.py deleted file mode 100644 index df7ca6f3f..000000000 --- a/third_party/python/Doc/includes/email-alternative.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -import smtplib - -from email.message import EmailMessage -from email.headerregistry import Address -from email.utils import make_msgid - -# Create the base text message. -msg = EmailMessage() -msg['Subject'] = "Ayons asperges pour le déjeuner" -msg['From'] = Address("Pepé Le Pew", "pepe", "example.com") -msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"), - Address("Fabrette Pussycat", "fabrette", "example.com")) -msg.set_content("""\ -Salut! - -Cela ressemble à un excellent recipie[1] déjeuner. - -[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 - ---Pepé -""") - -# Add the html version. This converts the message into a multipart/alternative -# container, with the original text message as the first part and the new html -# message as the second part. -asparagus_cid = make_msgid() -msg.add_alternative("""\ - - - -

Salut!

-

Cela ressemble à un excellent - - recipie - déjeuner. -

- - - -""".format(asparagus_cid=asparagus_cid[1:-1]), subtype='html') -# note that we needed to peel the <> off the msgid for use in the html. - -# Now add the related image to the html part. -with open("roasted-asparagus.jpg", 'rb') as img: - msg.get_payload()[1].add_related(img.read(), 'image', 'jpeg', - cid=asparagus_cid) - -# Make a local copy of what we are going to send. -with open('outgoing.msg', 'wb') as f: - f.write(bytes(msg)) - -# Send the message via local SMTP server. -with smtplib.SMTP('localhost') as s: - s.send_message(msg) diff --git a/third_party/python/Doc/includes/email-dir.py b/third_party/python/Doc/includes/email-dir.py deleted file mode 100644 index 0dcfbfb40..000000000 --- a/third_party/python/Doc/includes/email-dir.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -"""Send the contents of a directory as a MIME message.""" - -import os -import smtplib -# For guessing MIME type based on file name extension -import mimetypes - -from argparse import ArgumentParser - -from email.message import EmailMessage -from email.policy import SMTP - - -def main(): - parser = ArgumentParser(description="""\ -Send the contents of a directory as a MIME message. -Unless the -o option is given, the email is sent by forwarding to your local -SMTP server, which then does the normal delivery process. Your local machine -must be running an SMTP server. -""") - parser.add_argument('-d', '--directory', - help="""Mail the contents of the specified directory, - otherwise use the current directory. Only the regular - files in the directory are sent, and we don't recurse to - subdirectories.""") - parser.add_argument('-o', '--output', - metavar='FILE', - help="""Print the composed message to FILE instead of - sending the message to the SMTP server.""") - parser.add_argument('-s', '--sender', required=True, - help='The value of the From: header (required)') - parser.add_argument('-r', '--recipient', required=True, - action='append', metavar='RECIPIENT', - default=[], dest='recipients', - help='A To: header value (at least one required)') - args = parser.parse_args() - directory = args.directory - if not directory: - directory = '.' - # Create the message - msg = EmailMessage() - msg['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) - msg['To'] = ', '.join(args.recipients) - msg['From'] = args.sender - msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' - - for filename in os.listdir(directory): - path = os.path.join(directory, filename) - if not os.path.isfile(path): - continue - # Guess the content type based on the file's extension. Encoding - # will be ignored, although we should check for simple things like - # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) - if ctype is None or encoding is not None: - # No guess could be made, or the file is encoded (compressed), so - # use a generic bag-of-bits type. - ctype = 'application/octet-stream' - maintype, subtype = ctype.split('/', 1) - with open(path, 'rb') as fp: - msg.add_attachment(fp.read(), - maintype=maintype, - subtype=subtype, - filename=filename) - # Now send or store the message - if args.output: - with open(args.output, 'wb') as fp: - fp.write(msg.as_bytes(policy=SMTP)) - else: - with smtplib.SMTP('localhost') as s: - s.send_message(msg) - - -if __name__ == '__main__': - main() diff --git a/third_party/python/Doc/includes/email-headers.py b/third_party/python/Doc/includes/email-headers.py deleted file mode 100644 index 2c421451a..000000000 --- a/third_party/python/Doc/includes/email-headers.py +++ /dev/null @@ -1,24 +0,0 @@ -# Import the email modules we'll need -from email.parser import BytesParser, Parser -from email.policy import default - -# If the e-mail headers are in a file, uncomment these two lines: -# with open(messagefile, 'rb') as fp: -# headers = BytesParser(policy=default).parse(fp) - -# Or for parsing headers in a string (this is an uncommon operation), use: -headers = Parser(policy=default).parsestr( - 'From: Foo Bar \n' - 'To: \n' - 'Subject: Test message\n' - '\n' - 'Body would go here\n') - -# Now the header items can be accessed as a dictionary: -print('To: {}'.format(headers['to'])) -print('From: {}'.format(headers['from'])) -print('Subject: {}'.format(headers['subject'])) - -# You can also access the parts of the addresses: -print('Recipient username: {}'.format(headers['to'].addresses[0].username)) -print('Sender name: {}'.format(headers['from'].addresses[0].display_name)) diff --git a/third_party/python/Doc/includes/email-mime.py b/third_party/python/Doc/includes/email-mime.py deleted file mode 100644 index c610242f1..000000000 --- a/third_party/python/Doc/includes/email-mime.py +++ /dev/null @@ -1,29 +0,0 @@ -# Import smtplib for the actual sending function -import smtplib - -# And imghdr to find the types of our images -import imghdr - -# Here are the email package modules we'll need -from email.message import EmailMessage - -# Create the container email message. -msg = EmailMessage() -msg['Subject'] = 'Our family reunion' -# me == the sender's email address -# family = the list of all recipients' email addresses -msg['From'] = me -msg['To'] = ', '.join(family) -msg.preamble = 'Our family reunion' - -# Open the files in binary mode. Use imghdr to figure out the -# MIME subtype for each specific image. -for file in pngfiles: - with open(file, 'rb') as fp: - img_data = fp.read() - msg.add_attachment(img_data, maintype='image', - subtype=imghdr.what(None, img_data)) - -# Send the email via our own SMTP server. -with smtplib.SMTP('localhost') as s: - s.send_message(msg) diff --git a/third_party/python/Doc/includes/email-read-alternative.py b/third_party/python/Doc/includes/email-read-alternative.py deleted file mode 100644 index 5ea84e625..000000000 --- a/third_party/python/Doc/includes/email-read-alternative.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import sys -import tempfile -import mimetypes -import webbrowser - -# Import the email modules we'll need -from email import policy -from email.parser import BytesParser - -# An imaginary module that would make this work and be safe. -from imaginary import magic_html_parser - -# In a real program you'd get the filename from the arguments. -with open('outgoing.msg', 'rb') as fp: - msg = BytesParser(policy=policy.default).parse(fp) - -# Now the header items can be accessed as a dictionary, and any non-ASCII will -# be converted to unicode: -print('To:', msg['to']) -print('From:', msg['from']) -print('Subject:', msg['subject']) - -# If we want to print a preview of the message content, we can extract whatever -# the least formatted payload is and print the first three lines. Of course, -# if the message has no plain text part printing the first three lines of html -# is probably useless, but this is just a conceptual example. -simplest = msg.get_body(preferencelist=('plain', 'html')) -print() -print(''.join(simplest.get_content().splitlines(keepends=True)[:3])) - -ans = input("View full message?") -if ans.lower()[0] == 'n': - sys.exit() - -# We can extract the richest alternative in order to display it: -richest = msg.get_body() -partfiles = {} -if richest['content-type'].maintype == 'text': - if richest['content-type'].subtype == 'plain': - for line in richest.get_content().splitlines(): - print(line) - sys.exit() - elif richest['content-type'].subtype == 'html': - body = richest - else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -elif richest['content-type'].content_type == 'multipart/related': - body = richest.get_body(preferencelist=('html')) - for part in richest.iter_attachments(): - fn = part.get_filename() - if fn: - extension = os.path.splitext(part.get_filename())[1] - else: - extension = mimetypes.guess_extension(part.get_content_type()) - with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as f: - f.write(part.get_content()) - # again strip the <> to go from email form of cid to html form. - partfiles[part['content-id'][1:-1]] = f.name -else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - # The magic_html_parser has to rewrite the href="cid:...." attributes to - # point to the filenames in partfiles. It also has to do a safety-sanitize - # of the html. It could be written using html.parser. - f.write(magic_html_parser(body.get_content(), partfiles)) -webbrowser.open(f.name) -os.remove(f.name) -for fn in partfiles.values(): - os.remove(fn) - -# Of course, there are lots of email messages that could break this simple -# minded program, but it will handle the most common ones. diff --git a/third_party/python/Doc/includes/email-simple.py b/third_party/python/Doc/includes/email-simple.py deleted file mode 100644 index f69ef40ff..000000000 --- a/third_party/python/Doc/includes/email-simple.py +++ /dev/null @@ -1,22 +0,0 @@ -# Import smtplib for the actual sending function -import smtplib - -# Import the email modules we'll need -from email.message import EmailMessage - -# Open the plain text file whose name is in textfile for reading. -with open(textfile) as fp: - # Create a text/plain message - msg = EmailMessage() - msg.set_content(fp.read()) - -# me == the sender's email address -# you == the recipient's email address -msg['Subject'] = 'The contents of %s' % textfile -msg['From'] = me -msg['To'] = you - -# Send the message via our own SMTP server. -s = smtplib.SMTP('localhost') -s.send_message(msg) -s.quit() diff --git a/third_party/python/Doc/includes/email-unpack.py b/third_party/python/Doc/includes/email-unpack.py deleted file mode 100644 index e0a7f01f5..000000000 --- a/third_party/python/Doc/includes/email-unpack.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -"""Unpack a MIME message into a directory of files.""" - -import os -import email -import mimetypes - -from email.policy import default - -from argparse import ArgumentParser - - -def main(): - parser = ArgumentParser(description="""\ -Unpack a MIME message into a directory of files. -""") - parser.add_argument('-d', '--directory', required=True, - help="""Unpack the MIME message into the named - directory, which will be created if it doesn't already - exist.""") - parser.add_argument('msgfile') - args = parser.parse_args() - - with open(args.msgfile, 'rb') as fp: - msg = email.message_from_binary_file(fp, policy=default) - - try: - os.mkdir(args.directory) - except FileExistsError: - pass - - counter = 1 - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - # Applications should really sanitize the given filename so that an - # email message can't be used to overwrite important files - filename = part.get_filename() - if not filename: - ext = mimetypes.guess_extension(part.get_content_type()) - if not ext: - # Use a generic bag-of-bits extension - ext = '.bin' - filename = 'part-%03d%s' % (counter, ext) - counter += 1 - with open(os.path.join(args.directory, filename), 'wb') as fp: - fp.write(part.get_payload(decode=True)) - - -if __name__ == '__main__': - main() diff --git a/third_party/python/Doc/includes/minidom-example.py b/third_party/python/Doc/includes/minidom-example.py deleted file mode 100644 index 5ee7682c1..000000000 --- a/third_party/python/Doc/includes/minidom-example.py +++ /dev/null @@ -1,64 +0,0 @@ -import xml.dom.minidom - -document = """\ - -Demo slideshow -Slide title -This is a demo -Of a program for processing slides - - -Another demo slide -It is important -To have more than -one slide - - -""" - -dom = xml.dom.minidom.parseString(document) - -def getText(nodelist): - rc = [] - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return ''.join(rc) - -def handleSlideshow(slideshow): - print("") - handleSlideshowTitle(slideshow.getElementsByTagName("title")[0]) - slides = slideshow.getElementsByTagName("slide") - handleToc(slides) - handleSlides(slides) - print("") - -def handleSlides(slides): - for slide in slides: - handleSlide(slide) - -def handleSlide(slide): - handleSlideTitle(slide.getElementsByTagName("title")[0]) - handlePoints(slide.getElementsByTagName("point")) - -def handleSlideshowTitle(title): - print("%s" % getText(title.childNodes)) - -def handleSlideTitle(title): - print("

%s

" % getText(title.childNodes)) - -def handlePoints(points): - print("
    ") - for point in points: - handlePoint(point) - print("
") - -def handlePoint(point): - print("
  • %s
    • " % getText(point.childNodes)) - -def handleToc(slides): - for slide in slides: - title = slide.getElementsByTagName("title")[0] - print("

    %s

    " % getText(title.childNodes)) - -handleSlideshow(dom) diff --git a/third_party/python/Doc/includes/mp_newtype.py b/third_party/python/Doc/includes/mp_newtype.py deleted file mode 100644 index 1c796a564..000000000 --- a/third_party/python/Doc/includes/mp_newtype.py +++ /dev/null @@ -1,89 +0,0 @@ -from multiprocessing import freeze_support -from multiprocessing.managers import BaseManager, BaseProxy -import operator - -## - -class Foo: - def f(self): - print('you called Foo.f()') - def g(self): - print('you called Foo.g()') - def _h(self): - print('you called Foo._h()') - -# A simple generator function -def baz(): - for i in range(10): - yield i*i - -# Proxy type for generator objects -class GeneratorProxy(BaseProxy): - _exposed_ = ['__next__'] - def __iter__(self): - return self - def __next__(self): - return self._callmethod('__next__') - -# Function to return the operator module -def get_operator_module(): - return operator - -## - -class MyManager(BaseManager): - pass - -# register the Foo class; make `f()` and `g()` accessible via proxy -MyManager.register('Foo1', Foo) - -# register the Foo class; make `g()` and `_h()` accessible via proxy -MyManager.register('Foo2', Foo, exposed=('g', '_h')) - -# register the generator function baz; use `GeneratorProxy` to make proxies -MyManager.register('baz', baz, proxytype=GeneratorProxy) - -# register get_operator_module(); make public functions accessible via proxy -MyManager.register('operator', get_operator_module) - -## - -def test(): - manager = MyManager() - manager.start() - - print('-' * 20) - - f1 = manager.Foo1() - f1.f() - f1.g() - assert not hasattr(f1, '_h') - assert sorted(f1._exposed_) == sorted(['f', 'g']) - - print('-' * 20) - - f2 = manager.Foo2() - f2.g() - f2._h() - assert not hasattr(f2, 'f') - assert sorted(f2._exposed_) == sorted(['g', '_h']) - - print('-' * 20) - - it = manager.baz() - for i in it: - print('<%d>' % i, end=' ') - print() - - print('-' * 20) - - op = manager.operator() - print('op.add(23, 45) =', op.add(23, 45)) - print('op.pow(2, 94) =', op.pow(2, 94)) - print('op._exposed_ =', op._exposed_) - -## - -if __name__ == '__main__': - freeze_support() - test() diff --git a/third_party/python/Doc/includes/mp_pool.py b/third_party/python/Doc/includes/mp_pool.py deleted file mode 100644 index 11101e1a9..000000000 --- a/third_party/python/Doc/includes/mp_pool.py +++ /dev/null @@ -1,153 +0,0 @@ -import multiprocessing -import time -import random -import sys - -# -# Functions used by test code -# - -def calculate(func, args): - result = func(*args) - return '%s says that %s%s = %s' % ( - multiprocessing.current_process().name, - func.__name__, args, result - ) - -def calculatestar(args): - return calculate(*args) - -def mul(a, b): - time.sleep(0.5 * random.random()) - return a * b - -def plus(a, b): - time.sleep(0.5 * random.random()) - return a + b - -def f(x): - return 1.0 / (x - 5.0) - -def pow3(x): - return x ** 3 - -def noop(x): - pass - -# -# Test code -# - -def test(): - PROCESSES = 4 - print('Creating pool with %d processes\n' % PROCESSES) - - with multiprocessing.Pool(PROCESSES) as pool: - # - # Tests - # - - TASKS = [(mul, (i, 7)) for i in range(10)] + \ - [(plus, (i, 8)) for i in range(10)] - - results = [pool.apply_async(calculate, t) for t in TASKS] - imap_it = pool.imap(calculatestar, TASKS) - imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) - - print('Ordered results using pool.apply_async():') - for r in results: - print('\t', r.get()) - print() - - print('Ordered results using pool.imap():') - for x in imap_it: - print('\t', x) - print() - - print('Unordered results using pool.imap_unordered():') - for x in imap_unordered_it: - print('\t', x) - print() - - print('Ordered results using pool.map() --- will block till complete:') - for x in pool.map(calculatestar, TASKS): - print('\t', x) - print() - - # - # Test error handling - # - - print('Testing error handling:') - - try: - print(pool.apply(f, (5,))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.apply()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(pool.map(f, list(range(10)))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.map()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(list(pool.imap(f, list(range(10))))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from list(pool.imap())') - else: - raise AssertionError('expected ZeroDivisionError') - - it = pool.imap(f, list(range(10))) - for i in range(10): - try: - x = next(it) - except ZeroDivisionError: - if i == 5: - pass - except StopIteration: - break - else: - if i == 5: - raise AssertionError('expected ZeroDivisionError') - - assert i == 9 - print('\tGot ZeroDivisionError as expected from IMapIterator.next()') - print() - - # - # Testing timeouts - # - - print('Testing ApplyResult.get() with timeout:', end=' ') - res = pool.apply_async(calculate, TASKS[0]) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % res.get(0.02)) - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() - - print('Testing IMapIterator.next() with timeout:', end=' ') - it = pool.imap(calculatestar, TASKS) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % it.next(0.02)) - except StopIteration: - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() - - -if __name__ == '__main__': - multiprocessing.freeze_support() - test() diff --git a/third_party/python/Doc/includes/mp_workers.py b/third_party/python/Doc/includes/mp_workers.py deleted file mode 100644 index 3b92269f8..000000000 --- a/third_party/python/Doc/includes/mp_workers.py +++ /dev/null @@ -1,77 +0,0 @@ -import time -import random - -from multiprocessing import Process, Queue, current_process, freeze_support - -# -# Function run by worker processes -# - -def worker(input, output): - for func, args in iter(input.get, 'STOP'): - result = calculate(func, args) - output.put(result) - -# -# Function used to calculate result -# - -def calculate(func, args): - result = func(*args) - return '%s says that %s%s = %s' % \ - (current_process().name, func.__name__, args, result) - -# -# Functions referenced by tasks -# - -def mul(a, b): - time.sleep(0.5*random.random()) - return a * b - -def plus(a, b): - time.sleep(0.5*random.random()) - return a + b - -# -# -# - -def test(): - NUMBER_OF_PROCESSES = 4 - TASKS1 = [(mul, (i, 7)) for i in range(20)] - TASKS2 = [(plus, (i, 8)) for i in range(10)] - - # Create queues - task_queue = Queue() - done_queue = Queue() - - # Submit tasks - for task in TASKS1: - task_queue.put(task) - - # Start worker processes - for i in range(NUMBER_OF_PROCESSES): - Process(target=worker, args=(task_queue, done_queue)).start() - - # Get and print results - print('Unordered results:') - for i in range(len(TASKS1)): - print('\t', done_queue.get()) - - # Add more tasks using `put()` - for task in TASKS2: - task_queue.put(task) - - # Get and print some more results - for i in range(len(TASKS2)): - print('\t', done_queue.get()) - - # Tell child processes to stop - for i in range(NUMBER_OF_PROCESSES): - task_queue.put('STOP') - - -if __name__ == '__main__': - freeze_support() - test() diff --git a/third_party/python/Doc/includes/run-func.c b/third_party/python/Doc/includes/run-func.c deleted file mode 100644 index 9caf1fdb2..000000000 --- a/third_party/python/Doc/includes/run-func.c +++ /dev/null @@ -1,70 +0,0 @@ -#include - -int -main(int argc, char *argv[]) -{ - PyObject *pName, *pModule, *pFunc; - PyObject *pArgs, *pValue; - int i; - - if (argc < 3) { - fprintf(stderr,"Usage: call pythonfile funcname [args]\n"); - return 1; - } - - Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); - /* Error checking of pName left out */ - - pModule = PyImport_Import(pName); - Py_DECREF(pName); - - if (pModule != NULL) { - pFunc = PyObject_GetAttrString(pModule, argv[2]); - /* pFunc is a new reference */ - - if (pFunc && PyCallable_Check(pFunc)) { - pArgs = PyTuple_New(argc - 3); - for (i = 0; i < argc - 3; ++i) { - pValue = PyLong_FromLong(atoi(argv[i + 3])); - if (!pValue) { - Py_DECREF(pArgs); - Py_DECREF(pModule); - fprintf(stderr, "Cannot convert argument\n"); - return 1; - } - /* pValue reference stolen here: */ - PyTuple_SetItem(pArgs, i, pValue); - } - pValue = PyObject_CallObject(pFunc, pArgs); - Py_DECREF(pArgs); - if (pValue != NULL) { - printf("Result of call: %ld\n", PyLong_AsLong(pValue)); - Py_DECREF(pValue); - } - else { - Py_DECREF(pFunc); - Py_DECREF(pModule); - PyErr_Print(); - fprintf(stderr,"Call failed\n"); - return 1; - } - } - else { - if (PyErr_Occurred()) - PyErr_Print(); - fprintf(stderr, "Cannot find function \"%s\"\n", argv[2]); - } - Py_XDECREF(pFunc); - Py_DECREF(pModule); - } - else { - PyErr_Print(); - fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); - return 1; - } - if (Py_FinalizeEx() < 0) { - return 120; - } - return 0; -} diff --git a/third_party/python/Doc/includes/setup.py b/third_party/python/Doc/includes/setup.py deleted file mode 100644 index a38a39de3..000000000 --- a/third_party/python/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) diff --git a/third_party/python/Doc/includes/sqlite3/adapter_datetime.py b/third_party/python/Doc/includes/sqlite3/adapter_datetime.py deleted file mode 100644 index be3339510..000000000 --- a/third_party/python/Doc/includes/sqlite3/adapter_datetime.py +++ /dev/null @@ -1,15 +0,0 @@ -import sqlite3 -import datetime -import time - -def adapt_datetime(ts): - return time.mktime(ts.timetuple()) - -sqlite3.register_adapter(datetime.datetime, adapt_datetime) - -con = sqlite3.connect(":memory:") -cur = con.cursor() - -now = datetime.datetime.now() -cur.execute("select ?", (now,)) -print(cur.fetchone()[0]) diff --git a/third_party/python/Doc/includes/sqlite3/adapter_point_1.py b/third_party/python/Doc/includes/sqlite3/adapter_point_1.py deleted file mode 100644 index 6b1af8415..000000000 --- a/third_party/python/Doc/includes/sqlite3/adapter_point_1.py +++ /dev/null @@ -1,16 +0,0 @@ -import sqlite3 - -class Point: - def __init__(self, x, y): - self.x, self.y = x, y - - def __conform__(self, protocol): - if protocol is sqlite3.PrepareProtocol: - return "%f;%f" % (self.x, self.y) - -con = sqlite3.connect(":memory:") -cur = con.cursor() - -p = Point(4.0, -3.2) -cur.execute("select ?", (p,)) -print(cur.fetchone()[0]) diff --git a/third_party/python/Doc/includes/sqlite3/adapter_point_2.py b/third_party/python/Doc/includes/sqlite3/adapter_point_2.py deleted file mode 100644 index d670700f0..000000000 --- a/third_party/python/Doc/includes/sqlite3/adapter_point_2.py +++ /dev/null @@ -1,17 +0,0 @@ -import sqlite3 - -class Point: - def __init__(self, x, y): - self.x, self.y = x, y - -def adapt_point(point): - return "%f;%f" % (point.x, point.y) - -sqlite3.register_adapter(Point, adapt_point) - -con = sqlite3.connect(":memory:") -cur = con.cursor() - -p = Point(4.0, -3.2) -cur.execute("select ?", (p,)) -print(cur.fetchone()[0]) diff --git a/third_party/python/Doc/includes/sqlite3/collation_reverse.py b/third_party/python/Doc/includes/sqlite3/collation_reverse.py deleted file mode 100644 index 3504a350a..000000000 --- a/third_party/python/Doc/includes/sqlite3/collation_reverse.py +++ /dev/null @@ -1,20 +0,0 @@ -import sqlite3 - -def collate_reverse(string1, string2): - if string1 == string2: - return 0 - elif string1 < string2: - return 1 - else: - return -1 - -con = sqlite3.connect(":memory:") -con.create_collation("reverse", collate_reverse) - -cur = con.cursor() -cur.execute("create table test(x)") -cur.executemany("insert into test(x) values (?)", [("a",), ("b",)]) -cur.execute("select x from test order by x collate reverse") -for row in cur: - print(row) -con.close() diff --git a/third_party/python/Doc/includes/sqlite3/complete_statement.py b/third_party/python/Doc/includes/sqlite3/complete_statement.py deleted file mode 100644 index cd38d7305..000000000 --- a/third_party/python/Doc/includes/sqlite3/complete_statement.py +++ /dev/null @@ -1,30 +0,0 @@ -# A minimal SQLite shell for experiments - -import sqlite3 - -con = sqlite3.connect(":memory:") -con.isolation_level = None -cur = con.cursor() - -buffer = "" - -print("Enter your SQL commands to execute in sqlite3.") -print("Enter a blank line to exit.") - -while True: - line = input() - if line == "": - break - buffer += line - if sqlite3.complete_statement(buffer): - try: - buffer = buffer.strip() - cur.execute(buffer) - - if buffer.lstrip().upper().startswith("SELECT"): - print(cur.fetchall()) - except sqlite3.Error as e: - print("An error occurred:", e.args[0]) - buffer = "" - -con.close() diff --git a/third_party/python/Doc/includes/sqlite3/connect_db_1.py b/third_party/python/Doc/includes/sqlite3/connect_db_1.py deleted file mode 100644 index 1b9752328..000000000 --- a/third_party/python/Doc/includes/sqlite3/connect_db_1.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") diff --git a/third_party/python/Doc/includes/sqlite3/connect_db_2.py b/third_party/python/Doc/includes/sqlite3/connect_db_2.py deleted file mode 100644 index f9728b361..000000000 --- a/third_party/python/Doc/includes/sqlite3/connect_db_2.py +++ /dev/null @@ -1,3 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") diff --git a/third_party/python/Doc/includes/sqlite3/converter_point.py b/third_party/python/Doc/includes/sqlite3/converter_point.py deleted file mode 100644 index 5df828e33..000000000 --- a/third_party/python/Doc/includes/sqlite3/converter_point.py +++ /dev/null @@ -1,47 +0,0 @@ -import sqlite3 - -class Point: - def __init__(self, x, y): - self.x, self.y = x, y - - def __repr__(self): - return "(%f;%f)" % (self.x, self.y) - -def adapt_point(point): - return ("%f;%f" % (point.x, point.y)).encode('ascii') - -def convert_point(s): - x, y = list(map(float, s.split(b";"))) - return Point(x, y) - -# Register the adapter -sqlite3.register_adapter(Point, adapt_point) - -# Register the converter -sqlite3.register_converter("point", convert_point) - -p = Point(4.0, -3.2) - -######################### -# 1) Using declared types -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES) -cur = con.cursor() -cur.execute("create table test(p point)") - -cur.execute("insert into test(p) values (?)", (p,)) -cur.execute("select p from test") -print("with declared types:", cur.fetchone()[0]) -cur.close() -con.close() - -####################### -# 1) Using column names -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute("create table test(p)") - -cur.execute("insert into test(p) values (?)", (p,)) -cur.execute('select p as "p [point]" from test') -print("with column names:", cur.fetchone()[0]) -cur.close() -con.close() diff --git a/third_party/python/Doc/includes/sqlite3/countcursors.py b/third_party/python/Doc/includes/sqlite3/countcursors.py deleted file mode 100644 index ef3e70a2a..000000000 --- a/third_party/python/Doc/includes/sqlite3/countcursors.py +++ /dev/null @@ -1,15 +0,0 @@ -import sqlite3 - -class CountCursorsConnection(sqlite3.Connection): - def __init__(self, *args, **kwargs): - sqlite3.Connection.__init__(self, *args, **kwargs) - self.numcursors = 0 - - def cursor(self, *args, **kwargs): - self.numcursors += 1 - return sqlite3.Connection.cursor(self, *args, **kwargs) - -con = sqlite3.connect(":memory:", factory=CountCursorsConnection) -cur1 = con.cursor() -cur2 = con.cursor() -print(con.numcursors) diff --git a/third_party/python/Doc/includes/sqlite3/createdb.py b/third_party/python/Doc/includes/sqlite3/createdb.py deleted file mode 100644 index ee2950bdf..000000000 --- a/third_party/python/Doc/includes/sqlite3/createdb.py +++ /dev/null @@ -1,28 +0,0 @@ -# Not referenced from the documentation, but builds the database file the other -# code snippets expect. - -import sqlite3 -import os - -DB_FILE = "mydb" - -if os.path.exists(DB_FILE): - os.remove(DB_FILE) - -con = sqlite3.connect(DB_FILE) -cur = con.cursor() -cur.execute(""" - create table people - ( - name_last varchar(20), - age integer - ) - """) - -cur.execute("insert into people (name_last, age) values ('Yeltsin', 72)") -cur.execute("insert into people (name_last, age) values ('Putin', 51)") - -con.commit() - -cur.close() -con.close() diff --git a/third_party/python/Doc/includes/sqlite3/ctx_manager.py b/third_party/python/Doc/includes/sqlite3/ctx_manager.py deleted file mode 100644 index 7af4ad1ec..000000000 --- a/third_party/python/Doc/includes/sqlite3/ctx_manager.py +++ /dev/null @@ -1,16 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") -con.execute("create table person (id integer primary key, firstname varchar unique)") - -# Successful, con.commit() is called automatically afterwards -with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) - -# con.rollback() is called after the with block finishes with an exception, the -# exception is still raised and must be caught -try: - with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) -except sqlite3.IntegrityError: - print("couldn't add Joe twice") diff --git a/third_party/python/Doc/includes/sqlite3/execsql_fetchonerow.py b/third_party/python/Doc/includes/sqlite3/execsql_fetchonerow.py deleted file mode 100644 index 078873bfc..000000000 --- a/third_party/python/Doc/includes/sqlite3/execsql_fetchonerow.py +++ /dev/null @@ -1,17 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() -SELECT = "select name_last, age from people order by age, name_last" - -# 1. Iterate over the rows available from the cursor, unpacking the -# resulting sequences to yield their elements (name_last, age): -cur.execute(SELECT) -for (name_last, age) in cur: - print('%s is %d years old.' % (name_last, age)) - -# 2. Equivalently: -cur.execute(SELECT) -for row in cur: - print('%s is %d years old.' % (row[0], row[1])) diff --git a/third_party/python/Doc/includes/sqlite3/execsql_printall_1.py b/third_party/python/Doc/includes/sqlite3/execsql_printall_1.py deleted file mode 100644 index a4ce5c528..000000000 --- a/third_party/python/Doc/includes/sqlite3/execsql_printall_1.py +++ /dev/null @@ -1,13 +0,0 @@ -import sqlite3 - -# Create a connection to the database file "mydb": -con = sqlite3.connect("mydb") - -# Get a Cursor object that operates in the context of Connection con: -cur = con.cursor() - -# Execute the SELECT statement: -cur.execute("select * from people order by age") - -# Retrieve all rows as a sequence and print that sequence: -print(cur.fetchall()) diff --git a/third_party/python/Doc/includes/sqlite3/execute_1.py b/third_party/python/Doc/includes/sqlite3/execute_1.py deleted file mode 100644 index f864a8984..000000000 --- a/third_party/python/Doc/includes/sqlite3/execute_1.py +++ /dev/null @@ -1,16 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") -cur = con.cursor() -cur.execute("create table people (name_last, age)") - -who = "Yeltsin" -age = 72 - -# This is the qmark style: -cur.execute("insert into people values (?, ?)", (who, age)) - -# And this is the named style: -cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) - -print(cur.fetchone()) diff --git a/third_party/python/Doc/includes/sqlite3/execute_3.py b/third_party/python/Doc/includes/sqlite3/execute_3.py deleted file mode 100644 index 0353683fc..000000000 --- a/third_party/python/Doc/includes/sqlite3/execute_3.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -who = "Yeltsin" -age = 72 - -cur.execute("select name_last, age from people where name_last=:who and age=:age", - locals()) -print(cur.fetchone()) diff --git a/third_party/python/Doc/includes/sqlite3/executemany_1.py b/third_party/python/Doc/includes/sqlite3/executemany_1.py deleted file mode 100644 index efae10637..000000000 --- a/third_party/python/Doc/includes/sqlite3/executemany_1.py +++ /dev/null @@ -1,24 +0,0 @@ -import sqlite3 - -class IterChars: - def __init__(self): - self.count = ord('a') - - def __iter__(self): - return self - - def __next__(self): - if self.count > ord('z'): - raise StopIteration - self.count += 1 - return (chr(self.count - 1),) # this is a 1-tuple - -con = sqlite3.connect(":memory:") -cur = con.cursor() -cur.execute("create table characters(c)") - -theIter = IterChars() -cur.executemany("insert into characters(c) values (?)", theIter) - -cur.execute("select c from characters") -print(cur.fetchall()) diff --git a/third_party/python/Doc/includes/sqlite3/executemany_2.py b/third_party/python/Doc/includes/sqlite3/executemany_2.py deleted file mode 100644 index 527358ebc..000000000 --- a/third_party/python/Doc/includes/sqlite3/executemany_2.py +++ /dev/null @@ -1,15 +0,0 @@ -import sqlite3 -import string - -def char_generator(): - for c in string.ascii_lowercase: - yield (c,) - -con = sqlite3.connect(":memory:") -cur = con.cursor() -cur.execute("create table characters(c)") - -cur.executemany("insert into characters(c) values (?)", char_generator()) - -cur.execute("select c from characters") -print(cur.fetchall()) diff --git a/third_party/python/Doc/includes/sqlite3/executescript.py b/third_party/python/Doc/includes/sqlite3/executescript.py deleted file mode 100644 index 7e5358178..000000000 --- a/third_party/python/Doc/includes/sqlite3/executescript.py +++ /dev/null @@ -1,24 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") -cur = con.cursor() -cur.executescript(""" - create table person( - firstname, - lastname, - age - ); - - create table book( - title, - author, - published - ); - - insert into book(title, author, published) - values ( - 'Dirk Gently''s Holistic Detective Agency', - 'Douglas Adams', - 1987 - ); - """) diff --git a/third_party/python/Doc/includes/sqlite3/insert_more_people.py b/third_party/python/Doc/includes/sqlite3/insert_more_people.py deleted file mode 100644 index edbc79e7e..000000000 --- a/third_party/python/Doc/includes/sqlite3/insert_more_people.py +++ /dev/null @@ -1,16 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -newPeople = ( - ('Lebed' , 53), - ('Zhirinovsky' , 57), - ) - -for person in newPeople: - cur.execute("insert into people (name_last, age) values (?, ?)", person) - -# The changes will not be saved unless the transaction is committed explicitly: -con.commit() diff --git a/third_party/python/Doc/includes/sqlite3/load_extension.py b/third_party/python/Doc/includes/sqlite3/load_extension.py deleted file mode 100644 index b997c7066..000000000 --- a/third_party/python/Doc/includes/sqlite3/load_extension.py +++ /dev/null @@ -1,26 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") - -# enable extension loading -con.enable_load_extension(True) - -# Load the fulltext search extension -con.execute("select load_extension('./fts3.so')") - -# alternatively you can load the extension using an API call: -# con.load_extension("./fts3.so") - -# disable extension loading again -con.enable_load_extension(False) - -# example from SQLite wiki -con.execute("create virtual table recipe using fts3(name, ingredients)") -con.executescript(""" - insert into recipe (name, ingredients) values ('broccoli stew', 'broccoli peppers cheese tomatoes'); - insert into recipe (name, ingredients) values ('pumpkin stew', 'pumpkin onions garlic celery'); - insert into recipe (name, ingredients) values ('broccoli pie', 'broccoli cheese onions flour'); - insert into recipe (name, ingredients) values ('pumpkin pie', 'pumpkin sugar flour butter'); - """) -for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"): - print(row) diff --git a/third_party/python/Doc/includes/sqlite3/md5func.py b/third_party/python/Doc/includes/sqlite3/md5func.py deleted file mode 100644 index 0056b2d6c..000000000 --- a/third_party/python/Doc/includes/sqlite3/md5func.py +++ /dev/null @@ -1,11 +0,0 @@ -import sqlite3 -import hashlib - -def md5sum(t): - return hashlib.md5(t).hexdigest() - -con = sqlite3.connect(":memory:") -con.create_function("md5", 1, md5sum) -cur = con.cursor() -cur.execute("select md5(?)", (b"foo",)) -print(cur.fetchone()[0]) diff --git a/third_party/python/Doc/includes/sqlite3/mysumaggr.py b/third_party/python/Doc/includes/sqlite3/mysumaggr.py deleted file mode 100644 index d2dfd2c0b..000000000 --- a/third_party/python/Doc/includes/sqlite3/mysumaggr.py +++ /dev/null @@ -1,20 +0,0 @@ -import sqlite3 - -class MySum: - def __init__(self): - self.count = 0 - - def step(self, value): - self.count += value - - def finalize(self): - return self.count - -con = sqlite3.connect(":memory:") -con.create_aggregate("mysum", 1, MySum) -cur = con.cursor() -cur.execute("create table test(i)") -cur.execute("insert into test(i) values (1)") -cur.execute("insert into test(i) values (2)") -cur.execute("select mysum(i) from test") -print(cur.fetchone()[0]) diff --git a/third_party/python/Doc/includes/sqlite3/parse_colnames.py b/third_party/python/Doc/includes/sqlite3/parse_colnames.py deleted file mode 100644 index cc68c7645..000000000 --- a/third_party/python/Doc/includes/sqlite3/parse_colnames.py +++ /dev/null @@ -1,8 +0,0 @@ -import sqlite3 -import datetime - -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) -dt = cur.fetchone()[0] -print(dt, type(dt)) diff --git a/third_party/python/Doc/includes/sqlite3/pysqlite_datetime.py b/third_party/python/Doc/includes/sqlite3/pysqlite_datetime.py deleted file mode 100644 index 68d49358a..000000000 --- a/third_party/python/Doc/includes/sqlite3/pysqlite_datetime.py +++ /dev/null @@ -1,20 +0,0 @@ -import sqlite3 -import datetime - -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute("create table test(d date, ts timestamp)") - -today = datetime.date.today() -now = datetime.datetime.now() - -cur.execute("insert into test(d, ts) values (?, ?)", (today, now)) -cur.execute("select d, ts from test") -row = cur.fetchone() -print(today, "=>", row[0], type(row[0])) -print(now, "=>", row[1], type(row[1])) - -cur.execute('select current_date as "d [date]", current_timestamp as "ts [timestamp]"') -row = cur.fetchone() -print("current_date", row[0], type(row[0])) -print("current_timestamp", row[1], type(row[1])) diff --git a/third_party/python/Doc/includes/sqlite3/row_factory.py b/third_party/python/Doc/includes/sqlite3/row_factory.py deleted file mode 100644 index e436ffc6c..000000000 --- a/third_party/python/Doc/includes/sqlite3/row_factory.py +++ /dev/null @@ -1,13 +0,0 @@ -import sqlite3 - -def dict_factory(cursor, row): - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - return d - -con = sqlite3.connect(":memory:") -con.row_factory = dict_factory -cur = con.cursor() -cur.execute("select 1 as a") -print(cur.fetchone()["a"]) diff --git a/third_party/python/Doc/includes/sqlite3/rowclass.py b/third_party/python/Doc/includes/sqlite3/rowclass.py deleted file mode 100644 index 92b5ad60c..000000000 --- a/third_party/python/Doc/includes/sqlite3/rowclass.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") -con.row_factory = sqlite3.Row - -cur = con.cursor() -cur.execute("select 'John' as name, 42 as age") -for row in cur: - assert row[0] == row["name"] - assert row["name"] == row["nAmE"] - assert row[1] == row["age"] - assert row[1] == row["AgE"] diff --git a/third_party/python/Doc/includes/sqlite3/shared_cache.py b/third_party/python/Doc/includes/sqlite3/shared_cache.py deleted file mode 100644 index 30e71c935..000000000 --- a/third_party/python/Doc/includes/sqlite3/shared_cache.py +++ /dev/null @@ -1,6 +0,0 @@ -import sqlite3 - -# The shared cache is only available in SQLite versions 3.3.3 or later -# See the SQLite documentation for details. - -sqlite3.enable_shared_cache(True) diff --git a/third_party/python/Doc/includes/sqlite3/shortcut_methods.py b/third_party/python/Doc/includes/sqlite3/shortcut_methods.py deleted file mode 100644 index 71600d4f6..000000000 --- a/third_party/python/Doc/includes/sqlite3/shortcut_methods.py +++ /dev/null @@ -1,20 +0,0 @@ -import sqlite3 - -persons = [ - ("Hugo", "Boss"), - ("Calvin", "Klein") - ] - -con = sqlite3.connect(":memory:") - -# Create the table -con.execute("create table person(firstname, lastname)") - -# Fill the table -con.executemany("insert into person(firstname, lastname) values (?, ?)", persons) - -# Print the table contents -for row in con.execute("select firstname, lastname from person"): - print(row) - -print("I just deleted", con.execute("delete from person").rowcount, "rows") diff --git a/third_party/python/Doc/includes/sqlite3/simple_tableprinter.py b/third_party/python/Doc/includes/sqlite3/simple_tableprinter.py deleted file mode 100644 index 231d8726c..000000000 --- a/third_party/python/Doc/includes/sqlite3/simple_tableprinter.py +++ /dev/null @@ -1,26 +0,0 @@ -import sqlite3 - -FIELD_MAX_WIDTH = 20 -TABLE_NAME = 'people' -SELECT = 'select * from %s order by age, name_last' % TABLE_NAME - -con = sqlite3.connect("mydb") - -cur = con.cursor() -cur.execute(SELECT) - -# Print a header. -for fieldDesc in cur.description: - print(fieldDesc[0].ljust(FIELD_MAX_WIDTH), end=' ') -print() # Finish the header with a newline. -print('-' * 78) - -# For each row, print the value of each field left-justified within -# the maximum possible width of that field. -fieldIndices = range(len(cur.description)) -for row in cur: - for fieldIndex in fieldIndices: - fieldValue = str(row[fieldIndex]) - print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') - - print() # Finish the row with a newline. diff --git a/third_party/python/Doc/includes/sqlite3/text_factory.py b/third_party/python/Doc/includes/sqlite3/text_factory.py deleted file mode 100644 index 5f96cdb58..000000000 --- a/third_party/python/Doc/includes/sqlite3/text_factory.py +++ /dev/null @@ -1,27 +0,0 @@ -import sqlite3 - -con = sqlite3.connect(":memory:") -cur = con.cursor() - -AUSTRIA = "\xd6sterreich" - -# by default, rows are returned as Unicode -cur.execute("select ?", (AUSTRIA,)) -row = cur.fetchone() -assert row[0] == AUSTRIA - -# but we can make sqlite3 always return bytestrings ... -con.text_factory = bytes -cur.execute("select ?", (AUSTRIA,)) -row = cur.fetchone() -assert type(row[0]) is bytes -# the bytestrings will be encoded in UTF-8, unless you stored garbage in the -# database ... -assert row[0] == AUSTRIA.encode("utf-8") - -# we can also implement a custom text_factory ... -# here we implement one that appends "foo" to all strings -con.text_factory = lambda x: x.decode("utf-8") + "foo" -cur.execute("select ?", ("bar",)) -row = cur.fetchone() -assert row[0] == "barfoo" diff --git a/third_party/python/Doc/includes/sublist.c b/third_party/python/Doc/includes/sublist.c deleted file mode 100644 index 376dddfac..000000000 --- a/third_party/python/Doc/includes/sublist.c +++ /dev/null @@ -1,63 +0,0 @@ -#include - -typedef struct { - PyListObject list; - int state; -} SubListObject; - -static PyObject * -SubList_increment(SubListObject *self, PyObject *unused) -{ - self->state++; - return PyLong_FromLong(self->state); -} - -static PyMethodDef SubList_methods[] = { - {"increment", (PyCFunction) SubList_increment, METH_NOARGS, - PyDoc_STR("increment state counter")}, - {NULL}, -}; - -static int -SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) -{ - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; -} - -static PyTypeObject SubListType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "sublist.SubList", - .tp_doc = "SubList objects", - .tp_basicsize = sizeof(SubListObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_init = (initproc) SubList_init, - .tp_methods = SubList_methods, -}; - -static PyModuleDef sublistmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "sublist", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_sublist(void) -{ - PyObject *m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - Py_INCREF(&SubListType); - PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); - return m; -} diff --git a/third_party/python/Doc/includes/test.py b/third_party/python/Doc/includes/test.py deleted file mode 100644 index 09ebe3fec..000000000 --- a/third_party/python/Doc/includes/test.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Test module for the custom examples - -Custom 1: - ->>> import custom ->>> c1 = custom.Custom() ->>> c2 = custom.Custom() ->>> del c1 ->>> del c2 - - -Custom 2 - ->>> import custom2 ->>> c1 = custom2.Custom('jim', 'fulton', 42) ->>> c1.first -'jim' ->>> c1.last -'fulton' ->>> c1.number -42 ->>> c1.name() -'jim fulton' ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c1.last = 'tell' ->>> c1.name() -'will tell' ->>> del c1.first ->>> c1.name() -Traceback (most recent call last): -... -AttributeError: first ->>> c1.first -Traceback (most recent call last): -... -AttributeError: first ->>> c1.first = 'drew' ->>> c1.first -'drew' ->>> del c1.number -Traceback (most recent call last): -... -TypeError: can't delete numeric/char attribute ->>> c1.number=2 ->>> c1.number -2 ->>> c1.first = 42 ->>> c1.name() -'42 tell' ->>> c2 = custom2.Custom() ->>> c2.name() -' ' ->>> c2.first -'' ->>> c2.last -'' ->>> del c2.first ->>> c2.first -Traceback (most recent call last): -... -AttributeError: first ->>> c2.first -Traceback (most recent call last): -... -AttributeError: first ->>> c2.name() -Traceback (most recent call last): - File "", line 1, in ? -AttributeError: first ->>> c2.number -0 ->>> n3 = custom2.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): - File "", line 1, in ? -TypeError: an integer is required (got type str) ->>> del c1 ->>> del c2 - - -Custom 3 - ->>> import custom3 ->>> c1 = custom3.Custom('jim', 'fulton', 42) ->>> c1 = custom3.Custom('jim', 'fulton', 42) ->>> c1.name() -'jim fulton' ->>> del c1.first -Traceback (most recent call last): - File "", line 1, in ? -TypeError: Cannot delete the first attribute ->>> c1.first = 42 -Traceback (most recent call last): - File "", line 1, in ? -TypeError: The first attribute value must be a string ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c2 = custom3.Custom() ->>> c2 = custom3.Custom() ->>> c2 = custom3.Custom() ->>> n3 = custom3.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): - File "", line 1, in ? -TypeError: an integer is required (got type str) ->>> del c1 ->>> del c2 - -Custom 4 - ->>> import custom4 ->>> c1 = custom4.Custom('jim', 'fulton', 42) ->>> c1.first -'jim' ->>> c1.last -'fulton' ->>> c1.number -42 ->>> c1.name() -'jim fulton' ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c1.last = 'tell' ->>> c1.name() -'will tell' ->>> del c1.first -Traceback (most recent call last): -... -TypeError: Cannot delete the first attribute ->>> c1.name() -'will tell' ->>> c1.first = 'drew' ->>> c1.first -'drew' ->>> del c1.number -Traceback (most recent call last): -... -TypeError: can't delete numeric/char attribute ->>> c1.number=2 ->>> c1.number -2 ->>> c1.first = 42 -Traceback (most recent call last): -... -TypeError: The first attribute value must be a string ->>> c1.name() -'drew tell' ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2.name() -' ' ->>> c2.first -'' ->>> c2.last -'' ->>> c2.number -0 ->>> n3 = custom4.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): -... -TypeError: an integer is required (got type str) - - -Test cyclic gc(?) - ->>> import gc ->>> gc.disable() - ->>> class Subclass(custom4.Custom): pass -... ->>> s = Subclass() ->>> s.cycle = [s] ->>> s.cycle.append(s.cycle) ->>> x = object() ->>> s.x = x ->>> del s ->>> sys.getrefcount(x) -3 ->>> ignore = gc.collect() ->>> sys.getrefcount(x) -2 - ->>> gc.enable() -""" - -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - -if __name__ == "__main__": - import doctest, __main__ - doctest.testmod(__main__) diff --git a/third_party/python/Doc/includes/turtle-star.py b/third_party/python/Doc/includes/turtle-star.py deleted file mode 100644 index 1a5db761b..000000000 --- a/third_party/python/Doc/includes/turtle-star.py +++ /dev/null @@ -1,10 +0,0 @@ -from turtle import * -color('red', 'yellow') -begin_fill() -while True: - forward(200) - left(170) - if abs(pos()) < 1: - break -end_fill() -done() diff --git a/third_party/python/Doc/includes/typestruct.h b/third_party/python/Doc/includes/typestruct.h deleted file mode 100644 index 9f47899a1..000000000 --- a/third_party/python/Doc/includes/typestruct.h +++ /dev/null @@ -1,80 +0,0 @@ -typedef struct _typeobject { - PyObject_VAR_HEAD - const char *tp_name; /* For printing, in format "." */ - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ - - /* Methods to implement standard operations */ - - destructor tp_dealloc; - printfunc tp_print; - getattrfunc tp_getattr; - setattrfunc tp_setattr; - PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) - or tp_reserved (Python 3) */ - reprfunc tp_repr; - - /* Method suites for standard classes */ - - PyNumberMethods *tp_as_number; - PySequenceMethods *tp_as_sequence; - PyMappingMethods *tp_as_mapping; - - /* More standard operations (here for binary compatibility) */ - - hashfunc tp_hash; - ternaryfunc tp_call; - reprfunc tp_str; - getattrofunc tp_getattro; - setattrofunc tp_setattro; - - /* Functions to access object as input/output buffer */ - PyBufferProcs *tp_as_buffer; - - /* Flags to define presence of optional/expanded features */ - unsigned long tp_flags; - - const char *tp_doc; /* Documentation string */ - - /* call function for all accessible objects */ - traverseproc tp_traverse; - - /* delete references to contained objects */ - inquiry tp_clear; - - /* rich comparisons */ - richcmpfunc tp_richcompare; - - /* weak reference enabler */ - Py_ssize_t tp_weaklistoffset; - - /* Iterators */ - getiterfunc tp_iter; - iternextfunc tp_iternext; - - /* Attribute descriptor and subclassing stuff */ - struct PyMethodDef *tp_methods; - struct PyMemberDef *tp_members; - struct PyGetSetDef *tp_getset; - struct _typeobject *tp_base; - PyObject *tp_dict; - descrgetfunc tp_descr_get; - descrsetfunc tp_descr_set; - Py_ssize_t tp_dictoffset; - initproc tp_init; - allocfunc tp_alloc; - newfunc tp_new; - freefunc tp_free; /* Low-level free-memory routine */ - inquiry tp_is_gc; /* For PyObject_IS_GC */ - PyObject *tp_bases; - PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; - PyObject *tp_subclasses; - PyObject *tp_weaklist; - destructor tp_del; - - /* Type attribute cache version tag. Added in version 2.6 */ - unsigned int tp_version_tag; - - destructor tp_finalize; - -} PyTypeObject; diff --git a/third_party/python/Doc/includes/tzinfo-examples.py b/third_party/python/Doc/includes/tzinfo-examples.py deleted file mode 100644 index ae5a50926..000000000 --- a/third_party/python/Doc/includes/tzinfo-examples.py +++ /dev/null @@ -1,175 +0,0 @@ -from datetime import tzinfo, timedelta, datetime, timezone - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) - -# A class capturing the platform's idea of local time. -# (May result in wrong values on historical times in -# timezones where UTC offset and/or the DST rules had -# changed in the past.) -import time as _time - -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) -else: - DSTOFFSET = STDOFFSET - -DSTDIFF = DSTOFFSET - STDOFFSET - -class LocalTimezone(tzinfo): - - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] - dst_diff = DSTDIFF // SECOND - # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) - - def utcoffset(self, dt): - if self._isdst(dt): - return DSTOFFSET - else: - return STDOFFSET - - def dst(self, dt): - if self._isdst(dt): - return DSTDIFF - else: - return ZERO - - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] - - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) - return tt.tm_isdst > 0 - -Local = LocalTimezone() - - -# A complete implementation of current DST rules for major US time zones. - -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() - if days_to_go: - dt += timedelta(days_to_go) - return dt - - -# US DST Rules -# -# This is a simplified (i.e., wrong for a few cases) set of rules for US -# DST start and end times. For a complete and up-to-date set of DST rules -# and timezone definitions, visit the Olson Database (or try pytz): -# http://www.twinsun.com/tz/tz-link.htm -# http://sourceforge.net/projects/pytz/ (might not be up-to-date) -# -# In the US, since 2007, DST starts at 2am (standard time) on the second -# Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) -# and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) -# From 1987 to 2006, DST used to start at 2am (standard time) on the first -# Sunday in April and to end at 2am (DST time) on the last -# Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) -# From 1967 to 1986, DST used to start at 2am (standard time) on the last -# Sunday in April (the one on or after April 24) and to end at 2am (DST time) -# on the last Sunday of October, which is the first Sunday -# on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) -DSTEND_1967_1986 = DSTEND_1987_2006 - -def us_dst_range(year): - # Find start and end times for US DST. For years before 1967, return - # start = end for no DST. - if 2006 < year: - dststart, dstend = DSTSTART_2007, DSTEND_2007 - elif 1986 < year < 2007: - dststart, dstend = DSTSTART_1987_2006, DSTEND_1987_2006 - elif 1966 < year < 1987: - dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 - else: - return (datetime(year, 1, 1), ) * 2 - - start = first_sunday_on_or_after(dststart.replace(year=year)) - end = first_sunday_on_or_after(dstend.replace(year=year)) - return start, end - - -class USTimeZone(tzinfo): - - def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) - self.reprname = reprname - self.stdname = stdname - self.dstname = dstname - - def __repr__(self): - return self.reprname - - def tzname(self, dt): - if self.dst(dt): - return self.dstname - else: - return self.stdname - - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) - - def dst(self, dt): - if dt is None or dt.tzinfo is None: - # An exception may be sensible here, in one or both cases. - # It depends on how you want to treat them. The default - # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. - return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) - # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: - # DST is in effect. - return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: - # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO - # DST is off. - return ZERO - - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) - start = start.replace(tzinfo=self) - end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset - dst_time = std_time + HOUR - if end <= dst_time < end + HOUR: - # Repeated hour - return std_time.replace(fold=1) - if std_time < start or dst_time >= end: - # Standard time - return std_time - if start <= std_time < end - HOUR: - # Daylight saving time - return dst_time - - -Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") -Central = USTimeZone(-6, "Central", "CST", "CDT") -Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") -Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") diff --git a/third_party/python/Doc/install/index.rst b/third_party/python/Doc/install/index.rst deleted file mode 100644 index 5a59b4d80..000000000 --- a/third_party/python/Doc/install/index.rst +++ /dev/null @@ -1,1117 +0,0 @@ -.. highlightlang:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentations - -.. The audience for this document includes people who don't know anything - about Python and aren't about to learn the language just in order to - install and maintain it for their users, i.e. system administrators. - Thus, I have to be sure to explain the basics at some point: - sys.path and PYTHONPATH at least. Should probably give pointers to - other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. - - Finally, it might be useful to include all the material from my "Care - and Feeding of a Python Installation" talk in here somewhere. Yow! - -This document describes the Python Distribution Utilities ("Distutils") from the -end-user's point-of-view, describing how to extend the capabilities of a -standard Python installation by building and installing third-party Python -modules and extensions. - - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -Although Python's extensive standard library covers many programming needs, -there often comes a time when you need to add some new functionality to your -Python installation in the form of third-party modules. This might be necessary -to support your own programming, or to support an application that you want to -use and that happens to be written in Python. - -In the past, there has been little support for adding third-party modules to an -existing Python installation. With the introduction of the Python Distribution -Utilities (Distutils for short) in Python 2.0, this changed. - -This document is aimed primarily at the people who need to install third-party -Python modules: end-users and system administrators who just need to get some -Python application running, and existing Python programmers who want to add some -new goodies to their toolbox. You don't need to know Python to read this -document; there will be some brief forays into using Python's interactive mode -to explore your installation, but that's it. If you're looking for information -on how to distribute your own Python modules so that others may use them, see -the :ref:`distutils-index` manual. :ref:`debug-setup-script` may also be of -interest. - - -.. _inst-trivial-install: - -Best case: trivial installation -------------------------------- - -In the best case, someone will have prepared a special version of the module -distribution you want to install that is targeted specifically at your platform -and is installed just like any other software on your platform. For example, -the module developer might make an executable installer available for Windows -users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, -Mandrake, and many others), a Debian package for users of Debian-based Linux -systems, and so forth. - -In that case, you would download the installer appropriate to your platform and -do the obvious thing with it: run it if it's an executable installer, ``rpm ---install`` it if it's an RPM, etc. You don't need to run Python or a setup -script, you don't need to compile anything---you might not even need to read any -instructions (although it's always a good idea to do so anyway). - -Of course, things will not always be that easy. You might be interested in a -module distribution that doesn't have an easy-to-use installer for your -platform. In that case, you'll have to start with the source distribution -released by the module's author/maintainer. Installing from a source -distribution is not too hard, as long as the modules are packaged in the -standard way. The bulk of this document is about building and installing -modules from standard source distributions. - - -.. _inst-new-standard: - -The new standard: Distutils ---------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and Mac OS X, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and Mac OS X. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :data:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :data:`site.USER_SITE`). -Here are the values for UNIX, including Mac OS X: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme," where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same major version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and Mac OS X, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the `--no-user-cfg` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState Web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ and http://www.mingw.org/ for more - information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/third_party/python/Doc/installing/index.rst b/third_party/python/Doc/installing/index.rst deleted file mode 100644 index 18f5b2939..000000000 --- a/third_party/python/Doc/installing/index.rst +++ /dev/null @@ -1,246 +0,0 @@ -.. highlightlang:: none - -.. _installing-index: - -************************* -Installing Python Modules -************************* - -:Email: distutils-sig@python.org - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the installation part of the process. For a guide to -creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. - -.. note:: - - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* ``pip`` is the preferred installer program. Starting with Python 3.4, it - is included by default with the Python binary installers. -* A *virtual environment* is a semi-isolated Python environment that allows - packages to be installed for use by a particular application, rather than - being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Packaging Index `__ is a public - repository of open source licensed packages made available for use by - other Python users. -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation, - and issue trackers on both `GitHub `__ and - `BitBucket `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). - -.. deprecated:: 3.6 - ``pyvenv`` was the recommended tool for creating virtual environments for - Python 3.3 and 3.4, and is `deprecated in Python 3.6 - `_. - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - - -Basic usage -=========== - -The standard packaging tools are all designed to be used from the command -line. - -The following command will install the latest version of a module and its -dependencies from the Python Packaging Index:: - - python -m pip install SomePackage - -.. note:: - - For POSIX users (including Mac OS X and Linux users), the examples in - this guide assume the use of a :term:`virtual environment`. - - For Windows users, the examples in this guide assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -It's also possible to specify an exact or minimum version directly on the -command line. When using comparator operators such as ``>``, ``<`` or some other -special character which get interpreted by shell, the package name and the -version should be enclosed within double quotes:: - - python -m pip install SomePackage==1.0.4 # specific version - python -m pip install "SomePackage>=1.0.4" # minimum version - -Normally, if a suitable module is already installed, attempting to install -it again will have no effect. Upgrading existing modules must be requested -explicitly:: - - python -m pip install --upgrade SomePackage - -More information and resources regarding ``pip`` and its capabilities can be -found in the `Python Packaging User Guide `__. - -Creation of virtual environments is done through the :mod:`venv` module. -Installing packages into an active virtual environment uses the commands shown -above. - -.. seealso:: - - `Python Packaging User Guide: Installing Python Distribution Packages - `__ - - -How do I ...? -============= - -These are quick answers or links for some common tasks. - -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - -.. installing-per-user-installation: - -... install packages just for the current user? ------------------------------------------------ - -Passing the ``--user`` option to ``python -m pip install`` will install a -package just for the current user, rather than for all users of the system. - - -... install scientific Python packages? ---------------------------------------- - -A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by -`other means `__ -rather than attempting to install them with ``pip``. - -.. seealso:: - - `Python Packaging User Guide: Installing Scientific Packages - `__ - - -... work with multiple versions of Python installed in parallel? ----------------------------------------------------------------- - -On Linux, Mac OS X, and other POSIX systems, use the versioned Python commands -in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: - - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 - -Appropriately versioned ``pip`` commands may also be available. - -On Windows, use the ``py`` Python launcher in combination with the ``-m`` -switch:: - - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 - -.. other questions: - - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) - - -Common installation issues -========================== - -Installing into the system Python on Linux ------------------------------------------- - -On Linux systems, a Python installation will typically be included as part -of the distribution. Installing into this Python installation requires -root access to the system, and may interfere with the operation of the -system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. - -On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. - - -Pip not installed ------------------ - -It is possible that ``pip`` does not get installed by default. One potential fix is:: - - python -m ensurepip --default-pip - -There are also additional resources for `installing pip. -`__ - - -Installing binary extensions ----------------------------- - -Python has typically relied heavily on source based distribution, with end -users being expected to compile extension modules from source as part of -the installation process. - -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and Mac OS X through the -Python Packaging Index, this problem is expected to diminish over time, -as users are more regularly able to install pre-built extensions rather -than needing to build them themselves. - -Some of the solutions for installing `scientific software -`__ -that are not yet available as pre-built ``wheel`` files may also help with -obtaining other binary extensions without needing to build them locally. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ diff --git a/third_party/python/Doc/library/2to3.rst b/third_party/python/Doc/library/2to3.rst deleted file mode 100644 index 1ab05a62a..000000000 --- a/third_party/python/Doc/library/2to3.rst +++ /dev/null @@ -1,474 +0,0 @@ -.. _2to3-reference: - -2to3 - Automated Python 2 to 3 code translation -=============================================== - -.. sectionauthor:: Benjamin Peterson - -2to3 is a Python program that reads Python 2.x source code and applies a series -of *fixers* to transform it into valid Python 3.x code. The standard library -contains a rich set of fixers that will handle almost all code. 2to3 supporting -library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. :mod:`lib2to3` could also be -adapted to custom applications in which Python code needs to be edited -automatically. - - -.. _2to3-using: - -Using 2to3 ----------- - -2to3 will usually be installed with the Python interpreter as a script. It is -also located in the :file:`Tools/scripts` directory of the Python root. - -2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. - -Here is a sample Python 2.x source file, :file:`example.py`:: - - def greet(name): - print "Hello, {0}!".format(name) - print "What's your name?" - name = raw_input() - greet(name) - -It can be converted to Python 3.x code via 2to3 on the command line: - -.. code-block:: shell-session - - $ 2to3 example.py - -A diff against the original source file is printed. 2to3 can also write the -needed modifications right back to the source file. (A backup of the original -file is made unless :option:`!-n` is also given.) Writing the changes back is -enabled with the :option:`!-w` flag: - -.. code-block:: shell-session - - $ 2to3 -w example.py - -After transformation, :file:`example.py` looks like this:: - - def greet(name): - print("Hello, {0}!".format(name)) - print("What's your name?") - name = input() - greet(name) - -Comments and exact indentation are preserved throughout the translation process. - -By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The -:option:`!-l` flag lists all available fixers. An explicit set of fixers to run -can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a -fixer. The following example runs only the ``imports`` and ``has_key`` fixers: - -.. code-block:: shell-session - - $ 2to3 -f imports -f has_key example.py - -This command runs every fixer except the ``apply`` fixer: - -.. code-block:: shell-session - - $ 2to3 -x apply example.py - -Some fixers are *explicit*, meaning they aren't run by default and must be -listed on the command line to be run. Here, in addition to the default fixers, -the ``idioms`` fixer is run: - -.. code-block:: shell-session - - $ 2to3 -f all -f idioms example.py - -Notice how passing ``all`` enables all default fixers. - -Sometimes 2to3 will find a place in your source code that needs to be changed, -but 2to3 cannot fix automatically. In this case, 2to3 will print a warning -beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. - -2to3 can also refactor doctests. To enable this mode, use the :option:`!-d` -flag. Note that *only* doctests will be refactored. This also doesn't require -the module to be valid Python. For example, doctest like examples in a reST -document could also be refactored with this option. - -The :option:`!-v` option enables output of more information on the translation -process. - -Since some print statements can be parsed as function calls or statements, 2to3 -cannot always read files containing the print function. When 2to3 detects the -presence of the ``from __future__ import print_function`` compiler directive, it -modifies its internal grammar to interpret :func:`print` as a function. This -change can also be enabled manually with the :option:`!-p` flag. Use -:option:`!-p` to run fixers on code that already has had its print statements -converted. - -The :option:`!-o` or :option:`!--output-dir` option allows specification of an -alternate directory for processed output files to be written to. The -:option:`!-n` flag is required when using this as backup files do not make sense -when not overwriting the input files. - -.. versionadded:: 3.2.3 - The :option:`!-o` option was added. - -The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always -write output files even if no changes were required to the file. This is most -useful with :option:`!-o` so that an entire Python source tree is copied with -translation from one directory to another. -This option implies the :option:`!-w` flag as it would not make sense otherwise. - -.. versionadded:: 3.2.3 - The :option:`!-W` flag was added. - -The :option:`!--add-suffix` option specifies a string to append to all output -filenames. The :option:`!-n` flag is required when specifying this as backups -are not necessary when writing to different filenames. Example: - -.. code-block:: shell-session - - $ 2to3 -n -W --add-suffix=3 example.py - -Will cause a converted file named ``example.py3`` to be written. - -.. versionadded:: 3.2.3 - The :option:`!--add-suffix` option was added. - -To translate an entire project from one directory tree to another use: - -.. code-block:: shell-session - - $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode - - -.. _2to3-fixers: - -Fixers ------- - -Each step of transforming code is encapsulated in a fixer. The command ``2to3 --l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on -and off individually. They are described here in more detail. - - -.. 2to3fixer:: apply - - Removes usage of :func:`apply`. For example ``apply(function, *args, - **kwargs)`` is converted to ``function(*args, **kwargs)``. - -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - -.. 2to3fixer:: basestring - - Converts :class:`basestring` to :class:`str`. - -.. 2to3fixer:: buffer - - Converts :class:`buffer` to :class:`memoryview`. This fixer is optional - because the :class:`memoryview` API is similar but not exactly the same as - that of :class:`buffer`. - -.. 2to3fixer:: dict - - Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to - :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and - :meth:`dict.itervalues` to :meth:`dict.values`. Similarly, - :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are - converted respectively to :meth:`dict.items`, :meth:`dict.keys` and - :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`, - :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`. - -.. 2to3fixer:: except - - Converts ``except X, T`` to ``except X as T``. - -.. 2to3fixer:: exec - - Converts the ``exec`` statement to the :func:`exec` function. - -.. 2to3fixer:: execfile - - Removes usage of :func:`execfile`. The argument to :func:`execfile` is - wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`. - -.. 2to3fixer:: exitfunc - - Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit` - module. - -.. 2to3fixer:: filter - - Wraps :func:`filter` usage in a :class:`list` call. - -.. 2to3fixer:: funcattrs - - Fixes function attributes that have been renamed. For example, - ``my_function.func_closure`` is converted to ``my_function.__closure__``. - -.. 2to3fixer:: future - - Removes ``from __future__ import new_feature`` statements. - -.. 2to3fixer:: getcwdu - - Renames :func:`os.getcwdu` to :func:`os.getcwd`. - -.. 2to3fixer:: has_key - - Changes ``dict.has_key(key)`` to ``key in dict``. - -.. 2to3fixer:: idioms - - This optional fixer performs several transformations that make Python code - more idiomatic. Type comparisons like ``type(x) is SomeClass`` and - ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``. - ``while 1`` becomes ``while True``. This fixer also tries to make use of - :func:`sorted` in appropriate places. For example, this block :: - - L = list(some_iterable) - L.sort() - - is changed to :: - - L = sorted(some_iterable) - -.. 2to3fixer:: import - - Detects sibling imports and converts them to relative imports. - -.. 2to3fixer:: imports - - Handles module renames in the standard library. - -.. 2to3fixer:: imports2 - - Handles other modules renames in the standard library. It is separate from - the :2to3fixer:`imports` fixer only because of technical limitations. - -.. 2to3fixer:: input - - Converts ``input(prompt)`` to ``eval(input(prompt))``. - -.. 2to3fixer:: intern - - Converts :func:`intern` to :func:`sys.intern`. - -.. 2to3fixer:: isinstance - - Fixes duplicate types in the second argument of :func:`isinstance`. For - example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - int)`` and ``isinstance(x, (int, float, int))`` is converted to - ``isinstance(x, (int, float))``. - -.. 2to3fixer:: itertools_imports - - Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also - changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: itertools - - Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap` to their built-in equivalents. - :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: long - - Renames :class:`long` to :class:`int`. - -.. 2to3fixer:: map - - Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)`` - to ``list(x)``. Using ``from future_builtins import map`` disables this - fixer. - -.. 2to3fixer:: metaclass - - Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class - body) to the new (``class X(metaclass=Meta)``). - -.. 2to3fixer:: methodattrs - - Fixes old method attribute names. For example, ``meth.im_func`` is converted - to ``meth.__func__``. - -.. 2to3fixer:: ne - - Converts the old not-equal syntax, ``<>``, to ``!=``. - -.. 2to3fixer:: next - - Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. - -.. 2to3fixer:: nonzero - - Renames :meth:`__nonzero__` to :meth:`~object.__bool__`. - -.. 2to3fixer:: numliterals - - Converts octal literals into the new syntax. - -.. 2to3fixer:: operator - - Converts calls to various functions in the :mod:`operator` module to other, - but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections``. The following mapping - are made: - - ================================== ========================================== - From To - ================================== ========================================== - ``operator.isCallable(obj)`` ``hasattr(obj, '__call__')`` - ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.Mapping)`` - ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` - ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` - ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ========================================== - -.. 2to3fixer:: paren - - Add extra parenthesis where they are required in list comprehensions. For - example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``. - -.. 2to3fixer:: print - - Converts the ``print`` statement to the :func:`print` function. - -.. 2to3fixer:: raise - - Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise - E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be - incorrect because substituting tuples for exceptions has been removed in 3.0. - -.. 2to3fixer:: raw_input - - Converts :func:`raw_input` to :func:`input`. - -.. 2to3fixer:: reduce - - Handles the move of :func:`reduce` to :func:`functools.reduce`. - -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`imp.reload`. - -.. 2to3fixer:: renames - - Changes :data:`sys.maxint` to :data:`sys.maxsize`. - -.. 2to3fixer:: repr - - Replaces backtick repr with the :func:`repr` function. - -.. 2to3fixer:: set_literal - - Replaces use of the :class:`set` constructor with set literals. This fixer - is optional. - -.. 2to3fixer:: standarderror - - Renames :exc:`StandardError` to :exc:`Exception`. - -.. 2to3fixer:: sys_exc - - Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`, - :data:`sys.exc_traceback` to use :func:`sys.exc_info`. - -.. 2to3fixer:: throw - - Fixes the API change in generator's :meth:`throw` method. - -.. 2to3fixer:: tuple_params - - Removes implicit tuple parameter unpacking. This fixer inserts temporary - variables. - -.. 2to3fixer:: types - - Fixes code broken from the removal of some members in the :mod:`types` - module. - -.. 2to3fixer:: unicode - - Renames :class:`unicode` to :class:`str`. - -.. 2to3fixer:: urllib - - Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib` - package. - -.. 2to3fixer:: ws_comma - - Removes excess whitespace from comma separated items. This fixer is - optional. - -.. 2to3fixer:: xrange - - Renames :func:`xrange` to :func:`range` and wraps existing :func:`range` - calls with :class:`list`. - -.. 2to3fixer:: xreadlines - - Changes ``for x in file.xreadlines()`` to ``for x in file``. - -.. 2to3fixer:: zip - - Wraps :func:`zip` usage in a :class:`list` call. This is disabled when - ``from future_builtins import zip`` appears. - - -:mod:`lib2to3` - 2to3's library -------------------------------- - -.. module:: lib2to3 - :synopsis: the 2to3 library - -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Collin Winter -.. moduleauthor:: Benjamin Peterson - -**Source code:** :source:`Lib/lib2to3/` - --------------- - -.. note:: - - The :mod:`lib2to3` API should be considered unstable and may change - drastically in the future. - -.. XXX What is the public interface anyway? diff --git a/third_party/python/Doc/library/__future__.rst b/third_party/python/Doc/library/__future__.rst deleted file mode 100644 index 73d8b6b7e..000000000 --- a/third_party/python/Doc/library/__future__.rst +++ /dev/null @@ -1,98 +0,0 @@ -:mod:`__future__` --- Future statement definitions -================================================== - -.. module:: __future__ - :synopsis: Future statement definitions - -**Source code:** :source:`Lib/__future__.py` - --------------- - -:mod:`__future__` is a real module, and serves three purposes: - -* To avoid confusing existing tools that analyze import statements and expect to - find the modules they're importing. - -* To ensure that :ref:`future statements ` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - -* To document when incompatible changes were introduced, and when they will be - --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining - its contents. - -Each statement in :file:`__future__.py` is of the form:: - - FeatureName = _Feature(OptionalRelease, MandatoryRelease, - CompilerFlag) - - -where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are -5-tuples of the same form as :data:`sys.version_info`:: - - (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int - PY_MINOR_VERSION, # the 1; an int - PY_MICRO_VERSION, # the 0; an int - PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string - PY_RELEASE_SERIAL # the 3; an int - ) - -*OptionalRelease* records the first release in which the feature was accepted. - -In the case of a *MandatoryRelease* that has not yet occurred, -*MandatoryRelease* predicts the release in which the feature will become part of -the language. - -Else *MandatoryRelease* records when the feature became part of the language; in -releases at or after that, modules no longer need a future statement to use the -feature in question, but may continue to use such imports. - -*MandatoryRelease* may also be ``None``, meaning that a planned feature got -dropped. - -Instances of class :class:`_Feature` have two corresponding methods, -:meth:`getOptionalRelease` and :meth:`getMandatoryRelease`. - -*CompilerFlag* is the (bitfield) flag that should be passed in the fourth -argument to the built-in function :func:`compile` to enable the feature in -dynamically compiled code. This flag is stored in the :attr:`compiler_flag` -attribute on :class:`_Feature` instances. - -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ - - -.. seealso:: - - :ref:`future` - How the compiler treats future imports. diff --git a/third_party/python/Doc/library/__main__.rst b/third_party/python/Doc/library/__main__.rst deleted file mode 100644 index a64faf1bb..000000000 --- a/third_party/python/Doc/library/__main__.rst +++ /dev/null @@ -1,25 +0,0 @@ - -:mod:`__main__` --- Top-level script environment -================================================ - -.. module:: __main__ - :synopsis: The environment where the top-level script is run. - --------------- - -``'__main__'`` is the name of the scope in which top-level code executes. -A module's __name__ is set equal to ``'__main__'`` when read from -standard input, a script, or from an interactive prompt. - -A module can discover whether or not it is running in the main scope by -checking its own ``__name__``, which allows a common idiom for conditionally -executing code in a module when it is run as a script or with ``python --m`` but not when it is imported:: - - if __name__ == "__main__": - # execute only if run as a script - main() - -For a package, the same effect can be achieved by including a -``__main__.py`` module, the contents of which will be executed when the -module is run with ``-m``. diff --git a/third_party/python/Doc/library/_dummy_thread.rst b/third_party/python/Doc/library/_dummy_thread.rst deleted file mode 100644 index ebce74d5a..000000000 --- a/third_party/python/Doc/library/_dummy_thread.rst +++ /dev/null @@ -1,25 +0,0 @@ -:mod:`_dummy_thread` --- Drop-in replacement for the :mod:`_thread` module -========================================================================== - -.. module:: _dummy_thread - :synopsis: Drop-in replacement for the _thread module. - -**Source code:** :source:`Lib/_dummy_thread.py` - --------------- - -This module provides a duplicate interface to the :mod:`_thread` module. It is -meant to be imported when the :mod:`_thread` module is not provided on a -platform. - -Suggested usage is:: - - try: - import _thread - except ImportError: - import _dummy_thread as _thread - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. - diff --git a/third_party/python/Doc/library/_thread.rst b/third_party/python/Doc/library/_thread.rst deleted file mode 100644 index 0d2d818f5..000000000 --- a/third_party/python/Doc/library/_thread.rst +++ /dev/null @@ -1,192 +0,0 @@ -:mod:`_thread` --- Low-level threading API -========================================== - -.. module:: _thread - :synopsis: Low-level threading API. - -.. index:: - single: light-weight processes - single: processes, light-weight - single: binary semaphores - single: semaphores, binary - --------------- - -This module provides low-level primitives for working with multiple threads -(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of -control sharing their global data space. For synchronization, simple locks -(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. -The :mod:`threading` module provides an easier to use and higher-level -threading API built on top of this module. - -.. index:: - single: pthreads - pair: threads; POSIX - -The module is optional. It is supported on Windows, Linux, SGI IRIX, Solaris -2.x, as well as on systems that have a POSIX thread (a.k.a. "pthread") -implementation. For systems lacking the :mod:`_thread` module, the -:mod:`_dummy_thread` module is available. It duplicates this module's interface -and can be used as a drop-in replacement. - -It defines the following constants and functions: - - -.. exception:: error - - Raised on thread-specific errors. - - .. versionchanged:: 3.3 - This is now a synonym of the built-in :exc:`RuntimeError`. - - -.. data:: LockType - - This is the type of lock objects. - - -.. function:: start_new_thread(function, args[, kwargs]) - - Start a new thread and return its identifier. The thread executes the function - *function* with the argument list *args* (which must be a tuple). The optional - *kwargs* argument specifies a dictionary of keyword arguments. When the function - returns, the thread silently exits. When the function terminates with an - unhandled exception, a stack trace is printed and then the thread exits (but - other threads continue to run). - - -.. function:: interrupt_main() - - Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can - use this function to interrupt the main thread. - - -.. function:: exit() - - Raise the :exc:`SystemExit` exception. When not caught, this will cause the - thread to exit silently. - -.. - function:: exit_prog(status) - - Exit all threads and report the value of the integer argument - *status* as the exit status of the entire program. - **Caveat:** code in pending :keyword:`finally` clauses, in this thread - or in other threads, is not executed. - - -.. function:: allocate_lock() - - Return a new lock object. Methods of locks are described below. The lock is - initially unlocked. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie to - be used e.g. to index a dictionary of thread-specific data. Thread identifiers - may be recycled when a thread exits and another thread is created. - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - Availability: Windows, systems with POSIX threads. - - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -Lock objects have the following methods: - - -.. method:: lock.acquire(waitflag=1, timeout=-1) - - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). - - If the integer *waitflag* argument is present, the action depends on its - value: if it is zero, the lock is only acquired if it can be acquired - immediately without waiting, while if it is nonzero, the lock is acquired - unconditionally as above. - - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *waitflag* is zero. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. - - -.. method:: lock.release() - - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. - - -.. method:: lock.locked() - - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. - -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: - - import _thread - - a_lock = _thread.allocate_lock() - - with a_lock: - print("a_lock is locked while this executes") - -**Caveats:** - - .. index:: module: signal - -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) - -* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is - equivalent to calling :func:`_thread.exit`. - -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. - -* When the main thread exits, it is system defined whether the other threads - survive. On most systems, they are killed without executing - :keyword:`try` ... :keyword:`finally` clauses or executing object - destructors. - -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/third_party/python/Doc/library/abc.rst b/third_party/python/Doc/library/abc.rst deleted file mode 100644 index fc39a2e75..000000000 --- a/third_party/python/Doc/library/abc.rst +++ /dev/null @@ -1,342 +0,0 @@ -:mod:`abc` --- Abstract Base Classes -==================================== - -.. module:: abc - :synopsis: Abstract base classes according to PEP 3119. - -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Georg Brandl -.. much of the content adapted from docstrings - -**Source code:** :source:`Lib/abc.py` - --------------- - -This module provides the infrastructure for defining :term:`abstract base -classes ` (ABCs) in Python, as outlined in :pep:`3119`; -see the PEP for why this was added to Python. (See also :pep:`3141` and the -:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.) - -The :mod:`collections` module has some concrete classes that derive from -ABCs; these can, of course, be further derived. In addition, the -:mod:`collections.abc` submodule has some ABCs that can be used to test whether -a class or instance provides a particular interface, for example, if it is -hashable or if it is a mapping. - - -This module provides the metaclass :class:`ABCMeta` for defining ABCs and -a helper class :class:`ABC` to alternatively define ABCs through inheritance: - -.. class:: ABC - - A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC` - avoiding sometimes confusing metaclass usage, for example:: - - from abc import ABC - - class MyABC(ABC): - pass - - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding - metaclass usage, as multiple inheritance may lead to metaclass conflicts. - One may also define an abstract base class by passing the metaclass - keyword and using :class:`ABCMeta` directly, for example:: - - from abc import ABCMeta - - class MyABC(metaclass=ABCMeta): - pass - - .. versionadded:: 3.4 - - -.. class:: ABCMeta - - Metaclass for defining Abstract Base Classes (ABCs). - - Use this metaclass to create an ABC. An ABC can be subclassed directly, and - then acts as a mix-in class. You can also register unrelated concrete - classes (even built-in classes) and unrelated ABCs as "virtual subclasses" -- - these and their descendants will be considered subclasses of the registering - ABC by the built-in :func:`issubclass` function, but the registering ABC - won't show up in their MRO (Method Resolution Order) nor will method - implementations defined by the registering ABC be callable (not even via - :func:`super`). [#]_ - - Classes created with a metaclass of :class:`ABCMeta` have the following method: - - .. method:: register(subclass) - - Register *subclass* as a "virtual subclass" of this ABC. For - example:: - - from abc import ABC - - class MyABC(ABC): - pass - - MyABC.register(tuple) - - assert issubclass(tuple, MyABC) - assert isinstance((), MyABC) - - .. versionchanged:: 3.3 - Returns the registered subclass, to allow usage as a class decorator. - - .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the - :func:`get_cache_token` function. - - You can also override this method in an abstract base class: - - .. method:: __subclasshook__(subclass) - - (Must be defined as a class method.) - - Check whether *subclass* is considered a subclass of this ABC. This means - that you can customize the behavior of ``issubclass`` further without the - need to call :meth:`register` on every class you want to consider a - subclass of the ABC. (This class method is called from the - :meth:`__subclasscheck__` method of the ABC.) - - This method should return ``True``, ``False`` or ``NotImplemented``. If - it returns ``True``, the *subclass* is considered a subclass of this ABC. - If it returns ``False``, the *subclass* is not considered a subclass of - this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual - mechanism. - - .. XXX explain the "usual mechanism" - - - For a demonstration of these concepts, look at this example ABC definition:: - - class Foo: - def __getitem__(self, index): - ... - def __len__(self): - ... - def get_iterator(self): - return iter(self) - - class MyIterable(ABC): - - @abstractmethod - def __iter__(self): - while False: - yield None - - def get_iterator(self): - return self.__iter__() - - @classmethod - def __subclasshook__(cls, C): - if cls is MyIterable: - if any("__iter__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - MyIterable.register(Foo) - - The ABC ``MyIterable`` defines the standard iterable method, - :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method - is also part of the ``MyIterable`` abstract base class, but it does not have - to be overridden in non-abstract derived classes. - - The :meth:`__subclasshook__` class method defined here says that any class - that has an :meth:`~iterator.__iter__` method in its - :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. - - Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, - even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and - :meth:`__getitem__`). Note that this will not make ``get_iterator`` - available as a method of ``Foo``, so it is provided separately. - - - - -The :mod:`abc` module also provides the following decorator: - -.. decorator:: abstractmethod - - A decorator indicating abstract methods. - - Using this decorator requires that the class's metaclass is :class:`ABCMeta` - or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods - and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`abstractmethod` may be used - to declare abstract methods for properties and descriptors. - - Dynamically adding abstract methods to a class, or attempting to modify the - abstraction status of a method or class once it is created, are not - supported. The :func:`abstractmethod` only affects subclasses derived using - regular inheritance; "virtual subclasses" registered with the ABC's - :meth:`register` method are not affected. - - When :func:`abstractmethod` is applied in combination with other method - descriptors, it should be applied as the innermost decorator, as shown in - the following usage examples:: - - class C(ABC): - @abstractmethod - def my_abstract_method(self, ...): - ... - @classmethod - @abstractmethod - def my_abstract_classmethod(cls, ...): - ... - @staticmethod - @abstractmethod - def my_abstract_staticmethod(...): - ... - - @property - @abstractmethod - def my_abstract_property(self): - ... - @my_abstract_property.setter - @abstractmethod - def my_abstract_property(self, val): - ... - - @abstractmethod - def _get_x(self): - ... - @abstractmethod - def _set_x(self, val): - ... - x = property(_get_x, _set_x) - - In order to correctly interoperate with the abstract base class machinery, - the descriptor must identify itself as abstract using - :attr:`__isabstractmethod__`. In general, this attribute should be ``True`` - if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: - - class Descriptor: - ... - @property - def __isabstractmethod__(self): - return any(getattr(f, '__isabstractmethod__', False) for - f in (self._fget, self._fset, self._fdel)) - - .. note:: - - Unlike Java abstract methods, these abstract - methods may have an implementation. This implementation can be - called via the :func:`super` mechanism from the class that - overrides it. This could be useful as an end-point for a - super-call in a framework that uses cooperative - multiple-inheritance. - - -The :mod:`abc` module also supports the following legacy decorators: - -.. decorator:: abstractclassmethod - - .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. - - This special case is deprecated, as the :func:`classmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @classmethod - @abstractmethod - def my_abstract_classmethod(cls, ...): - ... - - -.. decorator:: abstractstaticmethod - - .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. - - This special case is deprecated, as the :func:`staticmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @staticmethod - @abstractmethod - def my_abstract_staticmethod(...): - ... - - -.. decorator:: abstractproperty - - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`property`, indicating an abstract - property. - - This special case is deprecated, as the :func:`property` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @property - @abstractmethod - def my_abstract_property(self): - ... - - The above example defines a read-only property; you can also define a - read-write abstract property by appropriately marking one or more of the - underlying methods as abstract:: - - class C(ABC): - @property - def x(self): - ... - - @x.setter - @abstractmethod - def x(self, val): - ... - - If only some components are abstract, only those components need to be - updated to create a concrete property in a subclass:: - - class D(C): - @C.x.setter - def x(self, val): - ... - - -The :mod:`abc` module also provides the following functions: - -.. function:: get_cache_token() - - Returns the current abstract base class cache token. - - The token is an opaque object (that supports equality testing) identifying - the current version of the abstract base class cache for virtual subclasses. - The token changes with every call to :meth:`ABCMeta.register` on any ABC. - - .. versionadded:: 3.4 - - -.. rubric:: Footnotes - -.. [#] C++ programmers should note that Python's virtual base class - concept is not the same as C++'s. diff --git a/third_party/python/Doc/library/aifc.rst b/third_party/python/Doc/library/aifc.rst deleted file mode 100644 index 970a7aeb9..000000000 --- a/third_party/python/Doc/library/aifc.rst +++ /dev/null @@ -1,239 +0,0 @@ -:mod:`aifc` --- Read and write AIFF and AIFC files -================================================== - -.. module:: aifc - :synopsis: Read and write audio files in AIFF or AIFC format. - -**Source code:** :source:`Lib/aifc.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - --------------- - -This module provides support for reading and writing AIFF and AIFF-C files. -AIFF is Audio Interchange File Format, a format for storing digital audio -samples in a file. AIFF-C is a newer version of the format that includes the -ability to compress the audio data. - -Audio files have a number of parameters that describe the audio data. The -sampling rate or frame rate is the number of times per second the sound is -sampled. The number of channels indicate if the audio is mono, stereo, or -quadro. Each frame consists of one sample per channel. The sample size is the -size in bytes of each sample. Thus a frame consists of -``nchannels * samplesize`` bytes, and a second's worth of audio consists of -``nchannels * samplesize * framerate`` bytes. - -For example, CD quality audio has a sample size of two bytes (16 bits), uses two -channels (stereo) and has a frame rate of 44,100 frames/second. This gives a -frame size of 4 bytes (2\*2), and a second's worth occupies 2\*2\*44100 bytes -(176,400 bytes). - -Module :mod:`aifc` defines the following function: - - -.. function:: open(file, mode=None) - - Open an AIFF or AIFF-C file and return an object instance with methods that are - described below. The argument *file* is either a string naming a file or a - :term:`file object`. *mode* must be ``'r'`` or ``'rb'`` when the file must be - opened for reading, or ``'w'`` or ``'wb'`` when the file must be opened for writing. - If omitted, ``file.mode`` is used if it exists, otherwise ``'rb'`` is used. When - used for writing, the file object should be seekable, unless you know ahead of - time how many samples you are going to write in total and use - :meth:`writeframesraw` and :meth:`setnframes`. - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`with` block completes, the :meth:`~aifc.close` method is called. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - -Objects returned by :func:`.open` when a file is opened for reading have the -following methods: - - -.. method:: aifc.getnchannels() - - Return the number of audio channels (1 for mono, 2 for stereo). - - -.. method:: aifc.getsampwidth() - - Return the size in bytes of individual samples. - - -.. method:: aifc.getframerate() - - Return the sampling rate (number of audio frames per second). - - -.. method:: aifc.getnframes() - - Return the number of audio frames in the file. - - -.. method:: aifc.getcomptype() - - Return a bytes array of length 4 describing the type of compression - used in the audio file. For AIFF files, the returned value is - ``b'NONE'``. - - -.. method:: aifc.getcompname() - - Return a bytes array convertible to a human-readable description - of the type of compression used in the audio file. For AIFF files, - the returned value is ``b'not compressed'``. - - -.. method:: aifc.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: aifc.getmarkers() - - Return a list of markers in the audio file. A marker consists of a tuple of - three elements. The first is the mark ID (an integer), the second is the mark - position in frames from the beginning of the data (an integer), the third is the - name of the mark (a string). - - -.. method:: aifc.getmark(id) - - Return the tuple as described in :meth:`getmarkers` for the mark with the given - *id*. - - -.. method:: aifc.readframes(nframes) - - Read and return the next *nframes* frames from the audio file. The returned - data is a string containing for each frame the uncompressed samples of all - channels. - - -.. method:: aifc.rewind() - - Rewind the read pointer. The next :meth:`readframes` will start from the - beginning. - - -.. method:: aifc.setpos(pos) - - Seek to the specified frame number. - - -.. method:: aifc.tell() - - Return the current frame number. - - -.. method:: aifc.close() - - Close the AIFF file. After calling this method, the object can no longer be - used. - -Objects returned by :func:`.open` when a file is opened for writing have all the -above methods, except for :meth:`readframes` and :meth:`setpos`. In addition -the following methods exist. The :meth:`get\*` methods can only be called after -the corresponding :meth:`set\*` methods have been called. Before the first -:meth:`writeframes` or :meth:`writeframesraw`, all parameters except for the -number of frames must be filled in. - - -.. method:: aifc.aiff() - - Create an AIFF file. The default is that an AIFF-C file is created, unless the - name of the file ends in ``'.aiff'`` in which case the default is an AIFF file. - - -.. method:: aifc.aifc() - - Create an AIFF-C file. The default is that an AIFF-C file is created, unless - the name of the file ends in ``'.aiff'`` in which case the default is an AIFF - file. - - -.. method:: aifc.setnchannels(nchannels) - - Specify the number of channels in the audio file. - - -.. method:: aifc.setsampwidth(width) - - Specify the size in bytes of audio samples. - - -.. method:: aifc.setframerate(rate) - - Specify the sampling frequency in frames per second. - - -.. method:: aifc.setnframes(nframes) - - Specify the number of frames that are to be written to the audio file. If this - parameter is not set, or not set correctly, the file needs to support seeking. - - -.. method:: aifc.setcomptype(type, name) - - .. index:: - single: u-LAW - single: A-LAW - single: G.722 - - Specify the compression type. If not specified, the audio data will - not be compressed. In AIFF files, compression is not possible. - The name parameter should be a human-readable description of the - compression type as a bytes array, the type parameter should be a - bytes array of length 4. Currently the following compression types - are supported: ``b'NONE'``, ``b'ULAW'``, ``b'ALAW'``, ``b'G722'``. - - -.. method:: aifc.setparams(nchannels, sampwidth, framerate, comptype, compname) - - Set all the above parameters at once. The argument is a tuple consisting of the - various parameters. This means that it is possible to use the result of a - :meth:`getparams` call as argument to :meth:`setparams`. - - -.. method:: aifc.setmark(id, pos, name) - - Add a mark with the given id (larger than 0), and the given name at the given - position. This method can be called at any time before :meth:`close`. - - -.. method:: aifc.tell() - - Return the current write position in the output file. Useful in combination - with :meth:`setmark`. - - -.. method:: aifc.writeframes(data) - - Write data to the output file. This method can only be called after the audio - file parameters have been set. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.writeframesraw(data) - - Like :meth:`writeframes`, except that the header of the audio file is not - updated. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.close() - - Close the AIFF file. The header of the file is updated to reflect the actual - size of the audio data. After calling this method, the object can no longer be - used. - diff --git a/third_party/python/Doc/library/allos.rst b/third_party/python/Doc/library/allos.rst deleted file mode 100644 index f7105d8af..000000000 --- a/third_party/python/Doc/library/allos.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _allos: - -********************************* -Generic Operating System Services -********************************* - -The modules described in this chapter provide interfaces to operating system -features that are available on (almost) all operating systems, such as files and -a clock. The interfaces are generally modeled after the Unix or C interfaces, -but they are available on most other systems as well. Here's an overview: - - -.. toctree:: - - os.rst - io.rst - time.rst - argparse.rst - getopt.rst - logging.rst - logging.config.rst - logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst - platform.rst - errno.rst - ctypes.rst diff --git a/third_party/python/Doc/library/archiving.rst b/third_party/python/Doc/library/archiving.rst deleted file mode 100644 index c9284949a..000000000 --- a/third_party/python/Doc/library/archiving.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _archiving: - -****************************** -Data Compression and Archiving -****************************** - -The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format -archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` -module. - - -.. toctree:: - - zlib.rst - gzip.rst - bz2.rst - lzma.rst - zipfile.rst - tarfile.rst diff --git a/third_party/python/Doc/library/argparse.rst b/third_party/python/Doc/library/argparse.rst deleted file mode 100644 index 5196202c4..000000000 --- a/third_party/python/Doc/library/argparse.rst +++ /dev/null @@ -1,2048 +0,0 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== - -.. module:: argparse - :synopsis: Command-line option and argument parsing library. - -.. moduleauthor:: Steven Bethard -.. sectionauthor:: Steven Bethard - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/argparse.py` - --------------- - -.. sidebar:: Tutorial - - This page contains the API reference information. For a more gentle - introduction to Python command-line parsing, have a look at the - :ref:`argparse tutorial `. - -The :mod:`argparse` module makes it easy to write user-friendly command-line -interfaces. The program defines what arguments it requires, and :mod:`argparse` -will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` -module also automatically generates help and usage messages and issues errors -when users give the program invalid arguments. - - -Example -------- - -The following code is a Python program that takes a list of integers and -produces either the sum or the max:: - - import argparse - - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument('integers', metavar='N', type=int, nargs='+', - help='an integer for the accumulator') - parser.add_argument('--sum', dest='accumulate', action='store_const', - const=sum, default=max, - help='sum the integers (default: find the max)') - - args = parser.parse_args() - print(args.accumulate(args.integers)) - -Assuming the Python code above is saved into a file called ``prog.py``, it can -be run at the command line and provides useful help messages: - -.. code-block:: shell-session - - $ python prog.py -h - usage: prog.py [-h] [--sum] N [N ...] - - Process some integers. - - positional arguments: - N an integer for the accumulator - - optional arguments: - -h, --help show this help message and exit - --sum sum the integers (default: find the max) - -When run with the appropriate arguments, it prints either the sum or the max of -the command-line integers: - -.. code-block:: shell-session - - $ python prog.py 1 2 3 4 - 4 - - $ python prog.py 1 2 3 4 --sum - 10 - -If invalid arguments are passed in, it will issue an error: - -.. code-block:: shell-session - - $ python prog.py a b c - usage: prog.py [-h] [--sum] N [N ...] - prog.py: error: argument N: invalid int value: 'a' - -The following sections walk you through this example. - - -Creating a parser -^^^^^^^^^^^^^^^^^ - -The first step in using the :mod:`argparse` is creating an -:class:`ArgumentParser` object:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - -The :class:`ArgumentParser` object will hold all the information necessary to -parse the command line into Python data types. - - -Adding arguments -^^^^^^^^^^^^^^^^ - -Filling an :class:`ArgumentParser` with information about program arguments is -done by making calls to the :meth:`~ArgumentParser.add_argument` method. -Generally, these calls tell the :class:`ArgumentParser` how to take the strings -on the command line and turn them into objects. This information is stored and -used when :meth:`~ArgumentParser.parse_args` is called. For example:: - - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.add_argument('--sum', dest='accumulate', action='store_const', - ... const=sum, default=max, - ... help='sum the integers (default: find the max)') - -Later, calling :meth:`~ArgumentParser.parse_args` will return an object with -two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute -will be a list of one or more ints, and the ``accumulate`` attribute will be -either the :func:`sum` function, if ``--sum`` was specified at the command line, -or the :func:`max` function if it was not. - - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` parses arguments through the -:meth:`~ArgumentParser.parse_args` method. This will inspect the command line, -convert each argument to the appropriate type and then invoke the appropriate action. -In most cases, this means a simple :class:`Namespace` object will be built up from -attributes parsed out of the command line:: - - >>> parser.parse_args(['--sum', '7', '-1', '42']) - Namespace(accumulate=, integers=[7, -1, 42]) - -In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no -arguments, and the :class:`ArgumentParser` will automatically determine the -command-line arguments from :data:`sys.argv`. - - -ArgumentParser objects ----------------------- - -.. class:: ArgumentParser(prog=None, usage=None, description=None, \ - epilog=None, parents=[], \ - formatter_class=argparse.HelpFormatter, \ - prefix_chars='-', fromfile_prefix_chars=None, \ - argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True) - - Create a new :class:`ArgumentParser` object. All parameters should be passed - as keyword arguments. Each parameter has its own more detailed description - below, but in short they are: - - * prog_ - The name of the program (default: ``sys.argv[0]``) - - * usage_ - The string describing the program usage (default: generated from - arguments added to parser) - - * description_ - Text to display before the argument help (default: none) - - * epilog_ - Text to display after the argument help (default: none) - - * parents_ - A list of :class:`ArgumentParser` objects whose arguments should - also be included - - * formatter_class_ - A class for customizing the help output - - * prefix_chars_ - The set of characters that prefix optional arguments - (default: '-') - - * fromfile_prefix_chars_ - The set of characters that prefix files from - which additional arguments should be read (default: ``None``) - - * argument_default_ - The global default value for arguments - (default: ``None``) - - * conflict_handler_ - The strategy for resolving conflicting optionals - (usually unnecessary) - - * add_help_ - Add a ``-h/--help`` option to the parser (default: ``True``) - - * allow_abbrev_ - Allows long options to be abbreviated if the - abbreviation is unambiguous. (default: ``True``) - - .. versionchanged:: 3.5 - *allow_abbrev* parameter was added. - -The following sections describe how each of these are used. - - -prog -^^^^ - -By default, :class:`ArgumentParser` objects use ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir/myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.print_help() - usage: myprogram [-h] - - optional arguments: - -h, --help show this help message and exit - -Note that the program name, whether determined from ``sys.argv[0]`` or from the -``prog=`` argument, is available to help messages using the ``%(prog)s`` format -specifier. - -:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.add_argument('--foo', help='foo of the %(prog)s program') - >>> parser.print_help() - usage: myprogram [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo of the myprogram program - - -usage -^^^^^ - -By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: - - >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [options] - - positional arguments: - bar bar help - - optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The ``%(prog)s`` format specifier is available to fill in the program name in -your usage messages. - - -description -^^^^^^^^^^^ - -Most calls to the :class:`ArgumentParser` constructor will use the -``description=`` keyword argument. This argument gives a brief description of -what the program does and how it works. In help messages, the description is -displayed between the command-line usage string and the help messages for the -various arguments:: - - >>> parser = argparse.ArgumentParser(description='A foo that bars') - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - optional arguments: - -h, --help show this help message and exit - -By default, the description will be line-wrapped so that it fits within the -given space. To change this behavior, see the formatter_class_ argument. - - -epilog -^^^^^^ - -Some programs like to display additional description of the program after the -description of the arguments. Such text can be specified using the ``epilog=`` -argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser( - ... description='A foo that bars', - ... epilog="And that's how you'd foo a bar") - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - optional arguments: - -h, --help show this help message and exit - - And that's how you'd foo a bar - -As with the description_ argument, the ``epilog=`` text is by default -line-wrapped, but this behavior can be adjusted with the formatter_class_ -argument to :class:`ArgumentParser`. - - -parents -^^^^^^^ - -Sometimes, several parsers share a common set of arguments. Rather than -repeating the definitions of these arguments, a single parser with all the -shared arguments and passed to ``parents=`` argument to :class:`ArgumentParser` -can be used. The ``parents=`` argument takes a list of :class:`ArgumentParser` -objects, collects all the positional and optional actions from them, and adds -these actions to the :class:`ArgumentParser` object being constructed:: - - >>> parent_parser = argparse.ArgumentParser(add_help=False) - >>> parent_parser.add_argument('--parent', type=int) - - >>> foo_parser = argparse.ArgumentParser(parents=[parent_parser]) - >>> foo_parser.add_argument('foo') - >>> foo_parser.parse_args(['--parent', '2', 'XXX']) - Namespace(foo='XXX', parent=2) - - >>> bar_parser = argparse.ArgumentParser(parents=[parent_parser]) - >>> bar_parser.add_argument('--bar') - >>> bar_parser.parse_args(['--bar', 'YYY']) - Namespace(bar='YYY', parent=None) - -Note that most parent parsers will specify ``add_help=False``. Otherwise, the -:class:`ArgumentParser` will see two ``-h/--help`` options (one in the parent -and one in the child) and raise an error. - -.. note:: - You must fully initialize the parsers before passing them via ``parents=``. - If you change the parent parsers after the child parser, those changes will - not be reflected in the child. - - -formatter_class -^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` objects allow the help formatting to be customized by -specifying an alternate formatting class. Currently, there are four such -classes: - -.. class:: RawDescriptionHelpFormatter - RawTextHelpFormatter - ArgumentDefaultsHelpFormatter - MetavarTypeHelpFormatter - -:class:`RawDescriptionHelpFormatter` and :class:`RawTextHelpFormatter` give -more control over how textual descriptions are displayed. -By default, :class:`ArgumentParser` objects line-wrap the description_ and -epilog_ texts in command-line help messages:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... description='''this description - ... was indented weird - ... but that is okay''', - ... epilog=''' - ... likewise for this epilog whose whitespace will - ... be cleaned up and whose words will be wrapped - ... across a couple lines''') - >>> parser.print_help() - usage: PROG [-h] - - this description was indented weird but that is okay - - optional arguments: - -h, --help show this help message and exit - - likewise for this epilog whose whitespace will be cleaned up and whose words - will be wrapped across a couple lines - -Passing :class:`RawDescriptionHelpFormatter` as ``formatter_class=`` -indicates that description_ and epilog_ are already correctly formatted and -should not be line-wrapped:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.RawDescriptionHelpFormatter, - ... description=textwrap.dedent('''\ - ... Please do not mess up this text! - ... -------------------------------- - ... I have indented it - ... exactly the way - ... I want it - ... ''')) - >>> parser.print_help() - usage: PROG [-h] - - Please do not mess up this text! - -------------------------------- - I have indented it - exactly the way - I want it - - optional arguments: - -h, --help show this help message and exit - -:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. However, multiple new lines are replaced with -one. If you wish to preserve multiple blank lines, add spaces between the -newlines. - -:class:`ArgumentDefaultsHelpFormatter` automatically adds information about -default values to each of the argument help messages:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.ArgumentDefaultsHelpFormatter) - >>> parser.add_argument('--foo', type=int, default=42, help='FOO!') - >>> parser.add_argument('bar', nargs='*', default=[1, 2, 3], help='BAR!') - >>> parser.print_help() - usage: PROG [-h] [--foo FOO] [bar [bar ...]] - - positional arguments: - bar BAR! (default: [1, 2, 3]) - - optional arguments: - -h, --help show this help message and exit - --foo FOO FOO! (default: 42) - -:class:`MetavarTypeHelpFormatter` uses the name of the type_ argument for each -argument as the display name for its values (rather than using the dest_ -as the regular formatter does):: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.MetavarTypeHelpFormatter) - >>> parser.add_argument('--foo', type=int) - >>> parser.add_argument('bar', type=float) - >>> parser.print_help() - usage: PROG [-h] [--foo int] float - - positional arguments: - float - - optional arguments: - -h, --help show this help message and exit - --foo int - - -prefix_chars -^^^^^^^^^^^^ - -Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. -Parsers that need to support different or additional prefix -characters, e.g. for options -like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') - >>> parser.add_argument('+f') - >>> parser.add_argument('++bar') - >>> parser.parse_args('+f X ++bar Y'.split()) - Namespace(bar='Y', f='X') - -The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of -characters that does not include ``-`` will cause ``-f/--foo`` options to be -disallowed. - - -fromfile_prefix_chars -^^^^^^^^^^^^^^^^^^^^^ - -Sometimes, for example when dealing with a particularly long argument lists, it -may make sense to keep the list of arguments in a file rather than typing it out -at the command line. If the ``fromfile_prefix_chars=`` argument is given to the -:class:`ArgumentParser` constructor, then arguments that start with any of the -specified characters will be treated as files, and will be replaced by the -arguments they contain. For example:: - - >>> with open('args.txt', 'w') as fp: - ... fp.write('-f\nbar') - >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') - >>> parser.add_argument('-f') - >>> parser.parse_args(['-f', 'foo', '@args.txt']) - Namespace(f='bar') - -Arguments read from a file must by default be one per line (but see also -:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they -were in the same place as the original file referencing argument on the command -line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` -is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. - -The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that -arguments will never be treated as file references. - - -argument_default -^^^^^^^^^^^^^^^^ - -Generally, argument defaults are specified either by passing a default to -:meth:`~ArgumentParser.add_argument` or by calling the -:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value -pairs. Sometimes however, it may be useful to specify a single parser-wide -default for arguments. This can be accomplished by passing the -``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, -to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` -calls, we supply ``argument_default=SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar', nargs='?') - >>> parser.parse_args(['--foo', '1', 'BAR']) - Namespace(bar='BAR', foo='1') - >>> parser.parse_args([]) - Namespace() - -.. _allow_abbrev: - -allow_abbrev -^^^^^^^^^^^^ - -Normally, when you pass an argument list to the -:meth:`~ArgumentParser.parse_args` method of an :class:`ArgumentParser`, -it :ref:`recognizes abbreviations ` of long options. - -This feature can be disabled by setting ``allow_abbrev`` to ``False``:: - - >>> parser = argparse.ArgumentParser(prog='PROG', allow_abbrev=False) - >>> parser.add_argument('--foobar', action='store_true') - >>> parser.add_argument('--foonley', action='store_false') - >>> parser.parse_args(['--foon']) - usage: PROG [-h] [--foobar] [--foonley] - PROG: error: unrecognized arguments: --foon - -.. versionadded:: 3.5 - - -conflict_handler -^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` objects do not allow two actions with the same option -string. By default, :class:`ArgumentParser` objects raise an exception if an -attempt is made to create an argument with an option string that is already in -use:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-f', '--foo', help='old foo help') - >>> parser.add_argument('--foo', help='new foo help') - Traceback (most recent call last): - .. - ArgumentError: argument --foo: conflicting option string(s): --foo - -Sometimes (e.g. when using parents_) it may be useful to simply override any -older arguments with the same option string. To get this behavior, the value -``'resolve'`` can be supplied to the ``conflict_handler=`` argument of -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', conflict_handler='resolve') - >>> parser.add_argument('-f', '--foo', help='old foo help') - >>> parser.add_argument('--foo', help='new foo help') - >>> parser.print_help() - usage: PROG [-h] [-f FOO] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - -f FOO old foo help - --foo FOO new foo help - -Note that :class:`ArgumentParser` objects only remove an action if all of its -option strings are overridden. So, in the example above, the old ``-f/--foo`` -action is retained as the ``-f`` action, because only the ``--foo`` option -string was overridden. - - -add_help -^^^^^^^^ - -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed: - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -Occasionally, it may be useful to disable the addition of this help option. -This can be achieved by passing ``False`` as the ``add_help=`` argument to -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> parser.add_argument('--foo', help='foo help') - >>> parser.print_help() - usage: PROG [--foo FOO] - - optional arguments: - --foo FOO foo help - -The help option is typically ``-h/--help``. The exception to this is -if the ``prefix_chars=`` is specified and does not include ``-``, in -which case ``-h`` and ``--help`` are not valid options. In -this case, the first character in ``prefix_chars`` is used to prefix -the help options:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') - >>> parser.print_help() - usage: PROG [+h] - - optional arguments: - +h, ++help show this help message and exit - - -The add_argument() method -------------------------- - -.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ - [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) - - Define how a single command-line argument should be parsed. Each parameter - has its own more detailed description below, but in short they are: - - * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo`` - or ``-f, --foo``. - - * action_ - The basic type of action to be taken when this argument is - encountered at the command line. - - * nargs_ - The number of command-line arguments that should be consumed. - - * const_ - A constant value required by some action_ and nargs_ selections. - - * default_ - The value produced if the argument is absent from the - command line. - - * type_ - The type to which the command-line argument should be converted. - - * choices_ - A container of the allowable values for the argument. - - * required_ - Whether or not the command-line option may be omitted - (optionals only). - - * help_ - A brief description of what the argument does. - - * metavar_ - A name for the argument in usage messages. - - * dest_ - The name of the attribute to be added to the object returned by - :meth:`parse_args`. - -The following sections describe how each of these are used. - - -name or flags -^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.add_argument` method must know whether an optional -argument, like ``-f`` or ``--foo``, or a positional argument, like a list of -filenames, is expected. The first arguments passed to -:meth:`~ArgumentParser.add_argument` must therefore be either a series of -flags, or a simple argument name. For example, an optional argument could -be created like:: - - >>> parser.add_argument('-f', '--foo') - -while a positional argument could be created like:: - - >>> parser.add_argument('bar') - -When :meth:`~ArgumentParser.parse_args` is called, optional arguments will be -identified by the ``-`` prefix, and the remaining arguments will be assumed to -be positional:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-f', '--foo') - >>> parser.add_argument('bar') - >>> parser.parse_args(['BAR']) - Namespace(bar='BAR', foo=None) - >>> parser.parse_args(['BAR', '--foo', 'FOO']) - Namespace(bar='BAR', foo='FOO') - >>> parser.parse_args(['--foo', 'FOO']) - usage: PROG [-h] [-f FOO] bar - PROG: error: the following arguments are required: bar - - -action -^^^^^^ - -:class:`ArgumentParser` objects associate command-line arguments with actions. These -actions can do just about anything with the command-line arguments associated with -them, though most actions simply add an attribute to the object returned by -:meth:`~ArgumentParser.parse_args`. The ``action`` keyword argument specifies -how the command-line arguments should be handled. The supplied actions are: - -* ``'store'`` - This just stores the argument's value. This is the default - action. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args('--foo 1'.split()) - Namespace(foo='1') - -* ``'store_const'`` - This stores the value specified by the const_ keyword - argument. The ``'store_const'`` action is most commonly used with - optional arguments that specify some sort of flag. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_const', const=42) - >>> parser.parse_args(['--foo']) - Namespace(foo=42) - -* ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('--bar', action='store_false') - >>> parser.add_argument('--baz', action='store_false') - >>> parser.parse_args('--foo --bar'.split()) - Namespace(foo=True, bar=False, baz=True) - -* ``'append'`` - This stores a list, and appends each argument value to the - list. This is useful to allow an option to be specified multiple times. - Example usage:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') - >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) - -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list. (Note that the const_ keyword - argument defaults to ``None``.) The ``'append_const'`` action is typically - useful when multiple arguments need to store constants to the same list. For - example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--str', dest='types', action='append_const', const=str) - >>> parser.add_argument('--int', dest='types', action='append_const', const=int) - >>> parser.parse_args('--str --int'.split()) - Namespace(types=[, ]) - -* ``'count'`` - This counts the number of times a keyword argument occurs. For - example, this is useful for increasing verbosity levels:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--verbose', '-v', action='count') - >>> parser.parse_args(['-vvv']) - Namespace(verbose=3) - -* ``'help'`` - This prints a complete help message for all the options in the - current parser and then exits. By default a help action is automatically - added to the parser. See :class:`ArgumentParser` for details of how the - output is created. - -* ``'version'`` - This expects a ``version=`` keyword argument in the - :meth:`~ArgumentParser.add_argument` call, and prints version information - and exits when invoked:: - - >>> import argparse - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--version', action='version', version='%(prog)s 2.0') - >>> parser.parse_args(['--version']) - PROG 2.0 - -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The recommended way to do -this is to extend :class:`Action`, overriding the ``__call__`` method -and optionally the ``__init__`` method. - -An example of a custom action:: - - >>> class FooAction(argparse.Action): - ... def __init__(self, option_strings, dest, nargs=None, **kwargs): - ... if nargs is not None: - ... raise ValueError("nargs not allowed") - ... super(FooAction, self).__init__(option_strings, dest, **kwargs) - ... def __call__(self, parser, namespace, values, option_string=None): - ... print('%r %r %r' % (namespace, values, option_string)) - ... setattr(namespace, self.dest, values) - ... - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=FooAction) - >>> parser.add_argument('bar', action=FooAction) - >>> args = parser.parse_args('1 --foo 2'.split()) - Namespace(bar=None, foo=None) '1' None - Namespace(bar='1', foo=None) '2' '--foo' - >>> args - Namespace(bar='1', foo='2') - -For more details, see :class:`Action`. - -nargs -^^^^^ - -ArgumentParser objects usually associate a single command-line argument with a -single action to be taken. The ``nargs`` keyword argument associates a -different number of command-line arguments with a single action. The supported -values are: - -* ``N`` (an integer). ``N`` arguments from the command line will be gathered - together into a list. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs=2) - >>> parser.add_argument('bar', nargs=1) - >>> parser.parse_args('c --foo a b'.split()) - Namespace(bar=['c'], foo=['a', 'b']) - - Note that ``nargs=1`` produces a list of one item. This is different from - the default, in which the item is produced by itself. - -.. index:: single: ? (question mark); in argparse module - -* ``'?'``. One argument will be consumed from the command line if possible, and - produced as a single item. If no command-line argument is present, the value from - default_ will be produced. Note that for optional arguments, there is an - additional case - the option string is present but not followed by a - command-line argument. In this case the value from const_ will be produced. Some - examples to illustrate this:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs='?', const='c', default='d') - >>> parser.add_argument('bar', nargs='?', default='d') - >>> parser.parse_args(['XX', '--foo', 'YY']) - Namespace(bar='XX', foo='YY') - >>> parser.parse_args(['XX', '--foo']) - Namespace(bar='XX', foo='c') - >>> parser.parse_args([]) - Namespace(bar='d', foo='d') - - One of the more common uses of ``nargs='?'`` is to allow optional input and - output files:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), - ... default=sys.stdin) - >>> parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), - ... default=sys.stdout) - >>> parser.parse_args(['input.txt', 'output.txt']) - Namespace(infile=<_io.TextIOWrapper name='input.txt' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='output.txt' encoding='UTF-8'>) - >>> parser.parse_args([]) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>) - -.. index:: single: * (asterisk); in argparse module - -* ``'*'``. All command-line arguments present are gathered into a list. Note that - it generally doesn't make much sense to have more than one positional argument - with ``nargs='*'``, but multiple optional arguments with ``nargs='*'`` is - possible. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs='*') - >>> parser.add_argument('--bar', nargs='*') - >>> parser.add_argument('baz', nargs='*') - >>> parser.parse_args('a b --foo x y --bar 1 2'.split()) - Namespace(bar=['1', '2'], baz=['a', 'b'], foo=['x', 'y']) - -.. index:: single: + (plus); in argparse module - -* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a - list. Additionally, an error message will be generated if there wasn't at - least one command-line argument present. For example:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', nargs='+') - >>> parser.parse_args(['a', 'b']) - Namespace(foo=['a', 'b']) - >>> parser.parse_args([]) - usage: PROG [-h] foo [foo ...] - PROG: error: the following arguments are required: foo - -.. _`argparse.REMAINDER`: - -* ``argparse.REMAINDER``. All the remaining command-line arguments are gathered - into a list. This is commonly useful for command line utilities that dispatch - to other command line utilities:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo') - >>> parser.add_argument('command') - >>> parser.add_argument('args', nargs=argparse.REMAINDER) - >>> print(parser.parse_args('--foo B cmd --arg1 XX ZZ'.split())) - Namespace(args=['--arg1', 'XX', 'ZZ'], command='cmd', foo='B') - -If the ``nargs`` keyword argument is not provided, the number of arguments consumed -is determined by the action_. Generally this means a single command-line argument -will be consumed and a single item (not a list) will be produced. - - -const -^^^^^ - -The ``const`` argument of :meth:`~ArgumentParser.add_argument` is used to hold -constant values that are not read from the command line but are required for -the various :class:`ArgumentParser` actions. The two most common uses of it are: - -* When :meth:`~ArgumentParser.add_argument` is called with - ``action='store_const'`` or ``action='append_const'``. These actions add the - ``const`` value to one of the attributes of the object returned by - :meth:`~ArgumentParser.parse_args`. See the action_ description for examples. - -* When :meth:`~ArgumentParser.add_argument` is called with option strings - (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional - argument that can be followed by zero or one command-line arguments. - When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed instead. - See the nargs_ description for examples. - -With the ``'store_const'`` and ``'append_const'`` actions, the ``const`` -keyword argument must be given. For other actions, it defaults to ``None``. - - -default -^^^^^^^ - -All optional arguments and some positional arguments may be omitted at the -command line. The ``default`` keyword argument of -:meth:`~ArgumentParser.add_argument`, whose value defaults to ``None``, -specifies what value should be used if the command-line argument is not present. -For optional arguments, the ``default`` value is used when the option string -was not present at the command line:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default=42) - >>> parser.parse_args(['--foo', '2']) - Namespace(foo='2') - >>> parser.parse_args([]) - Namespace(foo=42) - -If the ``default`` value is a string, the parser parses the value as if it -were a command-line argument. In particular, the parser applies any type_ -conversion argument, if provided, before setting the attribute on the -:class:`Namespace` return value. Otherwise, the parser uses the value as is:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--length', default='10', type=int) - >>> parser.add_argument('--width', default=10.5, type=int) - >>> parser.parse_args() - Namespace(length=10, width=10.5) - -For positional arguments with nargs_ equal to ``?`` or ``*``, the ``default`` value -is used when no command-line argument was present:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', nargs='?', default=42) - >>> parser.parse_args(['a']) - Namespace(foo='a') - >>> parser.parse_args([]) - Namespace(foo=42) - - -Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the -command-line argument was not present:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default=argparse.SUPPRESS) - >>> parser.parse_args([]) - Namespace() - >>> parser.parse_args(['--foo', '1']) - Namespace(foo='1') - - -type -^^^^ - -By default, :class:`ArgumentParser` objects read command-line arguments in as simple -strings. However, quite often the command-line string should instead be -interpreted as another type, like a :class:`float` or :class:`int`. The -``type`` keyword argument of :meth:`~ArgumentParser.add_argument` allows any -necessary type-checking and type conversions to be performed. Common built-in -types and functions can be used directly as the value of the ``type`` argument:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', type=int) - >>> parser.add_argument('bar', type=open) - >>> parser.parse_args('2 temp.txt'.split()) - Namespace(bar=<_io.TextIOWrapper name='temp.txt' encoding='UTF-8'>, foo=2) - -See the section on the default_ keyword argument for information on when the -``type`` argument is applied to default arguments. - -To ease the use of various types of files, the argparse module provides the -factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and -``errors=`` arguments of the :func:`open` function. For example, -``FileType('w')`` can be used to create a writable file:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('bar', type=argparse.FileType('w')) - >>> parser.parse_args(['out.txt']) - Namespace(bar=<_io.TextIOWrapper name='out.txt' encoding='UTF-8'>) - -``type=`` can take any callable that takes a single string argument and returns -the converted value:: - - >>> def perfect_square(string): - ... value = int(string) - ... sqrt = math.sqrt(value) - ... if sqrt != int(sqrt): - ... msg = "%r is not a perfect square" % string - ... raise argparse.ArgumentTypeError(msg) - ... return value - ... - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=perfect_square) - >>> parser.parse_args(['9']) - Namespace(foo=9) - >>> parser.parse_args(['7']) - usage: PROG [-h] foo - PROG: error: argument foo: '7' is not a perfect square - -The choices_ keyword argument may be more convenient for type checkers that -simply check against a range of values:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=int, choices=range(5, 10)) - >>> parser.parse_args(['7']) - Namespace(foo=7) - >>> parser.parse_args(['11']) - usage: PROG [-h] {5,6,7,8,9} - PROG: error: argument foo: invalid choice: 11 (choose from 5, 6, 7, 8, 9) - -See the choices_ section for more details. - - -choices -^^^^^^^ - -Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword -argument to :meth:`~ArgumentParser.add_argument`. When the command line is -parsed, argument values will be checked, and an error message will be displayed -if the argument was not one of the acceptable values:: - - >>> parser = argparse.ArgumentParser(prog='game.py') - >>> parser.add_argument('move', choices=['rock', 'paper', 'scissors']) - >>> parser.parse_args(['rock']) - Namespace(move='rock') - >>> parser.parse_args(['fire']) - usage: game.py [-h] {rock,paper,scissors} - game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', - 'paper', 'scissors') - -Note that inclusion in the *choices* container is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -container should match the type_ specified:: - - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) - -Any object that supports the ``in`` operator can be passed as the *choices* -value, so :class:`dict` objects, :class:`set` objects, custom containers, -etc. are all supported. - - -required -^^^^^^^^ - -In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar`` -indicate *optional* arguments, which can always be omitted at the command line. -To make an option *required*, ``True`` can be specified for the ``required=`` -keyword argument to :meth:`~ArgumentParser.add_argument`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', required=True) - >>> parser.parse_args(['--foo', 'BAR']) - Namespace(foo='BAR') - >>> parser.parse_args([]) - usage: argparse.py [-h] [--foo FOO] - argparse.py: error: option --foo is required - -As the example shows, if an option is marked as ``required``, -:meth:`~ArgumentParser.parse_args` will report an error if that option is not -present at the command line. - -.. note:: - - Required options are generally considered bad form because users expect - *options* to be *optional*, and thus they should be avoided when possible. - - -help -^^^^ - -The ``help`` value is a string containing a brief description of the argument. -When a user requests help (usually by using ``-h`` or ``--help`` at the -command line), these ``help`` descriptions will be displayed with each -argument:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', action='store_true', - ... help='foo the bars before frobbling') - >>> parser.add_argument('bar', nargs='+', - ... help='one of the bars to be frobbled') - >>> parser.parse_args(['-h']) - usage: frobble [-h] [--foo] bar [bar ...] - - positional arguments: - bar one of the bars to be frobbled - - optional arguments: - -h, --help show this help message and exit - --foo foo the bars before frobbling - -The ``help`` strings can include various format specifiers to avoid repetition -of things like the program name or the argument default_. The available -specifiers include the program name, ``%(prog)s`` and most keyword arguments to -:meth:`~ArgumentParser.add_argument`, e.g. ``%(default)s``, ``%(type)s``, etc.:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('bar', nargs='?', type=int, default=42, - ... help='the bar to %(prog)s (default: %(default)s)') - >>> parser.print_help() - usage: frobble [-h] [bar] - - positional arguments: - bar the bar to frobble (default: 42) - - optional arguments: - -h, --help show this help message and exit - -As the help string supports %-formatting, if you want a literal ``%`` to appear -in the help string, you must escape it as ``%%``. - -:mod:`argparse` supports silencing the help entry for certain options, by -setting the ``help`` value to ``argparse.SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', help=argparse.SUPPRESS) - >>> parser.print_help() - usage: frobble [-h] - - optional arguments: - -h, --help show this help message and exit - - -metavar -^^^^^^^ - -When :class:`ArgumentParser` generates help messages, it needs some way to refer -to each expected argument. By default, ArgumentParser objects use the dest_ -value as the "name" of each object. By default, for positional argument -actions, the dest_ value is used directly, and for optional argument actions, -the dest_ value is uppercased. So, a single positional argument with -``dest='bar'`` will be referred to as ``bar``. A single -optional argument ``--foo`` that should be followed by a single command-line argument -will be referred to as ``FOO``. An example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar') - >>> parser.parse_args('X --foo Y'.split()) - Namespace(bar='X', foo='Y') - >>> parser.print_help() - usage: [-h] [--foo FOO] bar - - positional arguments: - bar - - optional arguments: - -h, --help show this help message and exit - --foo FOO - -An alternative name can be specified with ``metavar``:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', metavar='YYY') - >>> parser.add_argument('bar', metavar='XXX') - >>> parser.parse_args('X --foo Y'.split()) - Namespace(bar='X', foo='Y') - >>> parser.print_help() - usage: [-h] [--foo YYY] XXX - - positional arguments: - XXX - - optional arguments: - -h, --help show this help message and exit - --foo YYY - -Note that ``metavar`` only changes the *displayed* name - the name of the -attribute on the :meth:`~ArgumentParser.parse_args` object is still determined -by the dest_ value. - -Different values of ``nargs`` may cause the metavar to be used multiple times. -Providing a tuple to ``metavar`` specifies a different display for each of the -arguments:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x', nargs=2) - >>> parser.add_argument('--foo', nargs=2, metavar=('bar', 'baz')) - >>> parser.print_help() - usage: PROG [-h] [-x X X] [--foo bar baz] - - optional arguments: - -h, --help show this help message and exit - -x X X - --foo bar baz - - -dest -^^^^ - -Most :class:`ArgumentParser` actions add some value as an attribute of the -object returned by :meth:`~ArgumentParser.parse_args`. The name of this -attribute is determined by the ``dest`` keyword argument of -:meth:`~ArgumentParser.add_argument`. For positional argument actions, -``dest`` is normally supplied as the first argument to -:meth:`~ArgumentParser.add_argument`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('bar') - >>> parser.parse_args(['XXX']) - Namespace(bar='XXX') - -For optional argument actions, the value of ``dest`` is normally inferred from -the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from -the first short option string by stripping the initial ``-`` character. Any -internal ``-`` characters will be converted to ``_`` characters to make sure -the string is a valid attribute name. The examples below illustrate this -behavior:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('-f', '--foo-bar', '--foo') - >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') - -``dest`` allows a custom attribute name to be provided:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', dest='bar') - >>> parser.parse_args('--foo XXX'.split()) - Namespace(bar='XXX') - -Action classes -^^^^^^^^^^^^^^ - -Action classes implement the Action API, a callable which returns a callable -which processes arguments from the command-line. Any object which follows -this API may be passed as the ``action`` parameter to -:meth:`add_argument`. - -.. class:: Action(option_strings, dest, nargs=None, const=None, default=None, \ - type=None, choices=None, required=False, help=None, \ - metavar=None) - -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. - -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. - -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: - -* ``parser`` - The ArgumentParser object which contains this action. - -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. - -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. - -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. - -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. - - -The parse_args() method ------------------------ - -.. method:: ArgumentParser.parse_args(args=None, namespace=None) - - Convert argument strings to objects and assign them as attributes of the - namespace. Return the populated namespace. - - Previous calls to :meth:`add_argument` determine exactly what objects are - created and how they are assigned. See the documentation for - :meth:`add_argument` for details. - - * args_ - List of strings to parse. The default is taken from - :data:`sys.argv`. - - * namespace_ - An object to take the attributes. The default is a new empty - :class:`Namespace` object. - - -Option value syntax -^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method supports several ways of -specifying the value of an option (if it takes one). In the simplest case, the -option and its value are passed as two separate arguments:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x') - >>> parser.add_argument('--foo') - >>> parser.parse_args(['-x', 'X']) - Namespace(foo=None, x='X') - >>> parser.parse_args(['--foo', 'FOO']) - Namespace(foo='FOO', x=None) - -For long options (options with names longer than a single character), the option -and value can also be passed as a single command-line argument, using ``=`` to -separate them:: - - >>> parser.parse_args(['--foo=FOO']) - Namespace(foo='FOO', x=None) - -For short options (options only one character long), the option and its value -can be concatenated:: - - >>> parser.parse_args(['-xX']) - Namespace(foo=None, x='X') - -Several short options can be joined together, using only a single ``-`` prefix, -as long as only the last option (or none of them) requires a value:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x', action='store_true') - >>> parser.add_argument('-y', action='store_true') - >>> parser.add_argument('-z') - >>> parser.parse_args(['-xyzZ']) - Namespace(x=True, y=True, z='Z') - - -Invalid arguments -^^^^^^^^^^^^^^^^^ - -While parsing the command line, :meth:`~ArgumentParser.parse_args` checks for a -variety of errors, including ambiguous options, invalid types, invalid options, -wrong number of positional arguments, etc. When it encounters such an error, -it exits and prints the error along with a usage message:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', type=int) - >>> parser.add_argument('bar', nargs='?') - - >>> # invalid type - >>> parser.parse_args(['--foo', 'spam']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: argument --foo: invalid int value: 'spam' - - >>> # invalid option - >>> parser.parse_args(['--bar']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: no such option: --bar - - >>> # wrong number of arguments - >>> parser.parse_args(['spam', 'badger']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: extra arguments found: badger - - -Arguments containing ``-`` -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method attempts to give errors whenever -the user has clearly made a mistake, but some situations are inherently -ambiguous. For example, the command-line argument ``-1`` could either be an -attempt to specify an option or an attempt to provide a positional argument. -The :meth:`~ArgumentParser.parse_args` method is cautious here: positional -arguments may only begin with ``-`` if they look like negative numbers and -there are no options in the parser that look like negative numbers:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x') - >>> parser.add_argument('foo', nargs='?') - - >>> # no negative number options, so -1 is a positional argument - >>> parser.parse_args(['-x', '-1']) - Namespace(foo=None, x='-1') - - >>> # no negative number options, so -1 and -5 are positional arguments - >>> parser.parse_args(['-x', '-1', '-5']) - Namespace(foo='-5', x='-1') - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-1', dest='one') - >>> parser.add_argument('foo', nargs='?') - - >>> # negative number options present, so -1 is an option - >>> parser.parse_args(['-1', 'X']) - Namespace(foo=None, one='X') - - >>> # negative number options present, so -2 is an option - >>> parser.parse_args(['-2']) - usage: PROG [-h] [-1 ONE] [foo] - PROG: error: no such option: -2 - - >>> # negative number options present, so both -1s are options - >>> parser.parse_args(['-1', '-1']) - usage: PROG [-h] [-1 ONE] [foo] - PROG: error: argument -1: expected one argument - -If you have positional arguments that must begin with ``-`` and don't look -like negative numbers, you can insert the pseudo-argument ``'--'`` which tells -:meth:`~ArgumentParser.parse_args` that everything after that is a positional -argument:: - - >>> parser.parse_args(['--', '-f']) - Namespace(foo='-f', one=None) - -.. _prefix-matching: - -Argument abbreviations (prefix matching) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method :ref:`by default ` -allows long options to be abbreviated to a prefix, if the abbreviation is -unambiguous (the prefix matches a unique option):: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-bacon') - >>> parser.add_argument('-badger') - >>> parser.parse_args('-bac MMM'.split()) - Namespace(bacon='MMM', badger=None) - >>> parser.parse_args('-bad WOOD'.split()) - Namespace(bacon=None, badger='WOOD') - >>> parser.parse_args('-ba BA'.split()) - usage: PROG [-h] [-bacon BACON] [-badger BADGER] - PROG: error: ambiguous option: -ba could match -badger, -bacon - -An error is produced for arguments that could produce more than one options. -This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. - -.. _args: - -Beyond ``sys.argv`` -^^^^^^^^^^^^^^^^^^^ - -Sometimes it may be useful to have an ArgumentParser parse arguments other than those -of :data:`sys.argv`. This can be accomplished by passing a list of strings to -:meth:`~ArgumentParser.parse_args`. This is useful for testing at the -interactive prompt:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument( - ... 'integers', metavar='int', type=int, choices=range(10), - ... nargs='+', help='an integer in the range 0..9') - >>> parser.add_argument( - ... '--sum', dest='accumulate', action='store_const', const=sum, - ... default=max, help='sum the integers (default: find the max)') - >>> parser.parse_args(['1', '2', '3', '4']) - Namespace(accumulate=, integers=[1, 2, 3, 4]) - >>> parser.parse_args(['1', '2', '3', '4', '--sum']) - Namespace(accumulate=, integers=[1, 2, 3, 4]) - -.. _namespace: - -The Namespace object -^^^^^^^^^^^^^^^^^^^^ - -.. class:: Namespace - - Simple class used by default by :meth:`~ArgumentParser.parse_args` to create - an object holding attributes and return it. - -This class is deliberately simple, just an :class:`object` subclass with a -readable string representation. If you prefer to have dict-like view of the -attributes, you can use the standard Python idiom, :func:`vars`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> args = parser.parse_args(['--foo', 'BAR']) - >>> vars(args) - {'foo': 'BAR'} - -It may also be useful to have an :class:`ArgumentParser` assign attributes to an -already existing object, rather than a new :class:`Namespace` object. This can -be achieved by specifying the ``namespace=`` keyword argument:: - - >>> class C: - ... pass - ... - >>> c = C() - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) - >>> c.foo - 'BAR' - - -Other utilities ---------------- - -Sub-commands -^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ - [parser_class], [action], \ - [option_string], [dest], [help], \ - [metavar]) - - Many programs split up their functionality into a number of sub-commands, - for example, the ``svn`` program can invoke sub-commands like ``svn - checkout``, ``svn update``, and ``svn commit``. Splitting up functionality - this way can be a particularly good idea when a program performs several - different functions which require different kinds of command-line arguments. - :class:`ArgumentParser` supports the creation of such sub-commands with the - :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally - called with no arguments and returns a special action object. This object - has a single method, :meth:`~ArgumentParser.add_parser`, which takes a - command name and any :class:`ArgumentParser` constructor arguments, and - returns an :class:`ArgumentParser` object that can be modified as usual. - - Description of parameters: - - * title - title for the sub-parser group in help output; by default - "subcommands" if description is provided, otherwise uses title for - positional arguments - - * description - description for the sub-parser group in help output, by - default ``None`` - - * prog - usage information that will be displayed with sub-command help, - by default the name of the program and any positional arguments before the - subparser argument - - * parser_class - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) - - * action_ - the basic type of action to be taken when this argument is - encountered at the command line - - * dest_ - name of the attribute under which sub-command name will be - stored; by default ``None`` and no value is stored - - * help_ - help for sub-parser group in help output, by default ``None`` - - * metavar_ - string presenting available sub-commands in help; by default it - is ``None`` and presents sub-commands in form {cmd1, cmd2, ..} - - Some example usage:: - - >>> # create the top-level parser - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', action='store_true', help='foo help') - >>> subparsers = parser.add_subparsers(help='sub-command help') - >>> - >>> # create the parser for the "a" command - >>> parser_a = subparsers.add_parser('a', help='a help') - >>> parser_a.add_argument('bar', type=int, help='bar help') - >>> - >>> # create the parser for the "b" command - >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') - >>> - >>> # parse some argument lists - >>> parser.parse_args(['a', '12']) - Namespace(bar=12, foo=False) - >>> parser.parse_args(['--foo', 'b', '--baz', 'Z']) - Namespace(baz='Z', foo=True) - - Note that the object returned by :meth:`parse_args` will only contain - attributes for the main parser and the subparser that was selected by the - command line (and not any other subparsers). So in the example above, when - the ``a`` command is specified, only the ``foo`` and ``bar`` attributes are - present, and when the ``b`` command is specified, only the ``foo`` and - ``baz`` attributes are present. - - Similarly, when a help message is requested from a subparser, only the help - for that particular parser will be printed. The help message will not - include parent parser or sibling parser messages. (A help message for each - subparser command, however, can be given by supplying the ``help=`` argument - to :meth:`add_parser` as above.) - - :: - - >>> parser.parse_args(['--help']) - usage: PROG [-h] [--foo] {a,b} ... - - positional arguments: - {a,b} sub-command help - a a help - b b help - - optional arguments: - -h, --help show this help message and exit - --foo foo help - - >>> parser.parse_args(['a', '--help']) - usage: PROG a [-h] bar - - positional arguments: - bar bar help - - optional arguments: - -h, --help show this help message and exit - - >>> parser.parse_args(['b', '--help']) - usage: PROG b [-h] [--baz {X,Y,Z}] - - optional arguments: - -h, --help show this help message and exit - --baz {X,Y,Z} baz help - - The :meth:`add_subparsers` method also supports ``title`` and ``description`` - keyword arguments. When either is present, the subparser's commands will - appear in their own group in the help output. For example:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers(title='subcommands', - ... description='valid subcommands', - ... help='additional help') - >>> subparsers.add_parser('foo') - >>> subparsers.add_parser('bar') - >>> parser.parse_args(['-h']) - usage: [-h] {foo,bar} ... - - optional arguments: - -h, --help show this help message and exit - - subcommands: - valid subcommands - - {foo,bar} additional help - - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, - which allows multiple strings to refer to the same subparser. This example, - like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> checkout = subparsers.add_parser('checkout', aliases=['co']) - >>> checkout.add_argument('foo') - >>> parser.parse_args(['co', 'bar']) - Namespace(foo='bar') - - One particularly effective way of handling sub-commands is to combine the use - of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so - that each subparser knows which Python function it should execute. For - example:: - - >>> # sub-command functions - >>> def foo(args): - ... print(args.x * args.y) - ... - >>> def bar(args): - ... print('((%s))' % args.z) - ... - >>> # create the top-level parser - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> - >>> # create the parser for the "foo" command - >>> parser_foo = subparsers.add_parser('foo') - >>> parser_foo.add_argument('-x', type=int, default=1) - >>> parser_foo.add_argument('y', type=float) - >>> parser_foo.set_defaults(func=foo) - >>> - >>> # create the parser for the "bar" command - >>> parser_bar = subparsers.add_parser('bar') - >>> parser_bar.add_argument('z') - >>> parser_bar.set_defaults(func=bar) - >>> - >>> # parse the args and call whatever function was selected - >>> args = parser.parse_args('foo 1 -x 2'.split()) - >>> args.func(args) - 2.0 - >>> - >>> # parse the args and call whatever function was selected - >>> args = parser.parse_args('bar XYZYX'.split()) - >>> args.func(args) - ((XYZYX)) - - This way, you can let :meth:`parse_args` do the job of calling the - appropriate function after argument parsing is complete. Associating - functions with actions like this is typically the easiest way to handle the - different actions for each of your subparsers. However, if it is necessary - to check the name of the subparser that was invoked, the ``dest`` keyword - argument to the :meth:`add_subparsers` call will work:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers(dest='subparser_name') - >>> subparser1 = subparsers.add_parser('1') - >>> subparser1.add_argument('-x') - >>> subparser2 = subparsers.add_parser('2') - >>> subparser2.add_argument('y') - >>> parser.parse_args(['2', 'frobble']) - Namespace(subparser_name='2', y='frobble') - - -FileType objects -^^^^^^^^^^^^^^^^ - -.. class:: FileType(mode='r', bufsize=-1, encoding=None, errors=None) - - The :class:`FileType` factory creates objects that can be passed to the type - argument of :meth:`ArgumentParser.add_argument`. Arguments that have - :class:`FileType` objects as their type will open command-line arguments as - files with the requested modes, buffer sizes, encodings and error handling - (see the :func:`open` function for more details):: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--raw', type=argparse.FileType('wb', 0)) - >>> parser.add_argument('out', type=argparse.FileType('w', encoding='UTF-8')) - >>> parser.parse_args(['--raw', 'raw.dat', 'file.txt']) - Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>) - - FileType objects understand the pseudo-argument ``'-'`` and automatically - convert this into ``sys.stdin`` for readable :class:`FileType` objects and - ``sys.stdout`` for writable :class:`FileType` objects:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', type=argparse.FileType('r')) - >>> parser.parse_args(['-']) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) - - .. versionadded:: 3.4 - The *encodings* and *errors* keyword arguments. - - -Argument groups -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_argument_group(title=None, description=None) - - By default, :class:`ArgumentParser` groups command-line arguments into - "positional arguments" and "optional arguments" when displaying help - messages. When there is a better conceptual grouping of arguments than this - default one, appropriate groups can be created using the - :meth:`add_argument_group` method:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> group = parser.add_argument_group('group') - >>> group.add_argument('--foo', help='foo help') - >>> group.add_argument('bar', help='bar help') - >>> parser.print_help() - usage: PROG [--foo FOO] bar - - group: - bar bar help - --foo FOO foo help - - The :meth:`add_argument_group` method returns an argument group object which - has an :meth:`~ArgumentParser.add_argument` method just like a regular - :class:`ArgumentParser`. When an argument is added to the group, the parser - treats it just like a normal argument, but displays the argument in a - separate group for help messages. The :meth:`add_argument_group` method - accepts *title* and *description* arguments which can be used to - customize this display:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> group1 = parser.add_argument_group('group1', 'group1 description') - >>> group1.add_argument('foo', help='foo help') - >>> group2 = parser.add_argument_group('group2', 'group2 description') - >>> group2.add_argument('--bar', help='bar help') - >>> parser.print_help() - usage: PROG [--bar BAR] foo - - group1: - group1 description - - foo foo help - - group2: - group2 description - - --bar BAR bar help - - Note that any arguments not in your user-defined groups will end up back - in the usual "positional arguments" and "optional arguments" sections. - - -Mutual exclusion -^^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_mutually_exclusive_group(required=False) - - Create a mutually exclusive group. :mod:`argparse` will make sure that only - one of the arguments in the mutually exclusive group was present on the - command line:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> group = parser.add_mutually_exclusive_group() - >>> group.add_argument('--foo', action='store_true') - >>> group.add_argument('--bar', action='store_false') - >>> parser.parse_args(['--foo']) - Namespace(bar=True, foo=True) - >>> parser.parse_args(['--bar']) - Namespace(bar=False, foo=False) - >>> parser.parse_args(['--foo', '--bar']) - usage: PROG [-h] [--foo | --bar] - PROG: error: argument --bar: not allowed with argument --foo - - The :meth:`add_mutually_exclusive_group` method also accepts a *required* - argument, to indicate that at least one of the mutually exclusive arguments - is required:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> group = parser.add_mutually_exclusive_group(required=True) - >>> group.add_argument('--foo', action='store_true') - >>> group.add_argument('--bar', action='store_false') - >>> parser.parse_args([]) - usage: PROG [-h] (--foo | --bar) - PROG: error: one of the arguments --foo --bar is required - - Note that currently mutually exclusive argument groups do not support the - *title* and *description* arguments of - :meth:`~ArgumentParser.add_argument_group`. - - -Parser defaults -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.set_defaults(**kwargs) - - Most of the time, the attributes of the object returned by :meth:`parse_args` - will be fully determined by inspecting the command-line arguments and the argument - actions. :meth:`set_defaults` allows some additional - attributes that are determined without any inspection of the command line to - be added:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', type=int) - >>> parser.set_defaults(bar=42, baz='badger') - >>> parser.parse_args(['736']) - Namespace(bar=42, baz='badger', foo=736) - - Note that parser-level defaults always override argument-level defaults:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default='bar') - >>> parser.set_defaults(foo='spam') - >>> parser.parse_args([]) - Namespace(foo='spam') - - Parser-level defaults can be particularly useful when working with multiple - parsers. See the :meth:`~ArgumentParser.add_subparsers` method for an - example of this type. - -.. method:: ArgumentParser.get_default(dest) - - Get the default value for a namespace attribute, as set by either - :meth:`~ArgumentParser.add_argument` or by - :meth:`~ArgumentParser.set_defaults`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default='badger') - >>> parser.get_default('foo') - 'badger' - - -Printing help -^^^^^^^^^^^^^ - -In most typical applications, :meth:`~ArgumentParser.parse_args` will take -care of formatting and printing any usage or error messages. However, several -formatting methods are available: - -.. method:: ArgumentParser.print_usage(file=None) - - Print a brief description of how the :class:`ArgumentParser` should be - invoked on the command line. If *file* is ``None``, :data:`sys.stdout` is - assumed. - -.. method:: ArgumentParser.print_help(file=None) - - Print a help message, including the program usage and information about the - arguments registered with the :class:`ArgumentParser`. If *file* is - ``None``, :data:`sys.stdout` is assumed. - -There are also variants of these methods that simply return a string instead of -printing it: - -.. method:: ArgumentParser.format_usage() - - Return a string containing a brief description of how the - :class:`ArgumentParser` should be invoked on the command line. - -.. method:: ArgumentParser.format_help() - - Return a string containing a help message, including the program usage and - information about the arguments registered with the :class:`ArgumentParser`. - - -Partial parsing -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - -Sometimes a script may only parse a few of the command-line arguments, passing -the remaining arguments on to another script or program. In these cases, the -:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like -:meth:`~ArgumentParser.parse_args` except that it does not produce an error when -extra arguments are present. Instead, it returns a two item tuple containing -the populated namespace and the list of remaining argument strings. - -:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('bar') - >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) - (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) - -.. warning:: - :ref:`Prefix matching ` rules apply to - :meth:`parse_known_args`. The parser may consume an option even if it's just - a prefix of one of its known options, instead of leaving it in the remaining - arguments list. - - -Customizing file parsing -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.convert_arg_line_to_args(arg_line) - - Arguments that are read from a file (see the *fromfile_prefix_chars* - keyword argument to the :class:`ArgumentParser` constructor) are read one - argument per line. :meth:`convert_arg_line_to_args` can be overridden for - fancier reading. - - This method takes a single argument *arg_line* which is a string read from - the argument file. It returns a list of arguments parsed from this string. - The method is called once per line read from the argument file, in order. - - A useful override of this method is one that treats each space-separated word - as an argument. The following example demonstrates how to do this:: - - class MyArgumentParser(argparse.ArgumentParser): - def convert_arg_line_to_args(self, arg_line): - return arg_line.split() - - -Exiting methods -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.exit(status=0, message=None) - - This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. - -.. method:: ArgumentParser.error(message) - - This method prints a usage message including the *message* to the - standard error and terminates the program with a status code of 2. - -.. _upgrading-optparse-code: - -Upgrading optparse code ------------------------ - -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support the new -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: - -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. - -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: - -* Replace all :meth:`optparse.OptionParser.add_option` calls with - :meth:`ArgumentParser.add_argument` calls. - -* Replace ``(options, args) = parser.parse_args()`` with ``args = - parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in the :mod:`argparse` context is called ``args``. - -* Replace :meth:`optparse.OptionParser.disable_interspersed_args` - by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or - use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument - strings in a separate list. - -* Replace callback actions and the ``callback_*`` keyword arguments with - ``type`` or ``action`` arguments. - -* Replace string names for ``type`` keyword arguments with the corresponding - type objects (e.g. int, float, complex, etc). - -* Replace :class:`optparse.Values` with :class:`Namespace` and - :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with - :exc:`ArgumentError`. - -* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with - the standard Python syntax to use dictionaries to format strings, that is, - ``%(default)s`` and ``%(prog)s``. - -* Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. diff --git a/third_party/python/Doc/library/array.rst b/third_party/python/Doc/library/array.rst deleted file mode 100644 index 4ac7bb539..000000000 --- a/third_party/python/Doc/library/array.rst +++ /dev/null @@ -1,278 +0,0 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== - -.. module:: array - :synopsis: Space efficient arrays of uniformly typed numeric values. - -.. index:: single: arrays - --------------- - -This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating point numbers. Arrays are sequence -types and behave very much like lists, except that the type of objects stored in -them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are -defined: - -+-----------+--------------------+-------------------+-----------------------+-------+ -| Type code | C Type | Python Type | Minimum size in bytes | Notes | -+===========+====================+===================+=======================+=======+ -| ``'b'`` | signed char | int | 1 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'B'`` | unsigned char | int | 1 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | Py_UNICODE | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'h'`` | signed short | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'H'`` | unsigned short | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'i'`` | signed int | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'I'`` | unsigned int | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'l'`` | signed long | int | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'L'`` | unsigned long | int | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | \(2) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | \(2) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'f'`` | float | float | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'d'`` | double | float | 8 | | -+-----------+--------------------+-------------------+-----------------------+-------+ - -Notes: - -(1) - The ``'u'`` type code corresponds to Python's obsolete unicode character - (:c:type:`Py_UNICODE` which is :c:type:`wchar_t`). Depending on the - platform, it can be 16 bits or 32 bits. - - ``'u'`` will be removed together with the rest of the :c:type:`Py_UNICODE` - API. - - .. deprecated-removed:: 3.3 4.0 - -(2) - The ``'q'`` and ``'Q'`` type codes are available only if - the platform C compiler used to build Python supports C :c:type:`long long`, - or, on Windows, :c:type:`__int64`. - - .. versionadded:: 3.3 - -The actual representation of values is determined by the machine architecture -(strictly speaking, by the C implementation). The actual size can be accessed -through the :attr:`itemsize` attribute. - -The module defines the following type: - - -.. class:: array(typecode[, initializer]) - - A new array whose items are restricted by *typecode*, and initialized - from the optional *initializer* value, which must be a list, a - :term:`bytes-like object`, or iterable over elements of the - appropriate type. - - If given a list or string, the initializer is passed to the new array's - :meth:`fromlist`, :meth:`frombytes`, or :meth:`fromunicode` method (see below) - to add initial items to the array. Otherwise, the iterable initializer is - passed to the :meth:`extend` method. - - -.. data:: typecodes - - A string with all available type codes. - -Array objects support the ordinary sequence operations of indexing, slicing, -concatenation, and multiplication. When using slice assignment, the assigned -value must be an array object with the same type code; in all other cases, -:exc:`TypeError` is raised. Array objects also implement the buffer interface, -and may be used wherever :term:`bytes-like objects ` are supported. - -The following data items and methods are also supported: - -.. attribute:: array.typecode - - The typecode character used to create the array. - - -.. attribute:: array.itemsize - - The length in bytes of one array item in the internal representation. - - -.. method:: array.append(x) - - Append a new item with value *x* to the end of the array. - - -.. method:: array.buffer_info() - - Return a tuple ``(address, length)`` giving the current memory address and the - length in elements of the buffer used to hold array's contents. The size of the - memory buffer in bytes can be computed as ``array.buffer_info()[1] * - array.itemsize``. This is occasionally useful when working with low-level (and - inherently unsafe) I/O interfaces that require memory addresses, such as certain - :c:func:`ioctl` operations. The returned numbers are valid as long as the array - exists and no length-changing operations are applied to it. - - .. note:: - - When using array objects from code written in C or C++ (the only way to - effectively make use of this information), it makes more sense to use the buffer - interface supported by array objects. This method is maintained for backward - compatibility and should be avoided in new code. The buffer interface is - documented in :ref:`bufferobjects`. - - -.. method:: array.byteswap() - - "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is - raised. It is useful when reading data from a file written on a machine with a - different byte order. - - -.. method:: array.count(x) - - Return the number of occurrences of *x* in the array. - - -.. method:: array.extend(iterable) - - Append items from *iterable* to the end of the array. If *iterable* is another - array, it must have *exactly* the same type code; if not, :exc:`TypeError` will - be raised. If *iterable* is not an array, it must be iterable and its elements - must be the right type to be appended to the array. - - -.. method:: array.frombytes(s) - - Appends items from the string, interpreting the string as an array of machine - values (as if it had been read from a file using the :meth:`fromfile` method). - - .. versionadded:: 3.2 - :meth:`fromstring` is renamed to :meth:`frombytes` for clarity. - - -.. method:: array.fromfile(f, n) - - Read *n* items (as machine values) from the :term:`file object` *f* and append - them to the end of the array. If less than *n* items are available, - :exc:`EOFError` is raised, but the items that were available are still - inserted into the array. *f* must be a real built-in file object; something - else with a :meth:`read` method won't do. - - -.. method:: array.fromlist(list) - - Append items from the list. This is equivalent to ``for x in list: - a.append(x)`` except that if there is a type error, the array is unchanged. - - -.. method:: array.fromstring() - - Deprecated alias for :meth:`frombytes`. - - -.. method:: array.fromunicode(s) - - Extends this array with data from the given unicode string. The array must - be a type ``'u'`` array; otherwise a :exc:`ValueError` is raised. Use - ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an - array of some other type. - - -.. method:: array.index(x) - - Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. - - -.. method:: array.insert(i, x) - - Insert a new item with value *x* in the array before position *i*. Negative - values are treated as being relative to the end of the array. - - -.. method:: array.pop([i]) - - Removes the item with the index *i* from the array and returns it. The optional - argument defaults to ``-1``, so that by default the last item is removed and - returned. - - -.. method:: array.remove(x) - - Remove the first occurrence of *x* from the array. - - -.. method:: array.reverse() - - Reverse the order of the items in the array. - - -.. method:: array.tobytes() - - Convert the array to an array of machine values and return the bytes - representation (the same sequence of bytes that would be written to a file by - the :meth:`tofile` method.) - - .. versionadded:: 3.2 - :meth:`tostring` is renamed to :meth:`tobytes` for clarity. - - -.. method:: array.tofile(f) - - Write all items (as machine values) to the :term:`file object` *f*. - - -.. method:: array.tolist() - - Convert the array to an ordinary list with the same items. - - -.. method:: array.tostring() - - Deprecated alias for :meth:`tobytes`. - - -.. method:: array.tounicode() - - Convert the array to a unicode string. The array must be a type ``'u'`` array; - otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to - obtain a unicode string from an array of some other type. - - -When an array object is printed or converted to a string, it is represented as -``array(typecode, initializer)``. The *initializer* is omitted if the array is -empty, otherwise it is a string if the *typecode* is ``'u'``, otherwise it is a -list of numbers. The string is guaranteed to be able to be converted back to an -array with the same type and value using :func:`eval`, so long as the -:class:`~array.array` class has been imported using ``from array import array``. -Examples:: - - array('l') - array('u', 'hello \u2641') - array('l', [1, 2, 3, 4, 5]) - array('d', [1.0, 2.0, 3.14]) - - -.. seealso:: - - Module :mod:`struct` - Packing and unpacking of heterogeneous binary data. - - Module :mod:`xdrlib` - Packing and unpacking of External Data Representation (XDR) data as used in some - remote procedure call systems. - - `The Numerical Python Documentation `_ - The Numeric Python extension (NumPy) defines another array type; see - http://www.numpy.org/ for further information about Numerical Python. - diff --git a/third_party/python/Doc/library/ast.rst b/third_party/python/Doc/library/ast.rst deleted file mode 100644 index eb6675497..000000000 --- a/third_party/python/Doc/library/ast.rst +++ /dev/null @@ -1,270 +0,0 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== - -.. module:: ast - :synopsis: Abstract Syntax Tree classes and manipulation. - -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - -**Source code:** :source:`Lib/ast.py` - --------------- - -The :mod:`ast` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as -a flag to the :func:`compile` built-in function, or using the :func:`parse` -helper provided in this module. The result will be a tree of objects whose -classes all inherit from :class:`ast.AST`. An abstract syntax tree can be -compiled into a Python code object using the built-in :func:`compile` function. - - -Node classes ------------- - -.. class:: AST - - This is the base of all AST node classes. The actual node classes are - derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`below `. They are defined in the :mod:`_ast` C - module and re-exported in :mod:`ast`. - - There is one class defined for each left-hand side symbol in the abstract - grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, - there is one class defined for each constructor on the right-hand side; these - classes inherit from the classes for the left-hand side trees. For example, - :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules - with alternatives (aka "sums"), the left-hand side class is abstract: only - instances of specific constructor nodes are ever created. - - .. index:: single: ? (question mark); in AST grammar - .. index:: single: * (asterisk); in AST grammar - - .. attribute:: _fields - - Each concrete class has an attribute :attr:`_fields` which gives the names - of all child nodes. - - Each instance of a concrete class has one attribute for each child node, - of the type as defined in the grammar. For example, :class:`ast.BinOp` - instances have an attribute :attr:`left` of type :class:`ast.expr`. - - If these attributes are marked as optional in the grammar (using a - question mark), the value might be ``None``. If the attributes can have - zero-or-more values (marked with an asterisk), the values are represented - as Python lists. All possible attributes must be present and have valid - values when compiling an AST with :func:`compile`. - - .. attribute:: lineno - col_offset - - Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno` and :attr:`col_offset` attributes. The :attr:`lineno` is - the line number of source text (1-indexed so the first line is line 1) and - the :attr:`col_offset` is the UTF-8 byte offset of the first token that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. - - The constructor of a class :class:`ast.T` parses its arguments as follows: - - * If there are positional arguments, there must be as many as there are items - in :attr:`T._fields`; they will be assigned as attributes of these names. - * If there are keyword arguments, they will set the attributes of the same - names to the given values. - - For example, to create and populate an :class:`ast.UnaryOp` node, you could - use :: - - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Num() - node.operand.n = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - - node = ast.UnaryOp(ast.USub(), ast.Num(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - - -.. _abstract-grammar: - -Abstract Grammar ----------------- - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl - :language: none - - -:mod:`ast` Helpers ------------------- - -Apart from the node classes, the :mod:`ast` module defines these utility functions -and classes for traversing abstract syntax trees: - -.. function:: parse(source, filename='', mode='exec') - - Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - -.. function:: literal_eval(node_or_string) - - Safely evaluate an expression node or a string containing a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, and ``None``. - - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - .. versionchanged:: 3.2 - Now allows bytes and set literals. - - -.. function:: get_docstring(node, clean=True) - - Return the docstring of the given *node* (which must be a - :class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None`` - if it has no docstring. If *clean* is true, clean up the docstring's - indentation with :func:`inspect.cleandoc`. - - -.. function:: fix_missing_locations(node) - - When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports - them. This is rather tedious to fill in for generated nodes, so this helper - adds these attributes recursively where not already set, by setting them to - the values of the parent node. It works recursively starting at *node*. - - -.. function:: increment_lineno(node, n=1) - - Increment the line number of each node in the tree starting at *node* by *n*. - This is useful to "move code" to a different location in a file. - - -.. function:: copy_location(new_node, old_node) - - Copy source location (:attr:`lineno` and :attr:`col_offset`) from *old_node* - to *new_node* if possible, and return *new_node*. - - -.. function:: iter_fields(node) - - Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` - that is present on *node*. - - -.. function:: iter_child_nodes(node) - - Yield all direct child nodes of *node*, that is, all fields that are nodes - and all items of fields that are lists of nodes. - - -.. function:: walk(node) - - Recursively yield all descendant nodes in the tree starting at *node* - (including *node* itself), in no specified order. This is useful if you only - want to modify nodes in place and don't care about the context. - - -.. class:: NodeVisitor() - - A node visitor base class that walks the abstract syntax tree and calls a - visitor function for every node found. This function may return a value - which is forwarded by the :meth:`visit` method. - - This class is meant to be subclassed, with the subclass adding visitor - methods. - - .. method:: visit(node) - - Visit a node. The default implementation calls the method called - :samp:`self.visit_{classname}` where *classname* is the name of the node - class, or :meth:`generic_visit` if that method doesn't exist. - - .. method:: generic_visit(node) - - This visitor calls :meth:`visit` on all children of the node. - - Note that child nodes of nodes that have a custom visitor method won't be - visited unless the visitor calls :meth:`generic_visit` or visits them - itself. - - Don't use the :class:`NodeVisitor` if you want to apply changes to nodes - during traversal. For this a special visitor exists - (:class:`NodeTransformer`) that allows modifications. - - -.. class:: NodeTransformer() - - A :class:`NodeVisitor` subclass that walks the abstract syntax tree and - allows modification of nodes. - - The :class:`NodeTransformer` will walk the AST and use the return value of - the visitor methods to replace or remove the old node. If the return value - of the visitor method is ``None``, the node will be removed from its - location, otherwise it is replaced with the return value. The return value - may be the original node in which case no replacement takes place. - - Here is an example transformer that rewrites all occurrences of name lookups - (``foo``) to ``data['foo']``:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return copy_location(Subscript( - value=Name(id='data', ctx=Load()), - slice=Index(value=Str(s=node.id)), - ctx=node.ctx - ), node) - - Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` - method for the node first. - - For nodes that were part of a collection of statements (that applies to all - statement nodes), the visitor may also return a list of nodes rather than - just a single node. - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - - -.. function:: dump(node, annotate_fields=True, include_attributes=False) - - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to ``False``. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to ``True``. - -.. seealso:: - - `Green Tree Snakes `_, an external documentation resource, has good - details on working with Python ASTs. diff --git a/third_party/python/Doc/library/asynchat.rst b/third_party/python/Doc/library/asynchat.rst deleted file mode 100644 index 9e51416b8..000000000 --- a/third_party/python/Doc/library/asynchat.rst +++ /dev/null @@ -1,213 +0,0 @@ -:mod:`asynchat` --- Asynchronous socket command/response handler -================================================================ - -.. module:: asynchat - :synopsis: Support for asynchronous command/response protocols. - -.. moduleauthor:: Sam Rushing -.. sectionauthor:: Steve Holden - -**Source code:** :source:`Lib/asynchat.py` - -.. deprecated:: 3.6 - Please use :mod:`asyncio` instead. - --------------- - -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - -This module builds on the :mod:`asyncore` infrastructure, simplifying -asynchronous clients and servers and making it easier to handle protocols -whose elements are terminated by arbitrary strings, or are of variable length. -:mod:`asynchat` defines the abstract class :class:`async_chat` that you -subclass, providing implementations of the :meth:`collect_incoming_data` and -:meth:`found_terminator` methods. It uses the same asynchronous loop as -:mod:`asyncore`, and the two types of channel, :class:`asyncore.dispatcher` -and :class:`asynchat.async_chat`, can freely be mixed in the channel map. -Typically an :class:`asyncore.dispatcher` server channel generates new -:class:`asynchat.async_chat` channel objects as it receives incoming -connection requests. - - -.. class:: async_chat() - - This class is an abstract subclass of :class:`asyncore.dispatcher`. To make - practical use of the code you must subclass :class:`async_chat`, providing - meaningful :meth:`collect_incoming_data` and :meth:`found_terminator` - methods. - The :class:`asyncore.dispatcher` methods can be used, although not all make - sense in a message/response context. - - Like :class:`asyncore.dispatcher`, :class:`async_chat` defines a set of - events that are generated by an analysis of socket conditions after a - :c:func:`select` call. Once the polling loop has been started the - :class:`async_chat` object's methods are called by the event-processing - framework with no action on the part of the programmer. - - Two class attributes can be modified, to improve performance, or possibly - even to conserve memory. - - - .. data:: ac_in_buffer_size - - The asynchronous input buffer size (default ``4096``). - - - .. data:: ac_out_buffer_size - - The asynchronous output buffer size (default ``4096``). - - Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to - define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need - have only one method, :meth:`more`, which should return data to be - transmitted on the channel. - The producer indicates exhaustion (*i.e.* that it contains no more data) by - having its :meth:`more` method return the empty bytes object. At this point - the :class:`async_chat` object removes the producer from the queue and starts - using the next producer, if any. When the producer queue is empty the - :meth:`handle_write` method does nothing. You use the channel object's - :meth:`set_terminator` method to describe how to recognize the end of, or - an important breakpoint in, an incoming transmission from the remote - endpoint. - - To build a functioning :class:`async_chat` subclass your input methods - :meth:`collect_incoming_data` and :meth:`found_terminator` must handle the - data that the channel receives asynchronously. The methods are described - below. - - -.. method:: async_chat.close_when_done() - - Pushes a ``None`` on to the producer queue. When this producer is popped off - the queue it causes the channel to be closed. - - -.. method:: async_chat.collect_incoming_data(data) - - Called with *data* holding an arbitrary amount of received data. The - default method, which must be overridden, raises a - :exc:`NotImplementedError` exception. - - -.. method:: async_chat.discard_buffers() - - In emergencies this method will discard any data held in the input and/or - output buffers and the producer queue. - - -.. method:: async_chat.found_terminator() - - Called when the incoming data stream matches the termination condition set - by :meth:`set_terminator`. The default method, which must be overridden, - raises a :exc:`NotImplementedError` exception. The buffered input data - should be available via an instance attribute. - - -.. method:: async_chat.get_terminator() - - Returns the current terminator for the channel. - - -.. method:: async_chat.push(data) - - Pushes data on to the channel's queue to ensure its transmission. - This is all you need to do to have the channel write the data out to the - network, although it is possible to use your own producers in more complex - schemes to implement encryption and chunking, for example. - - -.. method:: async_chat.push_with_producer(producer) - - Takes a producer object and adds it to the producer queue associated with - the channel. When all currently-pushed producers have been exhausted the - channel will consume this producer's data by calling its :meth:`more` - method and send the data to the remote endpoint. - - -.. method:: async_chat.set_terminator(term) - - Sets the terminating condition to be recognized on the channel. ``term`` - may be any of three types of value, corresponding to three different ways - to handle incoming protocol data. - - +-----------+---------------------------------------------+ - | term | Description | - +===========+=============================================+ - | *string* | Will call :meth:`found_terminator` when the | - | | string is found in the input stream | - +-----------+---------------------------------------------+ - | *integer* | Will call :meth:`found_terminator` when the | - | | indicated number of characters have been | - | | received | - +-----------+---------------------------------------------+ - | ``None`` | The channel continues to collect data | - | | forever | - +-----------+---------------------------------------------+ - - Note that any data following the terminator will be available for reading - by the channel after :meth:`found_terminator` is called. - - -.. _asynchat-example: - -asynchat Example ----------------- - -The following partial example shows how HTTP requests can be read with -:class:`async_chat`. A web server might create an -:class:`http_request_handler` object for each incoming client connection. -Notice that initially the channel terminator is set to match the blank line at -the end of the HTTP headers, and a flag indicates that the headers are being -read. - -Once the headers have been read, if the request is of type POST (indicating -that further data are present in the input stream) then the -``Content-Length:`` header is used to set a numeric terminator to read the -right amount of data from the channel. - -The :meth:`handle_request` method is called once all relevant input has been -marshalled, after setting the channel terminator to ``None`` to ensure that -any extraneous data sent by the web client are ignored. :: - - - import asynchat - - class http_request_handler(asynchat.async_chat): - - def __init__(self, sock, addr, sessions, log): - asynchat.async_chat.__init__(self, sock=sock) - self.addr = addr - self.sessions = sessions - self.ibuffer = [] - self.obuffer = b"" - self.set_terminator(b"\r\n\r\n") - self.reading_headers = True - self.handling = False - self.cgi_data = None - self.log = log - - def collect_incoming_data(self, data): - """Buffer the data""" - self.ibuffer.append(data) - - def found_terminator(self): - if self.reading_headers: - self.reading_headers = False - self.parse_headers(b"".join(self.ibuffer)) - self.ibuffer = [] - if self.op.upper() == b"POST": - clen = self.headers.getheader("content-length") - self.set_terminator(int(clen)) - else: - self.handling = True - self.set_terminator(None) - self.handle_request() - elif not self.handling: - self.set_terminator(None) # browsers sometimes over-send - self.cgi_data = parse(self.headers, b"".join(self.ibuffer)) - self.handling = True - self.ibuffer = [] - self.handle_request() diff --git a/third_party/python/Doc/library/asyncio-dev.rst b/third_party/python/Doc/library/asyncio-dev.rst deleted file mode 100644 index e2ad3eb01..000000000 --- a/third_party/python/Doc/library/asyncio-dev.rst +++ /dev/null @@ -1,418 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-dev: - -Develop with asyncio -==================== - -Asynchronous programming is different than classical "sequential" programming. -This page lists common traps and explains how to avoid them. - - -.. _asyncio-debug-mode: - -Debug mode of asyncio ---------------------- - -The implementation of :mod:`asyncio` has been written for performance. -In order to ease the development of asynchronous code, you may wish to -enable *debug mode*. - -To enable all debug checks for an application: - -* Enable the asyncio debug mode globally by setting the environment variable - :envvar:`PYTHONASYNCIODEBUG` to ``1``, or by calling :meth:`AbstractEventLoop.set_debug`. -* Set the log level of the :ref:`asyncio logger ` to - :py:data:`logging.DEBUG`. For example, call - ``logging.basicConfig(level=logging.DEBUG)`` at startup. -* Configure the :mod:`warnings` module to display :exc:`ResourceWarning` - warnings. For example, use the ``-Wdefault`` command line option of Python to - display them. - -Examples debug checks: - -* Log :ref:`coroutines defined but never "yielded from" - ` -* :meth:`~AbstractEventLoop.call_soon` and :meth:`~AbstractEventLoop.call_at` methods - raise an exception if they are called from the wrong thread. -* Log the execution time of the selector -* Log callbacks taking more than 100 ms to be executed. The - :attr:`AbstractEventLoop.slow_callback_duration` attribute is the minimum - duration in seconds of "slow" callbacks. -* :exc:`ResourceWarning` warnings are emitted when transports and event loops - are :ref:`not closed explicitly `. - -.. seealso:: - - The :meth:`AbstractEventLoop.set_debug` method and the :ref:`asyncio logger - `. - - -Cancellation ------------- - -Cancellation of tasks is not common in classic programming. In asynchronous -programming, not only is it something common, but you have to prepare your -code to handle it. - -Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel` -method. The :func:`wait_for` function cancels the waited task when the timeout -occurs. There are many other cases where a task can be cancelled indirectly. - -Don't call :meth:`~Future.set_result` or :meth:`~Future.set_exception` method -of :class:`Future` if the future is cancelled: it would fail with an exception. -For example, write:: - - if not fut.cancelled(): - fut.set_result('done') - -Don't schedule directly a call to the :meth:`~Future.set_result` or the -:meth:`~Future.set_exception` method of a future with -:meth:`AbstractEventLoop.call_soon`: the future can be cancelled before its method -is called. - -If you wait for a future, you should check early if the future was cancelled to -avoid useless operations. Example:: - - @coroutine - def slow_operation(fut): - if fut.cancelled(): - return - # ... slow computation ... - yield from fut - # ... - -The :func:`shield` function can also be used to ignore cancellation. - - -.. _asyncio-multithreading: - -Concurrency and multithreading ------------------------------- - -An event loop runs in a thread and executes all callbacks and tasks in the same -thread. While a task is running in the event loop, no other task is running in -the same thread. But when the task uses ``yield from``, the task is suspended -and the event loop executes the next task. - -To schedule a callback from a different thread, the -:meth:`AbstractEventLoop.call_soon_threadsafe` method should be used. Example:: - - loop.call_soon_threadsafe(callback, *args) - -Most asyncio objects are not thread safe. You should only worry if you access -objects outside the event loop. For example, to cancel a future, don't call -directly its :meth:`Future.cancel` method, but:: - - loop.call_soon_threadsafe(fut.cancel) - -To handle signals and to execute subprocesses, the event loop must be run in -the main thread. - -To schedule a coroutine object from a different thread, the -:func:`run_coroutine_threadsafe` function should be used. It returns a -:class:`concurrent.futures.Future` to access the result:: - - future = asyncio.run_coroutine_threadsafe(coro_func(), loop) - result = future.result(timeout) # Wait for the result with a timeout - -The :meth:`AbstractEventLoop.run_in_executor` method can be used with a thread pool -executor to execute a callback in different thread to not block the thread of -the event loop. - -.. seealso:: - - The :ref:`Synchronization primitives ` section describes ways - to synchronize tasks. - - The :ref:`Subprocess and threads ` section lists - asyncio limitations to run subprocesses from different threads. - - - - -.. _asyncio-handle-blocking: - -Handle blocking functions correctly ------------------------------------ - -Blocking functions should not be called directly. For example, if a function -blocks for 1 second, other tasks are delayed by 1 second which can have an -important impact on reactivity. - -For networking and subprocesses, the :mod:`asyncio` module provides high-level -APIs like :ref:`protocols `. - -An executor can be used to run a task in a different thread or even in a -different process, to not block the thread of the event loop. See the -:meth:`AbstractEventLoop.run_in_executor` method. - -.. seealso:: - - The :ref:`Delayed calls ` section details how the - event loop handles time. - - -.. _asyncio-logger: - -Logging -------- - -The :mod:`asyncio` module logs information with the :mod:`logging` module in -the logger ``'asyncio'``. - -The default log level for the :mod:`asyncio` module is :py:data:`logging.INFO`. -For those not wanting such verbosity from :mod:`asyncio` the log level can -be changed. For example, to change the level to :py:data:`logging.WARNING`: - -.. code-block:: none - - logging.getLogger('asyncio').setLevel(logging.WARNING) - - -.. _asyncio-coroutine-not-scheduled: - -Detect coroutine objects never scheduled ----------------------------------------- - -When a coroutine function is called and its result is not passed to -:func:`ensure_future` or to the :meth:`AbstractEventLoop.create_task` method, -the execution of the coroutine object will never be scheduled which is -probably a bug. :ref:`Enable the debug mode of asyncio ` -to :ref:`log a warning ` to detect it. - -Example with the bug:: - - import asyncio - - @asyncio.coroutine - def test(): - print("never scheduled") - - test() - -Output in debug mode:: - - Coroutine test() at test.py:3 was never yielded from - Coroutine object created at (most recent call last): - File "test.py", line 7, in - test() - -The fix is to call the :func:`ensure_future` function or the -:meth:`AbstractEventLoop.create_task` method with the coroutine object. - -.. seealso:: - - :ref:`Pending task destroyed `. - - -Detect exceptions never consumed --------------------------------- - -Python usually calls :func:`sys.excepthook` on unhandled exceptions. If -:meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted -` when the future is deleted by the garbage collector, with the -traceback where the exception was raised. - -Example of unhandled exception:: - - import asyncio - - @asyncio.coroutine - def bug(): - raise Exception("not consumed") - - loop = asyncio.get_event_loop() - asyncio.ensure_future(bug()) - loop.run_forever() - loop.close() - -Output:: - - Task exception was never retrieved - future: exception=Exception('not consumed',)> - Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug - raise Exception("not consumed") - Exception: not consumed - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Output in debug mode:: - - Task exception was never retrieved - future: exception=Exception('not consumed',) created at test.py:8> - source_traceback: Object created at (most recent call last): - File "test.py", line 8, in - asyncio.ensure_future(bug()) - Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 79, in __next__ - return next(self.gen) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug - raise Exception("not consumed") - Exception: not consumed - -There are different options to fix this issue. The first option is to chain the -coroutine in another coroutine and use classic try/except:: - - @asyncio.coroutine - def handle_exception(): - try: - yield from bug() - except Exception: - print("exception consumed") - - loop = asyncio.get_event_loop() - asyncio.ensure_future(handle_exception()) - loop.run_forever() - loop.close() - -Another option is to use the :meth:`AbstractEventLoop.run_until_complete` -function:: - - task = asyncio.ensure_future(bug()) - try: - loop.run_until_complete(task) - except Exception: - print("exception consumed") - -.. seealso:: - - The :meth:`Future.exception` method. - - -Chain coroutines correctly --------------------------- - -When a coroutine function calls other coroutine functions and tasks, they -should be chained explicitly with ``yield from``. Otherwise, the execution is -not guaranteed to be sequential. - -Example with different bugs using :func:`asyncio.sleep` to simulate slow -operations:: - - import asyncio - - @asyncio.coroutine - def create(): - yield from asyncio.sleep(3.0) - print("(1) create file") - - @asyncio.coroutine - def write(): - yield from asyncio.sleep(1.0) - print("(2) write into file") - - @asyncio.coroutine - def close(): - print("(3) close file") - - @asyncio.coroutine - def test(): - asyncio.ensure_future(create()) - asyncio.ensure_future(write()) - asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) - loop.stop() - - loop = asyncio.get_event_loop() - asyncio.ensure_future(test()) - loop.run_forever() - print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop)) - loop.close() - -Expected output: - -.. code-block:: none - - (1) create file - (2) write into file - (3) close file - Pending tasks at exit: set() - -Actual output: - -.. code-block:: none - - (3) close file - (2) write into file - Pending tasks at exit: {>} - Task was destroyed but it is pending! - task: > - -The loop stopped before the ``create()`` finished, ``close()`` has been called -before ``write()``, whereas coroutine functions were called in this order: -``create()``, ``write()``, ``close()``. - -To fix the example, tasks must be marked with ``yield from``:: - - @asyncio.coroutine - def test(): - yield from asyncio.ensure_future(create()) - yield from asyncio.ensure_future(write()) - yield from asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) - loop.stop() - -Or without ``asyncio.ensure_future()``:: - - @asyncio.coroutine - def test(): - yield from create() - yield from write() - yield from close() - yield from asyncio.sleep(2.0) - loop.stop() - - -.. _asyncio-pending-task-destroyed: - -Pending task destroyed ----------------------- - -If a pending task is destroyed, the execution of its wrapped :ref:`coroutine -` did not complete. It is probably a bug and so a warning is logged. - -Example of log: - -.. code-block:: none - - Task was destroyed but it is pending! - task: wait_for=> - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Example of log in debug mode: - -.. code-block:: none - - Task was destroyed but it is pending! - source_traceback: Object created at (most recent call last): - File "test.py", line 15, in - task = asyncio.ensure_future(coro, loop=loop) - task: wait_for= created at test.py:15> - - -.. seealso:: - - :ref:`Detect coroutine objects never scheduled `. - -.. _asyncio-close-transports: - -Close transports and event loops --------------------------------- - -When a transport is no more needed, call its ``close()`` method to release -resources. Event loops must also be closed explicitly. - -If a transport or an event loop is not closed explicitly, a -:exc:`ResourceWarning` warning will be emitted in its destructor. By default, -:exc:`ResourceWarning` warnings are ignored. The :ref:`Debug mode of asyncio -` section explains how to display them. diff --git a/third_party/python/Doc/library/asyncio-eventloop.rst b/third_party/python/Doc/library/asyncio-eventloop.rst deleted file mode 100644 index d053a60a9..000000000 --- a/third_party/python/Doc/library/asyncio-eventloop.rst +++ /dev/null @@ -1,1025 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-event-loop: - -Base Event Loop -=============== - -**Source code:** :source:`Lib/asyncio/events.py` - -The event loop is the central execution device provided by :mod:`asyncio`. -It provides multiple facilities, including: - -* Registering, executing and cancelling delayed calls (timeouts). - -* Creating client and server :ref:`transports ` for various - kinds of communication. - -* Launching subprocesses and the associated :ref:`transports - ` for communication with an external program. - -* Delegating costly function calls to a pool of threads. - -.. class:: BaseEventLoop - - This class is an implementation detail. It is a subclass of - :class:`AbstractEventLoop` and may be a base class of concrete - event loop implementations found in :mod:`asyncio`. It should not - be used directly; use :class:`AbstractEventLoop` instead. - ``BaseEventLoop`` should not be subclassed by third-party code; the - internal interface is not stable. - -.. class:: AbstractEventLoop - - Abstract base class of event loops. - - This class is :ref:`not thread safe `. - -Run an event loop ------------------ - -.. method:: AbstractEventLoop.run_forever() - - Run until :meth:`stop` is called. If :meth:`stop` is called before - :meth:`run_forever()` is called, this polls the I/O selector once - with a timeout of zero, runs all callbacks scheduled in response to - I/O events (and those that were already scheduled), and then exits. - If :meth:`stop` is called while :meth:`run_forever` is running, - this will run the current batch of callbacks and then exit. Note - that callbacks scheduled by callbacks will not run in that case; - they will run the next time :meth:`run_forever` is called. - - .. versionchanged:: 3.5.1 - -.. method:: AbstractEventLoop.run_until_complete(future) - - Run until the :class:`Future` is done. - - If the argument is a :ref:`coroutine object `, it is wrapped by - :func:`ensure_future`. - - Return the Future's result, or raise its exception. - -.. method:: AbstractEventLoop.is_running() - - Returns running status of event loop. - -.. method:: AbstractEventLoop.stop() - - Stop running the event loop. - - This causes :meth:`run_forever` to exit at the next suitable - opportunity (see there for more details). - - .. versionchanged:: 3.5.1 - -.. method:: AbstractEventLoop.is_closed() - - Returns ``True`` if the event loop was closed. - - .. versionadded:: 3.4.2 - -.. method:: AbstractEventLoop.close() - - Close the event loop. The loop must not be running. Pending - callbacks will be lost. - - This clears the queues and shuts down the executor, but does not wait for - the executor to finish. - - This is idempotent and irreversible. No other methods should be called after - this one. - - -.. coroutinemethod:: AbstractEventLoop.shutdown_asyncgens() - - Schedule all currently open :term:`asynchronous generator` objects to - close with an :meth:`~agen.aclose()` call. After calling this method, - the event loop will issue a warning whenever a new asynchronous generator - is iterated. Should be used to finalize all scheduled asynchronous - generators reliably. Example:: - - try: - loop.run_forever() - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - - .. versionadded:: 3.6 - - -.. _asyncio-pass-keywords: - -Calls ------ - -Most :mod:`asyncio` functions don't accept keywords. If you want to pass -keywords to your callback, use :func:`functools.partial`. For example, -``loop.call_soon(functools.partial(print, "Hello", flush=True))`` will call -``print("Hello", flush=True)``. - -.. note:: - :func:`functools.partial` is better than ``lambda`` functions, because - :mod:`asyncio` can inspect :func:`functools.partial` object to display - parameters in debug mode, whereas ``lambda`` functions have a poor - representation. - -.. method:: AbstractEventLoop.call_soon(callback, \*args) - - Arrange for a callback to be called as soon as possible. The callback is - called after :meth:`call_soon` returns, when control returns to the event - loop. - - This operates as a :abbr:`FIFO (first-in, first-out)` queue, callbacks - are called in the order in which they are registered. Each callback - will be called exactly once. - - Any positional arguments after the callback will be passed to the - callback when it is called. - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.call_soon_threadsafe(callback, \*args) - - Like :meth:`call_soon`, but thread safe. - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - -.. _asyncio-delayed-calls: - -Delayed calls -------------- - -The event loop has its own internal clock for computing timeouts. -Which clock is used depends on the (platform-specific) event loop -implementation; ideally it is a monotonic clock. This will generally be -a different clock than :func:`time.time`. - -.. note:: - - Timeouts (relative *delay* or absolute *when*) should not exceed one day. - - -.. method:: AbstractEventLoop.call_later(delay, callback, *args) - - Arrange for the *callback* to be called after the given *delay* - seconds (either an int or float). - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - *callback* will be called exactly once per call to :meth:`call_later`. - If two callbacks are scheduled for exactly the same time, it is - undefined which will be called first. - - The optional positional *args* will be passed to the callback when it - is called. If you want the callback to be called with some named - arguments, use a closure or :func:`functools.partial`. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.call_at(when, callback, *args) - - Arrange for the *callback* to be called at the given absolute timestamp - *when* (an int or float), using the same time reference as - :meth:`AbstractEventLoop.time`. - - This method's behavior is the same as :meth:`call_later`. - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.time() - - Return the current time, as a :class:`float` value, according to the - event loop's internal clock. - -.. seealso:: - - The :func:`asyncio.sleep` function. - - -Futures -------- - -.. method:: AbstractEventLoop.create_future() - - Create an :class:`asyncio.Future` object attached to the loop. - - This is a preferred way to create futures in asyncio, as event - loop implementations can provide alternative implementations - of the Future class (with better performance or instrumentation). - - .. versionadded:: 3.5.2 - - -Tasks ------ - -.. method:: AbstractEventLoop.create_task(coro) - - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. - - Third-party event loops can use their own subclass of :class:`Task` for - interoperability. In this case, the result type is a subclass of - :class:`Task`. - - This method was added in Python 3.4.2. Use the :func:`async` function to - support also older Python versions. - - .. versionadded:: 3.4.2 - -.. method:: AbstractEventLoop.set_task_factory(factory) - - Set a task factory that will be used by - :meth:`AbstractEventLoop.create_task`. - - If *factory* is ``None`` the default task factory will be set. - - If *factory* is a *callable*, it should have a signature matching - ``(loop, coro)``, where *loop* will be a reference to the active - event loop, *coro* will be a coroutine object. The callable - must return an :class:`asyncio.Future` compatible object. - - .. versionadded:: 3.4.4 - -.. method:: AbstractEventLoop.get_task_factory() - - Return a task factory, or ``None`` if the default one is in use. - - .. versionadded:: 3.4.4 - - -Creating connections --------------------- - -.. coroutinemethod:: AbstractEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None) - - Create a streaming transport connection to a given Internet *host* and - *port*: socket family :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), - socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - The chronological synopsis of the underlying operation is as follows: - - #. The connection is established, and a :ref:`transport ` - is created to represent it. - - #. *protocol_factory* is called without arguments and must return a - :ref:`protocol ` instance. - - #. The protocol instance is tied to the transport, and its - :meth:`connection_made` method is called. - - #. The coroutine returns successfully with the ``(transport, protocol)`` - pair. - - The created transport is an implementation-dependent bidirectional stream. - - .. note:: - *protocol_factory* can be any kind of callable, not necessarily - a class. For example, if you want to use a pre-created - protocol instance, you can pass ``lambda: my_protocol``. - - Options that change how the connection is created: - - * *ssl*: if given and not false, a SSL/TLS transport is created - (by default a plain TCP transport is created). If *ssl* is - a :class:`ssl.SSLContext` object, this context is used to create - the transport; if *ssl* is :const:`True`, a context with some - unspecified default settings is used. - - .. seealso:: :ref:`SSL/TLS security considerations ` - - * *server_hostname*, is only for use together with *ssl*, - and sets or overrides the hostname that the target server's certificate - will be matched against. By default the value of the *host* argument - is used. If *host* is empty, there is no default and you must pass a - value for *server_hostname*. If *server_hostname* is an empty - string, hostname matching is disabled (which is a serious security - risk, allowing for man-in-the-middle-attacks). - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to getaddrinfo() for *host* resolution. - If given, these should all be integers from the corresponding - :mod:`socket` module constants. - - * *sock*, if given, should be an existing, already connected - :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* - and *local_addr* should be specified. - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* - are looked up using getaddrinfo(), similarly to *host* and *port*. - - .. versionchanged:: 3.5 - - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. - - .. seealso:: - - The :func:`open_connection` function can be used to get a pair of - (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol. - - -.. coroutinemethod:: AbstractEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None) - - .. note:: - The parameter *reuse_address* is no longer supported, as using - :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for - UDP. Explicitly passing ``reuse_address=True`` will raise an exception. - - When multiple processes with differing UIDs assign sockets to an - indentical UDP socket address with ``SO_REUSEADDR``, incoming packets can - become randomly distributed among the sockets. - - For supported platforms, *reuse_port* can be used as a replacement for - similar functionality. With *reuse_port*, - :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically - prevents processes with differing UIDs from assigning sockets to the same - socket address. - - Create a datagram connection. - - Create datagram connection: socket family :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), - socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - Options changing how the connection is created: - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* - are looked up using :meth:`getaddrinfo`. - - * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used - to connect the socket to a remote address. The *remote_host* and - *remote_port* are looked up using :meth:`getaddrinfo`. - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to :meth:`getaddrinfo` for *host* - resolution. If given, these should all be integers from the - corresponding :mod:`socket` module constants. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on Windows - and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not - defined then this capability is unsupported. - - * *allow_broadcast* tells the kernel to allow this endpoint to send - messages to the broadcast address. - - * *sock* can optionally be specified in order to use a preexisting, - already connected, :class:`socket.socket` object to be used by the - transport. If specified, *local_addr* and *remote_addr* should be omitted - (must be :const:`None`). - - On Windows with :class:`ProactorEventLoop`, this method is not supported. - - See :ref:`UDP echo client protocol ` and - :ref:`UDP echo server protocol ` examples. - - .. versionchanged:: 3.4.4 - The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, - *allow_broadcast*, and *sock* parameters were added. - - .. versionchanged:: 3.6.10 - The *reuse_address* parameter is no longer supporter due to security - concerns - - -.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None) - - Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket - type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket - family is used to communicate between processes on the same machine - efficiently. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - *path* is the name of a UNIX domain socket, and is required unless a *sock* - parameter is specified. Abstract UNIX sockets, :class:`str`, and - :class:`bytes` paths are supported. - - See the :meth:`AbstractEventLoop.create_connection` method for parameters. - - Availability: UNIX. - - -Creating listening connections ------------------------------- - -.. coroutinemethod:: AbstractEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None) - - Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to - *host* and *port*. - - Return a :class:`Server` object, its :attr:`~Server.sockets` attribute - contains created sockets. Use the :meth:`Server.close` method to stop the - server: close listening sockets. - - Parameters: - - * The *host* parameter can be a string, in that case the TCP server is - bound to *host* and *port*. The *host* parameter can also be a sequence - of strings and in that case the TCP server is bound to all hosts of the - sequence. If *host* is an empty string or ``None``, all interfaces are - assumed and a list of multiple sockets will be returned (most likely one - for IPv4 and another one for IPv6). - - * *family* can be set to either :data:`socket.AF_INET` or - :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set - it will be determined from host (defaults to :data:`socket.AF_UNSPEC`). - - * *flags* is a bitmask for :meth:`getaddrinfo`. - - * *sock* can optionally be specified in order to use a preexisting - socket object. If specified, *host* and *port* should be omitted (must be - :const:`None`). - - * *backlog* is the maximum number of queued connections passed to - :meth:`~socket.socket.listen` (defaults to 100). - - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. - - * *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified will automatically be set to ``True`` on - UNIX. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on - Windows. - - This method is a :ref:`coroutine `. - - .. versionchanged:: 3.5 - - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. - - .. seealso:: - - The function :func:`start_server` creates a (:class:`StreamReader`, - :class:`StreamWriter`) pair and calls back a function with this pair. - - .. versionchanged:: 3.5.1 - - The *host* parameter can now be a sequence of strings. - - -.. coroutinemethod:: AbstractEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None) - - Similar to :meth:`AbstractEventLoop.create_server`, but specific to the - socket family :py:data:`~socket.AF_UNIX`. - - This method is a :ref:`coroutine `. - - Availability: UNIX. - -.. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None) - - Handle an accepted connection. - - This is used by servers that accept connections outside of - asyncio but that use asyncio to handle them. - - Parameters: - - * *sock* is a preexisting socket object returned from an ``accept`` - call. - - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. - - This method is a :ref:`coroutine `. When completed, the - coroutine returns a ``(transport, protocol)`` pair. - - .. versionadded:: 3.5.3 - - -Watch file descriptors ----------------------- - -On Windows with :class:`SelectorEventLoop`, only socket handles are supported -(ex: pipe file descriptors are not supported). - -On Windows with :class:`ProactorEventLoop`, these methods are not supported. - -.. method:: AbstractEventLoop.add_reader(fd, callback, \*args) - - Start watching the file descriptor for read availability and then call the - *callback* with specified arguments. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.remove_reader(fd) - - Stop watching the file descriptor for read availability. - -.. method:: AbstractEventLoop.add_writer(fd, callback, \*args) - - Start watching the file descriptor for write availability and then call the - *callback* with specified arguments. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.remove_writer(fd) - - Stop watching the file descriptor for write availability. - -The :ref:`watch a file descriptor for read events ` -example uses the low-level :meth:`AbstractEventLoop.add_reader` method to register -the file descriptor of a socket. - - -Low-level socket operations ---------------------------- - -.. coroutinemethod:: AbstractEventLoop.sock_recv(sock, nbytes) - - Receive data from the socket. Modeled after blocking - :meth:`socket.socket.recv` method. - - The return value is a bytes object - representing the data received. The maximum amount of data to be received - at once is specified by *nbytes*. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - -.. coroutinemethod:: AbstractEventLoop.sock_sendall(sock, data) - - Send data to the socket. Modeled after blocking - :meth:`socket.socket.sendall` method. - - The socket must be connected to a remote socket. - This method continues to send data from *data* until either all data has - been sent or an error occurs. ``None`` is returned on success. On error, - an exception is raised, and there is no way to determine how much data, if - any, was successfully processed by the receiving end of the connection. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - -.. coroutinemethod:: AbstractEventLoop.sock_connect(sock, address) - - Connect to a remote socket at *address*. Modeled after - blocking :meth:`socket.socket.connect` method. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - - .. versionchanged:: 3.5.2 - ``address`` no longer needs to be resolved. ``sock_connect`` - will try to check if the *address* is already resolved by calling - :func:`socket.inet_pton`. If not, - :meth:`AbstractEventLoop.getaddrinfo` will be used to resolve the - *address*. - - .. seealso:: - - :meth:`AbstractEventLoop.create_connection` - and :func:`asyncio.open_connection() `. - - -.. coroutinemethod:: AbstractEventLoop.sock_accept(sock) - - Accept a connection. Modeled after blocking - :meth:`socket.socket.accept`. - - The socket must be bound to an address and listening - for connections. The return value is a pair ``(conn, address)`` where *conn* - is a *new* socket object usable to send and receive data on the connection, - and *address* is the address bound to the socket on the other end of the - connection. - - The socket *sock* must be non-blocking. - - This method is a :ref:`coroutine `. - - .. seealso:: - - :meth:`AbstractEventLoop.create_server` and :func:`start_server`. - - -Resolve host name ------------------ - -.. coroutinemethod:: AbstractEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0) - - This method is a :ref:`coroutine `, similar to - :meth:`socket.getaddrinfo` function but non-blocking. - -.. coroutinemethod:: AbstractEventLoop.getnameinfo(sockaddr, flags=0) - - This method is a :ref:`coroutine `, similar to - :meth:`socket.getnameinfo` function but non-blocking. - - -Connect pipes -------------- - -On Windows with :class:`SelectorEventLoop`, these methods are not supported. -Use :class:`ProactorEventLoop` to support pipes on Windows. - -.. coroutinemethod:: AbstractEventLoop.connect_read_pipe(protocol_factory, pipe) - - Register read pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`Protocol` - interface. *pipe* is a :term:`file-like object `. - Return pair ``(transport, protocol)``, where *transport* supports the - :class:`ReadTransport` interface. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - - This method is a :ref:`coroutine `. - -.. coroutinemethod:: AbstractEventLoop.connect_write_pipe(protocol_factory, pipe) - - Register write pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`BaseProtocol` - interface. *pipe* is :term:`file-like object `. - Return pair ``(transport, protocol)``, where *transport* supports - :class:`WriteTransport` interface. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - - This method is a :ref:`coroutine `. - -.. seealso:: - - The :meth:`AbstractEventLoop.subprocess_exec` and - :meth:`AbstractEventLoop.subprocess_shell` methods. - - -UNIX signals ------------- - -Availability: UNIX only. - -.. method:: AbstractEventLoop.add_signal_handler(signum, callback, \*args) - - Add a handler for a signal. - - Raise :exc:`ValueError` if the signal number is invalid or uncatchable. - Raise :exc:`RuntimeError` if there is a problem setting up the handler. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: AbstractEventLoop.remove_signal_handler(sig) - - Remove a handler for a signal. - - Return ``True`` if a signal handler was removed, ``False`` if not. - -.. seealso:: - - The :mod:`signal` module. - - -Executor --------- - -Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or -pool of processes). By default, an event loop uses a thread pool executor -(:class:`~concurrent.futures.ThreadPoolExecutor`). - -.. coroutinemethod:: AbstractEventLoop.run_in_executor(executor, func, \*args) - - Arrange for a *func* to be called in the specified executor. - - The *executor* argument should be an :class:`~concurrent.futures.Executor` - instance. The default executor is used if *executor* is ``None``. - - :ref:`Use functools.partial to pass keywords to the *func* - `. - - This method is a :ref:`coroutine `. - - .. versionchanged:: 3.5.3 - :meth:`BaseEventLoop.run_in_executor` no longer configures the - ``max_workers`` of the thread pool executor it creates, instead - leaving it up to the thread pool executor - (:class:`~concurrent.futures.ThreadPoolExecutor`) to set the - default. - -.. method:: AbstractEventLoop.set_default_executor(executor) - - Set the default executor used by :meth:`run_in_executor`. - - -Error Handling API ------------------- - -Allows customizing how exceptions are handled in the event loop. - -.. method:: AbstractEventLoop.set_exception_handler(handler) - - Set *handler* as the new event loop exception handler. - - If *handler* is ``None``, the default exception handler will - be set. - - If *handler* is a callable object, it should have a - matching signature to ``(loop, context)``, where ``loop`` - will be a reference to the active event loop, ``context`` - will be a ``dict`` object (see :meth:`call_exception_handler` - documentation for details about context). - -.. method:: AbstractEventLoop.get_exception_handler() - - Return the exception handler, or ``None`` if the default one - is in use. - - .. versionadded:: 3.5.2 - -.. method:: AbstractEventLoop.default_exception_handler(context) - - Default exception handler. - - This is called when an exception occurs and no exception - handler is set, and can be called by a custom exception - handler that wants to defer to the default behavior. - - *context* parameter has the same meaning as in - :meth:`call_exception_handler`. - -.. method:: AbstractEventLoop.call_exception_handler(context) - - Call the current event loop exception handler. - - *context* is a ``dict`` object containing the following keys - (new keys may be introduced later): - - * 'message': Error message; - * 'exception' (optional): Exception object; - * 'future' (optional): :class:`asyncio.Future` instance; - * 'handle' (optional): :class:`asyncio.Handle` instance; - * 'protocol' (optional): :ref:`Protocol ` instance; - * 'transport' (optional): :ref:`Transport ` instance; - * 'socket' (optional): :class:`socket.socket` instance. - - .. note:: - - Note: this method should not be overloaded in subclassed - event loops. For any custom exception handling, use - :meth:`set_exception_handler()` method. - -Debug mode ----------- - -.. method:: AbstractEventLoop.get_debug() - - Get the debug mode (:class:`bool`) of the event loop. - - The default value is ``True`` if the environment variable - :envvar:`PYTHONASYNCIODEBUG` is set to a non-empty string, ``False`` - otherwise. - - .. versionadded:: 3.4.2 - -.. method:: AbstractEventLoop.set_debug(enabled: bool) - - Set the debug mode of the event loop. - - .. versionadded:: 3.4.2 - -.. seealso:: - - The :ref:`debug mode of asyncio `. - -Server ------- - -.. class:: Server - - Server listening on sockets. - - Object created by the :meth:`AbstractEventLoop.create_server` method and the - :func:`start_server` function. Don't instantiate the class directly. - - .. method:: close() - - Stop serving: close listening sockets and set the :attr:`sockets` - attribute to ``None``. - - The sockets that represent existing incoming client connections are left - open. - - The server is closed asynchronously, use the :meth:`wait_closed` - coroutine to wait until the server is closed. - - .. coroutinemethod:: wait_closed() - - Wait until the :meth:`close` method completes. - - This method is a :ref:`coroutine `. - - .. attribute:: sockets - - List of :class:`socket.socket` objects the server is listening to, or - ``None`` if the server is closed. - - -Handle ------- - -.. class:: Handle - - A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`, - :func:`AbstractEventLoop.call_soon_threadsafe`, :func:`AbstractEventLoop.call_later`, - and :func:`AbstractEventLoop.call_at`. - - .. method:: cancel() - - Cancel the call. If the callback is already canceled or executed, - this method has no effect. - - -Event loop examples -------------------- - -.. _asyncio-hello-world-callback: - -Hello World with call_soon() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example using the :meth:`AbstractEventLoop.call_soon` method to schedule a -callback. The callback displays ``"Hello World"`` and then stops the event -loop:: - - import asyncio - - def hello_world(loop): - print('Hello World') - loop.stop() - - loop = asyncio.get_event_loop() - - # Schedule a call to hello_world() - loop.call_soon(hello_world, loop) - - # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() - -.. seealso:: - - The :ref:`Hello World coroutine ` example - uses a :ref:`coroutine `. - - -.. _asyncio-date-callback: - -Display the current date with call_later() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of callback displaying the current date every second. The callback uses -the :meth:`AbstractEventLoop.call_later` method to reschedule itself during 5 -seconds, and then stops the event loop:: - - import asyncio - import datetime - - def display_date(end_time, loop): - print(datetime.datetime.now()) - if (loop.time() + 1.0) < end_time: - loop.call_later(1, display_date, end_time, loop) - else: - loop.stop() - - loop = asyncio.get_event_loop() - - # Schedule the first call to display_date() - end_time = loop.time() + 5.0 - loop.call_soon(display_date, end_time, loop) - - # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() - -.. seealso:: - - The :ref:`coroutine displaying the current date - ` example uses a :ref:`coroutine - `. - - -.. _asyncio-watch-read-event: - -Watch a file descriptor for read events -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Wait until a file descriptor received some data using the -:meth:`AbstractEventLoop.add_reader` method and then close the event loop:: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - # Create a pair of connected file descriptors - rsock, wsock = socketpair() - loop = asyncio.get_event_loop() - - def reader(): - data = rsock.recv(100) - print("Received:", data.decode()) - # We are done: unregister the file descriptor - loop.remove_reader(rsock) - # Stop the event loop - loop.stop() - - # Register the file descriptor for read event - loop.add_reader(rsock, reader) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() - -.. seealso:: - - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`AbstractEventLoop.create_connection` method. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. - - -Set signal handlers for SIGINT and SIGTERM -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` using -the :meth:`AbstractEventLoop.add_signal_handler` method:: - - import asyncio - import functools - import os - import signal - - def ask_exit(signame): - print("got signal %s: exit" % signame) - loop.stop() - - loop = asyncio.get_event_loop() - for signame in ('SIGINT', 'SIGTERM'): - loop.add_signal_handler(getattr(signal, signame), - functools.partial(ask_exit, signame)) - - print("Event loop running forever, press Ctrl+C to interrupt.") - print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid()) - try: - loop.run_forever() - finally: - loop.close() - -This example only works on UNIX. diff --git a/third_party/python/Doc/library/asyncio-eventloops.rst b/third_party/python/Doc/library/asyncio-eventloops.rst deleted file mode 100644 index 3aa9ffeff..000000000 --- a/third_party/python/Doc/library/asyncio-eventloops.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. currentmodule:: asyncio - -Event loops -=========== - -**Source code:** :source:`Lib/asyncio/events.py` - -Event loop functions --------------------- - -The following functions are convenient shortcuts to accessing the methods of the -global policy. Note that this provides access to the default policy, unless an -alternative policy was set by calling :func:`set_event_loop_policy` earlier in -the execution of the process. - -.. function:: get_event_loop() - - Equivalent to calling ``get_event_loop_policy().get_event_loop()``. - -.. function:: set_event_loop(loop) - - Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``. - -.. function:: new_event_loop() - - Equivalent to calling ``get_event_loop_policy().new_event_loop()``. - - -.. _asyncio-event-loops: - -Available event loops ---------------------- - -asyncio currently provides two implementations of event loops: -:class:`SelectorEventLoop` and :class:`ProactorEventLoop`. - -.. class:: SelectorEventLoop - - Event loop based on the :mod:`selectors` module. Subclass of - :class:`AbstractEventLoop`. - - Use the most efficient selector available on the platform. - - On Windows, only sockets are supported (ex: pipes are not supported): - see the `MSDN documentation of select - `_. - -.. class:: ProactorEventLoop - - Proactor event loop for Windows using "I/O Completion Ports" aka IOCP. - Subclass of :class:`AbstractEventLoop`. - - Availability: Windows. - - .. seealso:: - - `MSDN documentation on I/O Completion Ports - `_. - -Example to use a :class:`ProactorEventLoop` on Windows:: - - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. _asyncio-platform-support: - -Platform support ----------------- - -The :mod:`asyncio` module has been designed to be portable, but each platform -still has subtle differences and may not support all :mod:`asyncio` features. - -Windows -^^^^^^^ - -Common limits of Windows event loops: - -- :meth:`~AbstractEventLoop.create_unix_connection` and - :meth:`~AbstractEventLoop.create_unix_server` are not supported: the socket - family :data:`socket.AF_UNIX` is specific to UNIX -- :meth:`~AbstractEventLoop.add_signal_handler` and - :meth:`~AbstractEventLoop.remove_signal_handler` are not supported -- :meth:`EventLoopPolicy.set_child_watcher` is not supported. - :class:`ProactorEventLoop` supports subprocesses. It has only one - implementation to watch child processes, there is no need to configure it. - -:class:`SelectorEventLoop` specific limits: - -- :class:`~selectors.SelectSelector` is used which only supports sockets - and is limited to 512 sockets. -- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` only - accept file descriptors of sockets -- Pipes are not supported - (ex: :meth:`~AbstractEventLoop.connect_read_pipe`, - :meth:`~AbstractEventLoop.connect_write_pipe`) -- :ref:`Subprocesses ` are not supported - (ex: :meth:`~AbstractEventLoop.subprocess_exec`, - :meth:`~AbstractEventLoop.subprocess_shell`) - -:class:`ProactorEventLoop` specific limits: - -- :meth:`~AbstractEventLoop.create_datagram_endpoint` (UDP) is not supported -- :meth:`~AbstractEventLoop.add_reader` and :meth:`~AbstractEventLoop.add_writer` are - not supported - -The resolution of the monotonic clock on Windows is usually around 15.6 msec. -The best resolution is 0.5 msec. The resolution depends on the hardware -(availability of `HPET -`_) and on the Windows -configuration. See :ref:`asyncio delayed calls `. - -.. versionchanged:: 3.5 - - :class:`ProactorEventLoop` now supports SSL. - - -Mac OS X -^^^^^^^^ - -Character devices like PTY are only well supported since Mavericks (Mac OS -10.9). They are not supported at all on Mac OS 10.5 and older. - -On Mac OS 10.6, 10.7 and 10.8, the default event loop is -:class:`SelectorEventLoop` which uses :class:`selectors.KqueueSelector`. -:class:`selectors.KqueueSelector` does not support character devices on these -versions. The :class:`SelectorEventLoop` can be used with -:class:`~selectors.SelectSelector` or :class:`~selectors.PollSelector` to -support character devices on these versions of Mac OS X. Example:: - - import asyncio - import selectors - - selector = selectors.SelectSelector() - loop = asyncio.SelectorEventLoop(selector) - asyncio.set_event_loop(loop) - - -Event loop policies and the default policy ------------------------------------------- - -Event loop management is abstracted with a *policy* pattern, to provide maximal -flexibility for custom platforms and frameworks. Throughout the execution of a -process, a single global policy object manages the event loops available to the -process based on the calling context. A policy is an object implementing the -:class:`AbstractEventLoopPolicy` interface. - -For most users of :mod:`asyncio`, policies never have to be dealt with -explicitly, since the default global policy is sufficient (see below). - -The module-level functions -:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to -event loops managed by the default policy. - - -Event loop policy interface ---------------------------- - -An event loop policy must implement the following interface: - -.. class:: AbstractEventLoopPolicy - - Event loop policy. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Returns an event loop object implementing the :class:`AbstractEventLoop` - interface. In case called from coroutine, it returns the currently - running event loop. - - Raises an exception in case no event loop has been set for the current - context and the current policy does not specify to create one. It must - never return ``None``. - - .. versionchanged:: 3.6 - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object according to this policy's - rules. - - If there's need to set this loop as the event loop for the current - context, :meth:`set_event_loop` must be called explicitly. - - -The default policy defines context as the current thread, and manages an event -loop per thread that interacts with :mod:`asyncio`. If the current thread -doesn't already have an event loop associated with it, the default policy's -:meth:`~AbstractEventLoopPolicy.get_event_loop` method creates one when -called from the main thread, but raises :exc:`RuntimeError` otherwise. - - -Access to the global loop policy --------------------------------- - -.. function:: get_event_loop_policy() - - Get the current event loop policy. - -.. function:: set_event_loop_policy(policy) - - Set the current event loop policy. If *policy* is ``None``, the default - policy is restored. - - -Customizing the event loop policy ---------------------------------- - -To implement a new event loop policy, it is recommended you subclass the -concrete default event loop policy :class:`DefaultEventLoopPolicy` -and override the methods for which you want to change behavior, for example:: - - class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - loop = super().get_event_loop() - # Do something with loop ... - return loop - - asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff --git a/third_party/python/Doc/library/asyncio-protocol.rst b/third_party/python/Doc/library/asyncio-protocol.rst deleted file mode 100644 index 9605261c0..000000000 --- a/third_party/python/Doc/library/asyncio-protocol.rst +++ /dev/null @@ -1,750 +0,0 @@ -.. currentmodule:: asyncio - -+++++++++++++++++++++++++++++++++++++++++++++ -Transports and protocols (callback based API) -+++++++++++++++++++++++++++++++++++++++++++++ - -**Source code:** :source:`Lib/asyncio/transports.py` - -**Source code:** :source:`Lib/asyncio/protocols.py` - -.. _asyncio-transport: - -Transports -========== - -Transports are classes provided by :mod:`asyncio` in order to abstract -various kinds of communication channels. You generally won't instantiate -a transport yourself; instead, you will call an :class:`AbstractEventLoop` method -which will create the transport and try to initiate the underlying -communication channel, calling you back when it succeeds. - -Once the communication channel is established, a transport is always -paired with a :ref:`protocol ` instance. The protocol can -then call the transport's methods for various purposes. - -:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and -subprocess pipes. The methods available on a transport depend on -the transport's kind. - -The transport classes are :ref:`not thread safe `. - -.. versionchanged:: 3.6 - The socket option ``TCP_NODELAY`` is now set by default. - - -BaseTransport -------------- - -.. class:: BaseTransport - - Base class for transports. - - .. method:: close() - - Close the transport. If the transport has a buffer for outgoing - data, buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's :meth:`connection_lost` method will be called with - :const:`None` as its argument. - - .. method:: is_closing() - - Return ``True`` if the transport is closing or is closed. - - .. versionadded:: 3.5.1 - - .. method:: get_extra_info(name, default=None) - - Return optional transport information. *name* is a string representing - the piece of transport-specific information to get, *default* is the - value to return if the information doesn't exist. - - This method allows transport implementations to easily expose - channel-specific information. - - * socket: - - - ``'peername'``: the remote address to which the socket is connected, - result of :meth:`socket.socket.getpeername` (``None`` on error) - - ``'socket'``: :class:`socket.socket` instance - - ``'sockname'``: the socket's own address, - result of :meth:`socket.socket.getsockname` - - * SSL socket: - - - ``'compression'``: the compression algorithm being used as a string, - or ``None`` if the connection isn't compressed; result of - :meth:`ssl.SSLSocket.compression` - - ``'cipher'``: a three-value tuple containing the name of the cipher - being used, the version of the SSL protocol that defines its use, and - the number of secret bits being used; result of - :meth:`ssl.SSLSocket.cipher` - - ``'peercert'``: peer certificate; result of - :meth:`ssl.SSLSocket.getpeercert` - - ``'sslcontext'``: :class:`ssl.SSLContext` instance - - ``'ssl_object'``: :class:`ssl.SSLObject` or :class:`ssl.SSLSocket` - instance - - * pipe: - - - ``'pipe'``: pipe object - - * subprocess: - - - ``'subprocess'``: :class:`subprocess.Popen` instance - - .. method:: set_protocol(protocol) - - Set a new protocol. Switching protocol should only be done when both - protocols are documented to support the switch. - - .. versionadded:: 3.5.3 - - .. method:: get_protocol - - Return the current protocol. - - .. versionadded:: 3.5.3 - - .. versionchanged:: 3.5.1 - ``'ssl_object'`` info was added to SSL sockets. - - -ReadTransport -------------- - -.. class:: ReadTransport - - Interface for read-only transports. - - .. method:: pause_reading() - - Pause the receiving end of the transport. No data will be passed to - the protocol's :meth:`data_received` method until :meth:`resume_reading` - is called. - - .. versionchanged:: 3.6.7 - The method is idempotent, i.e. it can be called when the - transport is already paused or closed. - - .. method:: resume_reading() - - Resume the receiving end. The protocol's :meth:`data_received` method - will be called once again if some data is available for reading. - - .. versionchanged:: 3.6.7 - The method is idempotent, i.e. it can be called when the - transport is already reading. - - -WriteTransport --------------- - -.. class:: WriteTransport - - Interface for write-only transports. - - .. method:: abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. - - .. method:: get_write_buffer_size() - - Return the current size of the output buffer used by the transport. - - .. method:: get_write_buffer_limits() - - Get the *high*- and *low*-water limits for write flow control. Return a - tuple ``(low, high)`` where *low* and *high* are positive number of - bytes. - - Use :meth:`set_write_buffer_limits` to set the limits. - - .. versionadded:: 3.4.2 - - .. method:: set_write_buffer_limits(high=None, low=None) - - Set the *high*- and *low*-water limits for write flow control. - - These two values (measured in number of - bytes) control when the protocol's - :meth:`pause_writing` and :meth:`resume_writing` methods are called. - If specified, the low-water limit must be less than or equal to the - high-water limit. Neither *high* nor *low* can be negative. - - :meth:`pause_writing` is called when the buffer size becomes greater - than or equal to the *high* value. If writing has been paused, - :meth:`resume_writing` is called when the buffer size becomes less - than or equal to the *low* value. - - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to an - implementation-specific value less than or equal to the - high-water limit. Setting *high* to zero forces *low* to zero as - well, and causes :meth:`pause_writing` to be called whenever the - buffer becomes non-empty. Setting *low* to zero causes - :meth:`resume_writing` to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. - - Use :meth:`get_write_buffer_limits` to get the limits. - - .. method:: write(data) - - Write some *data* bytes to the transport. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - - .. method:: writelines(list_of_data) - - Write a list (or any iterable) of data bytes to the transport. - This is functionally equivalent to calling :meth:`write` on each - element yielded by the iterable, but may be implemented more efficiently. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data. - Data may still be received. - - This method can raise :exc:`NotImplementedError` if the transport - (e.g. SSL) doesn't support half-closes. - - -DatagramTransport ------------------ - -.. method:: DatagramTransport.sendto(data, addr=None) - - Send the *data* bytes to the remote peer given by *addr* (a - transport-dependent target address). If *addr* is :const:`None`, the - data is sent to the target address given on transport creation. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - -.. method:: DatagramTransport.abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - -BaseSubprocessTransport ------------------------ - -.. class:: BaseSubprocessTransport - - .. method:: get_pid() - - Return the subprocess process id as an integer. - - .. method:: get_pipe_transport(fd) - - Return the transport for the communication pipe corresponding to the - integer file descriptor *fd*: - - * ``0``: readable streaming transport of the standard input (*stdin*), - or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), - or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), - or :const:`None` if the subprocess was not created with ``stderr=PIPE`` - * other *fd*: :const:`None` - - .. method:: get_returncode() - - Return the subprocess returncode as an integer or :const:`None` - if it hasn't returned, similarly to the - :attr:`subprocess.Popen.returncode` attribute. - - .. method:: kill() - - Kill the subprocess, as in :meth:`subprocess.Popen.kill`. - - On POSIX systems, the function sends SIGKILL to the subprocess. - On Windows, this method is an alias for :meth:`terminate`. - - .. method:: send_signal(signal) - - Send the *signal* number to the subprocess, as in - :meth:`subprocess.Popen.send_signal`. - - .. method:: terminate() - - Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. - This method is an alias for the :meth:`close` method. - - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to - stop the subprocess. - - .. method:: close() - - Ask the subprocess to stop by calling the :meth:`terminate` method if the - subprocess hasn't returned yet, and close transports of all pipes - (*stdin*, *stdout* and *stderr*). - - -.. _asyncio-protocol: - -Protocols -========= - -:mod:`asyncio` provides base classes that you can subclass to implement -your network protocols. Those classes are used in conjunction with -:ref:`transports ` (see below): the protocol parses incoming -data and asks for the writing of outgoing data, while the transport is -responsible for the actual I/O and buffering. - -When subclassing a protocol class, it is recommended you override certain -methods. Those methods are callbacks: they will be called by the transport -on certain events (for example when some data is received); you shouldn't -call them yourself, unless you are implementing a transport. - -.. note:: - All callbacks have default implementations, which are empty. Therefore, - you only need to implement the callbacks for the events in which you - are interested. - - -Protocol classes ----------------- - -.. class:: Protocol - - The base class for implementing streaming protocols (for use with - e.g. TCP and SSL transports). - -.. class:: DatagramProtocol - - The base class for implementing datagram protocols (for use with - e.g. UDP transports). - -.. class:: SubprocessProtocol - - The base class for implementing protocols communicating with child - processes (through a set of unidirectional pipes). - - -Connection callbacks --------------------- - -These callbacks may be called on :class:`Protocol`, :class:`DatagramProtocol` -and :class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.connection_made(transport) - - Called when a connection is made. - - The *transport* argument is the transport representing the - connection. You are responsible for storing it somewhere - (e.g. as an attribute) if you need to. - -.. method:: BaseProtocol.connection_lost(exc) - - Called when the connection is lost or closed. - - The argument is either an exception object or :const:`None`. - The latter means a regular EOF is received, or the connection was - aborted or closed by this side of the connection. - -:meth:`~BaseProtocol.connection_made` and :meth:`~BaseProtocol.connection_lost` -are called exactly once per successful connection. All other callbacks will be -called between those two methods, which allows for easier resource management -in your protocol implementation. - -The following callbacks may be called only on :class:`SubprocessProtocol` -instances: - -.. method:: SubprocessProtocol.pipe_data_received(fd, data) - - Called when the child process writes data into its stdout or stderr pipe. - *fd* is the integer file descriptor of the pipe. *data* is a non-empty - bytes object containing the data. - -.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) - - Called when one of the pipes communicating with the child process - is closed. *fd* is the integer file descriptor that was closed. - -.. method:: SubprocessProtocol.process_exited() - - Called when the child process has exited. - - -Streaming protocols -------------------- - -The following callbacks are called on :class:`Protocol` instances: - -.. method:: Protocol.data_received(data) - - Called when some data is received. *data* is a non-empty bytes object - containing the incoming data. - - .. note:: - Whether the data is buffered, chunked or reassembled depends on - the transport. In general, you shouldn't rely on specific semantics - and instead make your parsing generic and flexible enough. However, - data is always received in the correct order. - -.. method:: Protocol.eof_received() - - Called when the other end signals it won't send any more data - (for example by calling :meth:`write_eof`, if the other end also uses - asyncio). - - This method may return a false value (including ``None``), in which case - the transport will close itself. Conversely, if this method returns a - true value, closing the transport is up to the protocol. Since the - default implementation returns ``None``, it implicitly closes the connection. - - .. note:: - Some transports such as SSL don't support half-closed connections, - in which case returning true from this method will not prevent closing - the connection. - -:meth:`data_received` can be called an arbitrary number of times during -a connection. However, :meth:`eof_received` is called at most once -and, if called, :meth:`data_received` won't be called after it. - -State machine: - - start -> :meth:`~BaseProtocol.connection_made` - [-> :meth:`~Protocol.data_received` \*] - [-> :meth:`~Protocol.eof_received` ?] - -> :meth:`~BaseProtocol.connection_lost` -> end - - -Datagram protocols ------------------- - -The following callbacks are called on :class:`DatagramProtocol` instances. - -.. method:: DatagramProtocol.datagram_received(data, addr) - - Called when a datagram is received. *data* is a bytes object containing - the incoming data. *addr* is the address of the peer sending the data; - the exact format depends on the transport. - -.. method:: DatagramProtocol.error_received(exc) - - Called when a previous send or receive operation raises an - :class:`OSError`. *exc* is the :class:`OSError` instance. - - This method is called in rare conditions, when the transport (e.g. UDP) - detects that a datagram couldn't be delivered to its recipient. - In many conditions though, undeliverable datagrams will be silently - dropped. - - -Flow control callbacks ----------------------- - -These callbacks may be called on :class:`Protocol`, -:class:`DatagramProtocol` and :class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.pause_writing() - - Called when the transport's buffer goes over the high-water mark. - -.. method:: BaseProtocol.resume_writing() - - Called when the transport's buffer drains below the low-water mark. - - -:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- -:meth:`pause_writing` is called once when the buffer goes strictly over -the high-water mark (even if subsequent writes increases the buffer size -even more), and eventually :meth:`resume_writing` is called once when the -buffer size reaches the low-water mark. - -.. note:: - If the buffer size equals the high-water mark, - :meth:`pause_writing` is not called -- it must go strictly over. - Conversely, :meth:`resume_writing` is called when the buffer size is - equal or lower than the low-water mark. These end conditions - are important to ensure that things go as expected when either - mark is zero. - -.. note:: - On BSD systems (OS X, FreeBSD, etc.) flow control is not supported - for :class:`DatagramProtocol`, because send failures caused by - writing too many packets cannot be detected easily. The socket - always appears 'ready' and excess packets are dropped; an - :class:`OSError` with errno set to :const:`errno.ENOBUFS` may or - may not be raised; if it is raised, it will be reported to - :meth:`DatagramProtocol.error_received` but otherwise ignored. - - -Coroutines and protocols ------------------------- - -Coroutines can be scheduled in a protocol method using :func:`ensure_future`, -but there is no guarantee made about the execution order. Protocols are not -aware of coroutines created in protocol methods and so will not wait for them. - -To have a reliable execution order, use :ref:`stream objects ` in a -coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain` -coroutine can be used to wait until the write buffer is flushed. - - -Protocol examples -================= - -.. _asyncio-tcp-echo-client-protocol: - -TCP echo client protocol ------------------------- - -TCP echo client using the :meth:`AbstractEventLoop.create_connection` method, send -data and wait until the connection is closed:: - - import asyncio - - class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, loop): - self.message = message - self.loop = loop - - def connection_made(self, transport): - transport.write(self.message.encode()) - print('Data sent: {!r}'.format(self.message)) - - def data_received(self, data): - print('Data received: {!r}'.format(data.decode())) - - def connection_lost(self, exc): - print('The server closed the connection') - print('Stop the event loop') - self.loop.stop() - - loop = asyncio.get_event_loop() - message = 'Hello World!' - coro = loop.create_connection(lambda: EchoClientProtocol(message, loop), - '127.0.0.1', 8888) - loop.run_until_complete(coro) - loop.run_forever() - loop.close() - -The event loop is running twice. The -:meth:`~AbstractEventLoop.run_until_complete` method is preferred in this short -example to raise an exception if the server is not listening, instead of -having to write a short coroutine to handle the exception and stop the -running loop. At :meth:`~AbstractEventLoop.run_until_complete` exit, the loop is -no longer running, so there is no need to stop the loop in case of an error. - -.. seealso:: - - The :ref:`TCP echo client using streams ` - example uses the :func:`asyncio.open_connection` function. - - -.. _asyncio-tcp-echo-server-protocol: - -TCP echo server protocol ------------------------- - -TCP echo server using the :meth:`AbstractEventLoop.create_server` method, send back -received data and close the connection:: - - import asyncio - - class EchoServerClientProtocol(asyncio.Protocol): - def connection_made(self, transport): - peername = transport.get_extra_info('peername') - print('Connection from {}'.format(peername)) - self.transport = transport - - def data_received(self, data): - message = data.decode() - print('Data received: {!r}'.format(message)) - - print('Send: {!r}'.format(message)) - self.transport.write(data) - - print('Close the client socket') - self.transport.close() - - loop = asyncio.get_event_loop() - # Each client connection will create a new protocol instance - coro = loop.create_server(EchoServerClientProtocol, '127.0.0.1', 8888) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - -:meth:`Transport.close` can be called immediately after -:meth:`WriteTransport.write` even if data are not sent yet on the socket: both -methods are asynchronous. ``yield from`` is not needed because these transport -methods are not coroutines. - -.. seealso:: - - The :ref:`TCP echo server using streams ` - example uses the :func:`asyncio.start_server` function. - - -.. _asyncio-udp-echo-client-protocol: - -UDP echo client protocol ------------------------- - -UDP echo client using the :meth:`AbstractEventLoop.create_datagram_endpoint` -method, send data and close the transport when we received the answer:: - - import asyncio - - class EchoClientProtocol: - def __init__(self, message, loop): - self.message = message - self.loop = loop - self.transport = None - - def connection_made(self, transport): - self.transport = transport - print('Send:', self.message) - self.transport.sendto(self.message.encode()) - - def datagram_received(self, data, addr): - print("Received:", data.decode()) - - print("Close the socket") - self.transport.close() - - def error_received(self, exc): - print('Error received:', exc) - - def connection_lost(self, exc): - print("Socket closed, stop the event loop") - loop = asyncio.get_event_loop() - loop.stop() - - loop = asyncio.get_event_loop() - message = "Hello World!" - connect = loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, loop), - remote_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(connect) - loop.run_forever() - transport.close() - loop.close() - - -.. _asyncio-udp-echo-server-protocol: - -UDP echo server protocol ------------------------- - -UDP echo server using the :meth:`AbstractEventLoop.create_datagram_endpoint` -method, send back received data:: - - import asyncio - - class EchoServerProtocol: - def connection_made(self, transport): - self.transport = transport - - def datagram_received(self, data, addr): - message = data.decode() - print('Received %r from %s' % (message, addr)) - print('Send %r to %s' % (message, addr)) - self.transport.sendto(data, addr) - - loop = asyncio.get_event_loop() - print("Starting UDP server") - # One protocol instance will be created to serve all client requests - listen = loop.create_datagram_endpoint( - EchoServerProtocol, local_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(listen) - - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - transport.close() - loop.close() - - -.. _asyncio-register-socket: - -Register an open socket to wait for data using a protocol ---------------------------------------------------------- - -Wait until a socket receives data using the -:meth:`AbstractEventLoop.create_connection` method with a protocol, and then close -the event loop :: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - # Create a pair of connected sockets - rsock, wsock = socketpair() - loop = asyncio.get_event_loop() - - class MyProtocol(asyncio.Protocol): - transport = None - - def connection_made(self, transport): - self.transport = transport - - def data_received(self, data): - print("Received:", data.decode()) - - # We are done: close the transport (it will call connection_lost()) - self.transport.close() - - def connection_lost(self, exc): - # The socket has been closed, stop the event loop - loop.stop() - - # Register the socket to wait for data - connect_coro = loop.create_connection(MyProtocol, sock=rsock) - transport, protocol = loop.run_until_complete(connect_coro) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() - -.. seealso:: - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`AbstractEventLoop.add_reader` method to register the file descriptor of a - socket. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. diff --git a/third_party/python/Doc/library/asyncio-queue.rst b/third_party/python/Doc/library/asyncio-queue.rst deleted file mode 100644 index ea7875500..000000000 --- a/third_party/python/Doc/library/asyncio-queue.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. currentmodule:: asyncio - -Queues -====== - -**Source code:** :source:`Lib/asyncio/queues.py` - -Queues: - -* :class:`Queue` -* :class:`PriorityQueue` -* :class:`LifoQueue` - -asyncio queue API was designed to be close to classes of the :mod:`queue` -module (:class:`~queue.Queue`, :class:`~queue.PriorityQueue`, -:class:`~queue.LifoQueue`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. - -Queue ------ - -.. class:: Queue(maxsize=0, \*, loop=None) - - A queue, useful for coordinating producer and consumer coroutines. - - If *maxsize* is less than or equal to zero, the queue size is infinite. If - it is an integer greater than ``0``, then ``yield from put()`` will block - when the queue reaches *maxsize*, until an item is removed by :meth:`get`. - - Unlike the standard library :mod:`queue`, you can reliably know this Queue's - size with :meth:`qsize`, since your single-threaded asyncio application won't - be interrupted between calling :meth:`qsize` and doing an operation on the - Queue. - - This class is :ref:`not thread safe `. - - .. versionchanged:: 3.4.4 - New :meth:`join` and :meth:`task_done` methods. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: full() - - Return ``True`` if there are :attr:`maxsize` items in the queue. - - .. note:: - - If the Queue was initialized with ``maxsize=0`` (the default), then - :meth:`full()` is never ``True``. - - .. coroutinemethod:: get() - - Remove and return an item from the queue. If queue is empty, wait until - an item is available. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`empty` method. - - .. method:: get_nowait() - - Remove and return an item from the queue. - - Return an item if one is immediately available, else raise - :exc:`QueueEmpty`. - - .. coroutinemethod:: join() - - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer thread calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`join` unblocks. - - This method is a :ref:`coroutine `. - - .. versionadded:: 3.4.4 - - .. coroutinemethod:: put(item) - - Put an item into the queue. If the queue is full, wait until a free slot - is available before adding item. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`full` method. - - .. method:: put_nowait(item) - - Put an item into the queue without blocking. - - If no free slot is immediately available, raise :exc:`QueueFull`. - - .. method:: qsize() - - Number of items in the queue. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a - subsequent call to :meth:`task_done` tells the queue that the processing - on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items - have been processed (meaning that a :meth:`task_done` call was received - for every item that had been :meth:`~Queue.put` into the queue). - - Raises :exc:`ValueError` if called more times than there were items - placed in the queue. - - .. versionadded:: 3.4.4 - - .. attribute:: maxsize - - Number of items allowed in the queue. - - -PriorityQueue -------------- - -.. class:: PriorityQueue - - A subclass of :class:`Queue`; retrieves entries in priority order (lowest - first). - - Entries are typically tuples of the form: (priority number, data). - - -LifoQueue ---------- - -.. class:: LifoQueue - - A subclass of :class:`Queue` that retrieves most recently added entries - first. - - -Exceptions -^^^^^^^^^^ - -.. exception:: QueueEmpty - - Exception raised when the :meth:`~Queue.get_nowait` method is called on a - :class:`Queue` object which is empty. - - -.. exception:: QueueFull - - Exception raised when the :meth:`~Queue.put_nowait` method is called on a - :class:`Queue` object which is full. diff --git a/third_party/python/Doc/library/asyncio-stream.rst b/third_party/python/Doc/library/asyncio-stream.rst deleted file mode 100644 index a510e1e63..000000000 --- a/third_party/python/Doc/library/asyncio-stream.rst +++ /dev/null @@ -1,471 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-streams: - -+++++++++++++++++++++++++++++ -Streams (coroutine based API) -+++++++++++++++++++++++++++++ - -**Source code:** :source:`Lib/asyncio/streams.py` - -Stream functions -================ - -.. note:: - - The top-level functions in this module are meant as convenience wrappers - only; there's really nothing special there, and if they don't do - exactly what you want, feel free to copy their code. - - -.. coroutinefunction:: open_connection(host=None, port=None, \*, loop=None, limit=None, \*\*kwds) - - A wrapper for :meth:`~AbstractEventLoop.create_connection()` returning a (reader, - writer) pair. - - The reader returned is a :class:`StreamReader` instance; the writer is - a :class:`StreamWriter` instance. - - The arguments are all the usual arguments to - :meth:`AbstractEventLoop.create_connection` except *protocol_factory*; most - common are positional host and port, with various optional keyword arguments - following. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - :class:`StreamReader`). - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, \*\*kwds) - - Start a socket server, with a callback for each client connected. The return - value is the same as :meth:`~AbstractEventLoop.create_server()`. - - The *client_connected_cb* parameter is called with two parameters: - *client_reader*, *client_writer*. *client_reader* is a - :class:`StreamReader` object, while *client_writer* is a - :class:`StreamWriter` object. The *client_connected_cb* parameter can - either be a plain callback function or a :ref:`coroutine function - `; if it is a coroutine function, it will be automatically - converted into a :class:`Task`. - - The rest of the arguments are all the usual arguments to - :meth:`~AbstractEventLoop.create_server()` except *protocol_factory*; most - common are positional *host* and *port*, with various optional keyword - arguments following. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - :class:`StreamReader`). - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, limit=None, **kwds) - - A wrapper for :meth:`~AbstractEventLoop.create_unix_connection()` returning - a (reader, writer) pair. - - See :func:`open_connection` for information about return value and other - details. - - This function is a :ref:`coroutine `. - - Availability: UNIX. - -.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, **kwds) - - Start a UNIX Domain Socket server, with a callback for each client connected. - - See :func:`start_server` for information about return value and other - details. - - This function is a :ref:`coroutine `. - - Availability: UNIX. - - -StreamReader -============ - -.. class:: StreamReader(limit=_DEFAULT_LIMIT, loop=None) - - This class is :ref:`not thread safe `. - - The *limit* argument's default value is set to _DEFAULT_LIMIT which is 2**16 (64 KiB) - - .. method:: exception() - - Get the exception. - - .. method:: feed_eof() - - Acknowledge the EOF. - - .. method:: feed_data(data) - - Feed *data* bytes in the internal buffer. Any operations waiting - for the data will be resumed. - - .. method:: set_exception(exc) - - Set the exception. - - .. method:: set_transport(transport) - - Set the transport. - - .. coroutinemethod:: read(n=-1) - - Read up to *n* bytes. If *n* is not provided, or set to ``-1``, - read until EOF and return all read bytes. - - If the EOF was received and the internal buffer is empty, - return an empty ``bytes`` object. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: readline() - - Read one line, where "line" is a sequence of bytes ending with ``\n``. - - If EOF is received, and ``\n`` was not found, the method will - return the partial read bytes. - - If the EOF was received and the internal buffer is empty, - return an empty ``bytes`` object. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: readexactly(n) - - Read exactly *n* bytes. Raise an :exc:`IncompleteReadError` if the end of - the stream is reached before *n* can be read, the - :attr:`IncompleteReadError.partial` attribute of the exception contains - the partial read bytes. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: readuntil(separator=b'\\n') - - Read data from the stream until ``separator`` is found. - - On success, the data and separator will be removed from the - internal buffer (consumed). Returned data will include the - separator at the end. - - Configured stream limit is used to check result. Limit sets the - maximal length of data that can be returned, not counting the - separator. - - If an EOF occurs and the complete separator is still not found, - an :exc:`IncompleteReadError` exception will be - raised, and the internal buffer will be reset. The - :attr:`IncompleteReadError.partial` attribute may contain the - separator partially. - - If the data cannot be read because of over limit, a - :exc:`LimitOverrunError` exception will be raised, and the data - will be left in the internal buffer, so it can be read again. - - .. versionadded:: 3.5.2 - - .. method:: at_eof() - - Return ``True`` if the buffer is empty and :meth:`feed_eof` was called. - - -StreamWriter -============ - -.. class:: StreamWriter(transport, protocol, reader, loop) - - Wraps a Transport. - - This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`, - :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds - :meth:`drain` which returns an optional :class:`Future` on which you can - wait for flow control. It also adds a transport attribute which references - the :class:`Transport` directly. - - This class is :ref:`not thread safe `. - - .. attribute:: transport - - Transport. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. See :meth:`WriteTransport.can_write_eof`. - - .. method:: close() - - Close the transport: see :meth:`BaseTransport.close`. - - .. coroutinemethod:: drain() - - Let the write buffer of the underlying transport a chance to be flushed. - - The intended use is to write:: - - w.write(data) - yield from w.drain() - - When the size of the transport buffer reaches the high-water limit (the - protocol is paused), block until the size of the buffer is drained down - to the low-water limit and the protocol is resumed. When there is nothing - to wait for, the yield-from continues immediately. - - Yielding from :meth:`drain` gives the opportunity for the loop to - schedule the write operation and flush the buffer. It should especially - be used when a possibly large amount of data is written to the transport, - and the coroutine does not yield-from between calls to :meth:`write`. - - This method is a :ref:`coroutine `. - - .. method:: get_extra_info(name, default=None) - - Return optional transport information: see - :meth:`BaseTransport.get_extra_info`. - - .. method:: write(data) - - Write some *data* bytes to the transport: see - :meth:`WriteTransport.write`. - - .. method:: writelines(data) - - Write a list (or any iterable) of data bytes to the transport: - see :meth:`WriteTransport.writelines`. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data: - see :meth:`WriteTransport.write_eof`. - - -StreamReaderProtocol -==================== - -.. class:: StreamReaderProtocol(stream_reader, client_connected_cb=None, loop=None) - - Trivial helper class to adapt between :class:`Protocol` and - :class:`StreamReader`. Subclass of :class:`Protocol`. - - *stream_reader* is a :class:`StreamReader` instance, *client_connected_cb* - is an optional function called with (stream_reader, stream_writer) when a - connection is made, *loop* is the event loop instance to use. - - (This is a helper class instead of making :class:`StreamReader` itself a - :class:`Protocol` subclass, because the :class:`StreamReader` has other - potential uses, and to prevent the user of the :class:`StreamReader` from - accidentally calling inappropriate methods of the protocol.) - - -IncompleteReadError -=================== - -.. exception:: IncompleteReadError - - Incomplete read error, subclass of :exc:`EOFError`. - - .. attribute:: expected - - Total number of expected bytes (:class:`int`). - - .. attribute:: partial - - Read bytes string before the end of stream was reached (:class:`bytes`). - - -LimitOverrunError -================= - -.. exception:: LimitOverrunError - - Reached the buffer limit while looking for a separator. - - .. attribute:: consumed - - Total number of to be consumed bytes. - - -Stream examples -=============== - -.. _asyncio-tcp-echo-client-streams: - -TCP echo client using streams ------------------------------ - -TCP echo client using the :func:`asyncio.open_connection` function:: - - import asyncio - - @asyncio.coroutine - def tcp_echo_client(message, loop): - reader, writer = yield from asyncio.open_connection('127.0.0.1', 8888, - loop=loop) - - print('Send: %r' % message) - writer.write(message.encode()) - - data = yield from reader.read(100) - print('Received: %r' % data.decode()) - - print('Close the socket') - writer.close() - - message = 'Hello World!' - loop = asyncio.get_event_loop() - loop.run_until_complete(tcp_echo_client(message, loop)) - loop.close() - -.. seealso:: - - The :ref:`TCP echo client protocol ` - example uses the :meth:`AbstractEventLoop.create_connection` method. - - -.. _asyncio-tcp-echo-server-streams: - -TCP echo server using streams ------------------------------ - -TCP echo server using the :func:`asyncio.start_server` function:: - - import asyncio - - @asyncio.coroutine - def handle_echo(reader, writer): - data = yield from reader.read(100) - message = data.decode() - addr = writer.get_extra_info('peername') - print("Received %r from %r" % (message, addr)) - - print("Send: %r" % message) - writer.write(data) - yield from writer.drain() - - print("Close the client socket") - writer.close() - - loop = asyncio.get_event_loop() - coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - -.. seealso:: - - The :ref:`TCP echo server protocol ` - example uses the :meth:`AbstractEventLoop.create_server` method. - - -Get HTTP headers ----------------- - -Simple example querying HTTP headers of the URL passed on the command line:: - - import asyncio - import urllib.parse - import sys - - @asyncio.coroutine - def print_http_headers(url): - url = urllib.parse.urlsplit(url) - if url.scheme == 'https': - connect = asyncio.open_connection(url.hostname, 443, ssl=True) - else: - connect = asyncio.open_connection(url.hostname, 80) - reader, writer = yield from connect - query = ('HEAD {path} HTTP/1.0\r\n' - 'Host: {hostname}\r\n' - '\r\n').format(path=url.path or '/', hostname=url.hostname) - writer.write(query.encode('latin-1')) - while True: - line = yield from reader.readline() - if not line: - break - line = line.decode('latin1').rstrip() - if line: - print('HTTP header> %s' % line) - - # Ignore the body, close the socket - writer.close() - - url = sys.argv[1] - loop = asyncio.get_event_loop() - task = asyncio.ensure_future(print_http_headers(url)) - loop.run_until_complete(task) - loop.close() - -Usage:: - - python example.py http://example.com/path/page.html - -or with HTTPS:: - - python example.py https://example.com/path/page.html - -.. _asyncio-register-socket-streams: - -Register an open socket to wait for data using streams ------------------------------------------------------- - -Coroutine waiting until a socket receives data using the -:func:`open_connection` function:: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - @asyncio.coroutine - def wait_for_data(loop): - # Create a pair of connected sockets - rsock, wsock = socketpair() - - # Register the open socket to wait for data - reader, writer = yield from asyncio.open_connection(sock=rsock, loop=loop) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Wait for data - data = yield from reader.read(100) - - # Got data, we are done: close the socket - print("Received:", data.decode()) - writer.close() - - # Close the second socket - wsock.close() - - loop = asyncio.get_event_loop() - loop.run_until_complete(wait_for_data(loop)) - loop.close() - -.. seealso:: - - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`AbstractEventLoop.create_connection` method. - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`AbstractEventLoop.add_reader` method to register the file descriptor of a - socket. - diff --git a/third_party/python/Doc/library/asyncio-subprocess.rst b/third_party/python/Doc/library/asyncio-subprocess.rst deleted file mode 100644 index 1c1d0be91..000000000 --- a/third_party/python/Doc/library/asyncio-subprocess.rst +++ /dev/null @@ -1,421 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-subprocess: - -Subprocess -========== - -**Source code:** :source:`Lib/asyncio/subprocess.py` - -Windows event loop ------------------- - -On Windows, the default event loop is :class:`SelectorEventLoop` which does not -support subprocesses. :class:`ProactorEventLoop` should be used instead. -Example to use it on Windows:: - - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. seealso:: - - :ref:`Available event loops ` and :ref:`Platform - support `. - - -Create a subprocess: high-level API using Process -------------------------------------------------- - -.. coroutinefunction:: create_subprocess_exec(\*args, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Create a subprocess. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`AbstractEventLoop.subprocess_exec` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Run the shell command *cmd*. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`AbstractEventLoop.subprocess_shell` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. - - This function is a :ref:`coroutine `. - -Use the :meth:`AbstractEventLoop.connect_read_pipe` and -:meth:`AbstractEventLoop.connect_write_pipe` methods to connect pipes. - - -Create a subprocess: low-level API using subprocess.Popen ---------------------------------------------------------- - -Run subprocesses asynchronously using the :mod:`subprocess` module. - -.. coroutinemethod:: AbstractEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) - - Create a subprocess from one or more string arguments (character strings or - bytes strings encoded to the :ref:`filesystem encoding - `), where the first string - specifies the program to execute, and the remaining strings specify the - program's arguments. (Thus, together the string arguments form the - ``sys.argv`` value of the program, assuming it is a Python script.) This is - similar to the standard library :class:`subprocess.Popen` class called with - shell=False and the list of strings passed as the first argument; - however, where :class:`~subprocess.Popen` takes a single argument which is - list of strings, :func:`subprocess_exec` takes multiple string arguments. - - The *protocol_factory* must instantiate a subclass of the - :class:`asyncio.SubprocessProtocol` class. - - Other parameters: - - * *stdin*: Either a file-like object representing the pipe to be connected - to the subprocess's standard input stream using - :meth:`~AbstractEventLoop.connect_write_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. - - * *stdout*: Either a file-like object representing the pipe to be connected - to the subprocess's standard output stream using - :meth:`~AbstractEventLoop.connect_read_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. - - * *stderr*: Either a file-like object representing the pipe to be connected - to the subprocess's standard error stream using - :meth:`~AbstractEventLoop.connect_read_pipe`, or one of the constants - :const:`subprocess.PIPE` (the default) or :const:`subprocess.STDOUT`. - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess's standard error - stream will be connected to the same pipe as the standard output stream. - - * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* and - *shell*, which should not be specified at all. - - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. - - This method is a :ref:`coroutine `. - - See the constructor of the :class:`subprocess.Popen` class for parameters. - -.. coroutinemethod:: AbstractEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) - - Create a subprocess from *cmd*, which is a character string or a bytes - string encoded to the :ref:`filesystem encoding `, - using the platform's "shell" syntax. This is similar to the standard library - :class:`subprocess.Popen` class called with ``shell=True``. - - The *protocol_factory* must instantiate a subclass of the - :class:`asyncio.SubprocessProtocol` class. - - See :meth:`~AbstractEventLoop.subprocess_exec` for more details about - the remaining arguments. - - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. - - It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. - - This method is a :ref:`coroutine `. - -.. seealso:: - - The :meth:`AbstractEventLoop.connect_read_pipe` and - :meth:`AbstractEventLoop.connect_write_pipe` methods. - - -Constants ---------- - -.. data:: asyncio.subprocess.PIPE - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that a pipe to the standard stream should be opened. - -.. data:: asyncio.subprocess.STDOUT - - Special value that can be used as the *stderr* argument to - :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that standard error should go into the same handle as standard - output. - -.. data:: asyncio.subprocess.DEVNULL - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that the special file :data:`os.devnull` will be used. - - -Process -------- - -.. class:: asyncio.subprocess.Process - - A subprocess created by the :func:`create_subprocess_exec` or the - :func:`create_subprocess_shell` function. - - The API of the :class:`~asyncio.subprocess.Process` class was designed to be - close to the API of the :class:`subprocess.Popen` class, but there are some - differences: - - * There is no explicit :meth:`~subprocess.Popen.poll` method - * The :meth:`~subprocess.Popen.communicate` and - :meth:`~subprocess.Popen.wait` methods don't take a *timeout* parameter: - use the :func:`wait_for` function - * The *universal_newlines* parameter is not supported (only bytes strings - are supported) - * The :meth:`~asyncio.subprocess.Process.wait` method of - the :class:`~asyncio.subprocess.Process` class is asynchronous whereas the - :meth:`~subprocess.Popen.wait` method of the :class:`~subprocess.Popen` - class is implemented as a busy loop. - - This class is :ref:`not thread safe `. See also the - :ref:`Subprocess and threads ` section. - - .. coroutinemethod:: wait() - - Wait for child process to terminate. Set and return :attr:`returncode` - attribute. - - This method is a :ref:`coroutine `. - - .. note:: - - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` and - the child process generates enough output to a pipe such that it - blocks waiting for the OS pipe buffer to accept more data. Use the - :meth:`communicate` method when using pipes to avoid that. - - .. coroutinemethod:: communicate(input=None) - - Interact with process: Send data to stdin. Read data from stdout and - stderr, until end-of-file is reached. Wait for process to terminate. - The optional *input* argument should be data to be sent to the child - process, or ``None``, if no data should be sent to the child. The type - of *input* must be bytes. - - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - - If a :exc:`BrokenPipeError` or :exc:`ConnectionResetError` exception is - raised when writing *input* into stdin, the exception is ignored. It - occurs when the process exits before all data are written into stdin. - - Note that if you want to send data to the process's stdin, you need to - create the Process object with ``stdin=PIPE``. Similarly, to get anything - other than ``None`` in the result tuple, you need to give ``stdout=PIPE`` - and/or ``stderr=PIPE`` too. - - This method is a :ref:`coroutine `. - - .. note:: - - The data read is buffered in memory, so do not use this method if the - data size is large or unlimited. - - .. versionchanged:: 3.4.2 - The method now ignores :exc:`BrokenPipeError` and - :exc:`ConnectionResetError`. - - .. method:: send_signal(signal) - - Sends the signal *signal* to the child process. - - .. note:: - - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. - ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes - started with a *creationflags* parameter which includes - ``CREATE_NEW_PROCESS_GROUP``. - - .. method:: terminate() - - Stop the child. On Posix OSs the method sends :py:data:`signal.SIGTERM` - to the child. On Windows the Win32 API function - :c:func:`TerminateProcess` is called to stop the child. - - .. method:: kill() - - Kills the child. On Posix OSs the function sends :py:data:`SIGKILL` to - the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. - - .. attribute:: stdin - - Standard input stream (:class:`StreamWriter`), ``None`` if the process - was created with ``stdin=None``. - - .. attribute:: stdout - - Standard output stream (:class:`StreamReader`), ``None`` if the process - was created with ``stdout=None``. - - .. attribute:: stderr - - Standard error stream (:class:`StreamReader`), ``None`` if the process - was created with ``stderr=None``. - - .. warning:: - - Use the :meth:`communicate` method rather than :attr:`.stdin.write - `, :attr:`.stdout.read ` or :attr:`.stderr.read ` - to avoid deadlocks due to streams pausing reading or writing and blocking - the child process. - - .. attribute:: pid - - The identifier of the process. - - Note that for processes created by the :func:`create_subprocess_shell` - function, this attribute is the process identifier of the spawned shell. - - .. attribute:: returncode - - Return code of the process when it exited. A ``None`` value indicates - that the process has not terminated yet. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (Unix only). - - -.. _asyncio-subprocess-threads: - -Subprocess and threads ----------------------- - -asyncio supports running subprocesses from different threads, but there -are limits: - -* An event loop must run in the main thread -* The child watcher must be instantiated in the main thread, before executing - subprocesses from other threads. Call the :func:`get_child_watcher` - function in the main thread to instantiate the child watcher. - -The :class:`asyncio.subprocess.Process` class is not thread safe. - -.. seealso:: - - The :ref:`Concurrency and multithreading in asyncio - ` section. - - -Subprocess examples -------------------- - -Subprocess using transport and protocol -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of a subprocess protocol using to get the output of a subprocess and to -wait for the subprocess exit. The subprocess is created by the -:meth:`AbstractEventLoop.subprocess_exec` method:: - - import asyncio - import sys - - class DateProtocol(asyncio.SubprocessProtocol): - def __init__(self, exit_future): - self.exit_future = exit_future - self.output = bytearray() - - def pipe_data_received(self, fd, data): - self.output.extend(data) - - def process_exited(self): - self.exit_future.set_result(True) - - @asyncio.coroutine - def get_date(loop): - code = 'import datetime; print(datetime.datetime.now())' - exit_future = asyncio.Future(loop=loop) - - # Create the subprocess controlled by the protocol DateProtocol, - # redirect the standard output into a pipe - create = loop.subprocess_exec(lambda: DateProtocol(exit_future), - sys.executable, '-c', code, - stdin=None, stderr=None) - transport, protocol = yield from create - - # Wait for the subprocess exit using the process_exited() method - # of the protocol - yield from exit_future - - # Close the stdout pipe - transport.close() - - # Read the output which was collected by the pipe_data_received() - # method of the protocol - data = bytes(protocol.output) - return data.decode('ascii').rstrip() - - if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date(loop)) - print("Current date: %s" % date) - loop.close() - - -Subprocess using streams -^^^^^^^^^^^^^^^^^^^^^^^^ - -Example using the :class:`~asyncio.subprocess.Process` class to control the -subprocess and the :class:`StreamReader` class to read from the standard -output. The subprocess is created by the :func:`create_subprocess_exec` -function:: - - import asyncio.subprocess - import sys - - @asyncio.coroutine - def get_date(): - code = 'import datetime; print(datetime.datetime.now())' - - # Create the subprocess, redirect the standard output into a pipe - create = asyncio.create_subprocess_exec(sys.executable, '-c', code, - stdout=asyncio.subprocess.PIPE) - proc = yield from create - - # Read one line of output - data = yield from proc.stdout.readline() - line = data.decode('ascii').rstrip() - - # Wait for the subprocess exit - yield from proc.wait() - return line - - if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date()) - print("Current date: %s" % date) - loop.close() diff --git a/third_party/python/Doc/library/asyncio-sync.rst b/third_party/python/Doc/library/asyncio-sync.rst deleted file mode 100644 index 14e3defbf..000000000 --- a/third_party/python/Doc/library/asyncio-sync.rst +++ /dev/null @@ -1,295 +0,0 @@ -.. currentmodule:: asyncio -.. _asyncio-sync: - -Synchronization primitives -========================== - -**Source code:** :source:`Lib/asyncio/locks.py` - -Locks: - -* :class:`Lock` -* :class:`Event` -* :class:`Condition` - -Semaphores: - -* :class:`Semaphore` -* :class:`BoundedSemaphore` - -asyncio lock API was designed to be close to classes of the :mod:`threading` -module (:class:`~threading.Lock`, :class:`~threading.Event`, -:class:`~threading.Condition`, :class:`~threading.Semaphore`, -:class:`~threading.BoundedSemaphore`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. - -Locks ------ - -Lock -^^^^ - -.. class:: Lock(\*, loop=None) - - Primitive lock objects. - - A primitive lock is a synchronization primitive that is not owned by a - particular coroutine when locked. A primitive lock is in one of two states, - 'locked' or 'unlocked'. - - It is created in the unlocked state. It has two basic methods, :meth:`acquire` - and :meth:`release`. When the state is unlocked, acquire() changes the state to - locked and returns immediately. When the state is locked, acquire() blocks - until a call to release() in another coroutine changes it to unlocked, then - the acquire() call resets it to locked and returns. The release() method - should only be called in the locked state; it changes the state to unlocked - and returns immediately. If an attempt is made to release an unlocked lock, - a :exc:`RuntimeError` will be raised. - - When more than one coroutine is blocked in acquire() waiting for the state - to turn to unlocked, only one coroutine proceeds when a release() call - resets the state to unlocked; first coroutine which is blocked in acquire() - is being processed. - - :meth:`acquire` is a coroutine and should be called with ``yield from``. - - Locks also support the context management protocol. ``(yield from lock)`` - should be used as the context manager expression. - - This class is :ref:`not thread safe `. - - Usage:: - - lock = Lock() - ... - yield from lock - try: - ... - finally: - lock.release() - - Context manager usage:: - - lock = Lock() - ... - with (yield from lock): - ... - - Lock objects can be tested for locking state:: - - if not lock.locked(): - yield from lock - else: - # lock is acquired - ... - - .. method:: locked() - - Return ``True`` if the lock is acquired. - - .. coroutinemethod:: acquire() - - Acquire a lock. - - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. - - This method is a :ref:`coroutine `. - - .. method:: release() - - Release a lock. - - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - -Event -^^^^^ - -.. class:: Event(\*, loop=None) - - An Event implementation, asynchronous equivalent to :class:`threading.Event`. - - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`set` method and reset to false with the :meth:`clear` - method. The :meth:`wait` method blocks until the flag is true. The flag is - initially false. - - This class is :ref:`not thread safe `. - - .. method:: clear() - - Reset the internal flag to false. Subsequently, coroutines calling - :meth:`wait` will block until :meth:`set` is called to set the internal - flag to true again. - - .. method:: is_set() - - Return ``True`` if and only if the internal flag is true. - - .. method:: set() - - Set the internal flag to true. All coroutines waiting for it to become - true are awakened. Coroutine that call :meth:`wait` once the flag is true - will not block at all. - - .. coroutinemethod:: wait() - - Block until the internal flag is true. - - If the internal flag is true on entry, return ``True`` immediately. - Otherwise, block until another coroutine calls :meth:`set` to set the - flag to true, then return ``True``. - - This method is a :ref:`coroutine `. - - -Condition -^^^^^^^^^ - -.. class:: Condition(lock=None, \*, loop=None) - - A Condition implementation, asynchronous equivalent to - :class:`threading.Condition`. - - This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. - - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` - object, and it is used as the underlying lock. Otherwise, - a new :class:`Lock` object is created and used as the underlying lock. - - This class is :ref:`not thread safe `. - - .. coroutinemethod:: acquire() - - Acquire the underlying lock. - - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. - - This method is a :ref:`coroutine `. - - .. method:: notify(n=1) - - By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method wakes up at most *n* of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. - - .. note:: - - An awakened coroutine does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. - - .. method:: locked() - - Return ``True`` if the underlying lock is acquired. - - .. method:: notify_all() - - Wake up all coroutines waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting coroutines instead of one. If the - calling coroutine has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - .. method:: release() - - Release the underlying lock. - - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - .. coroutinemethod:: wait() - - Wait until notified. - - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another coroutine. Once awakened, it re-acquires - the lock and returns ``True``. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: wait_for(predicate) - - Wait until a predicate becomes true. - - The predicate should be a callable which result will be interpreted as a - boolean value. The final predicate value is the return value. - - This method is a :ref:`coroutine `. - - -Semaphores ----------- - -Semaphore -^^^^^^^^^ - -.. class:: Semaphore(value=1, \*, loop=None) - - A Semaphore implementation. - - A semaphore manages an internal counter which is decremented by each - :meth:`acquire` call and incremented by each :meth:`release` call. The - counter can never go below zero; when :meth:`acquire` finds that it is zero, - it blocks, waiting until some other coroutine calls :meth:`release`. - - Semaphores also support the context management protocol. - - The optional argument gives the initial value for the internal counter; it - defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` - is raised. - - This class is :ref:`not thread safe `. - - .. coroutinemethod:: acquire() - - Acquire a semaphore. - - If the internal counter is larger than zero on entry, decrement it by one - and return ``True`` immediately. If it is zero on entry, block, waiting - until some other coroutine has called :meth:`release` to make it larger - than ``0``, and then return ``True``. - - This method is a :ref:`coroutine `. - - .. method:: locked() - - Returns ``True`` if semaphore can not be acquired immediately. - - .. method:: release() - - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another coroutine is waiting for it to become - larger than zero again, wake up that coroutine. - - -BoundedSemaphore -^^^^^^^^^^^^^^^^ - -.. class:: BoundedSemaphore(value=1, \*, loop=None) - - A bounded semaphore implementation. Inherit from :class:`Semaphore`. - - This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would - increase the value above the initial value. diff --git a/third_party/python/Doc/library/asyncio-task.rst b/third_party/python/Doc/library/asyncio-task.rst deleted file mode 100644 index c0383331c..000000000 --- a/third_party/python/Doc/library/asyncio-task.rst +++ /dev/null @@ -1,730 +0,0 @@ -.. currentmodule:: asyncio - -Tasks and coroutines -==================== - -**Source code:** :source:`Lib/asyncio/tasks.py` - -**Source code:** :source:`Lib/asyncio/coroutines.py` - -.. _coroutine: - -Coroutines ----------- - -Coroutines used with :mod:`asyncio` may be implemented using the -:keyword:`async def` statement, or by using :term:`generators `. -The :keyword:`async def` type of coroutine was added in Python 3.5, and -is recommended if there is no need to support older Python versions. - -Generator-based coroutines should be decorated with :func:`@asyncio.coroutine -`, although this is not strictly enforced. -The decorator enables compatibility with :keyword:`async def` coroutines, -and also serves as documentation. Generator-based -coroutines use the ``yield from`` syntax introduced in :pep:`380`, -instead of the original ``yield`` syntax. - -The word "coroutine", like the word "generator", is used for two -different (though related) concepts: - -- The function that defines a coroutine - (a function definition using :keyword:`async def` or - decorated with ``@asyncio.coroutine``). If disambiguation is needed - we will call this a *coroutine function* (:func:`iscoroutinefunction` - returns ``True``). - -- The object obtained by calling a coroutine function. This object - represents a computation or an I/O operation (usually a combination) - that will complete eventually. If disambiguation is needed we will - call it a *coroutine object* (:func:`iscoroutine` returns ``True``). - -Things a coroutine can do: - -- ``result = await future`` or ``result = yield from future`` -- - suspends the coroutine until the - future is done, then returns the future's result, or raises an - exception, which will be propagated. (If the future is cancelled, - it will raise a ``CancelledError`` exception.) Note that tasks are - futures, and everything said about futures also applies to tasks. - -- ``result = await coroutine`` or ``result = yield from coroutine`` -- - wait for another coroutine to - produce a result (or raise an exception, which will be propagated). - The ``coroutine`` expression must be a *call* to another coroutine. - -- ``return expression`` -- produce a result to the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -- ``raise exception`` -- raise an exception in the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -Calling a coroutine does not start its code running -- -the coroutine object returned by the call doesn't do anything until you -schedule its execution. There are two basic ways to start it running: -call ``await coroutine`` or ``yield from coroutine`` from another coroutine -(assuming the other coroutine is already running!), or schedule its execution -using the :func:`ensure_future` function or the :meth:`AbstractEventLoop.create_task` -method. - - -Coroutines (and tasks) can only run when the event loop is running. - -.. decorator:: coroutine - - Decorator to mark generator-based coroutines. This enables - the generator use :keyword:`!yield from` to call :keyword:`async - def` coroutines, and also enables the generator to be called by - :keyword:`async def` coroutines, for instance using an - :keyword:`await` expression. - - There is no need to decorate :keyword:`async def` coroutines themselves. - - If the generator is not yielded from before it is destroyed, an error - message is logged. See :ref:`Detect coroutines never scheduled - `. - -.. note:: - - In this documentation, some methods are documented as coroutines, - even if they are plain Python functions returning a :class:`Future`. - This is intentional to have a freedom of tweaking the implementation - of these functions in the future. If such a function is needed to be - used in a callback-style code, wrap its result with :func:`ensure_future`. - - -.. _asyncio-hello-world-coroutine: - -Example: Hello World coroutine -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of coroutine displaying ``"Hello World"``:: - - import asyncio - - async def hello_world(): - print("Hello World!") - - loop = asyncio.get_event_loop() - # Blocking call which returns when the hello_world() coroutine is done - loop.run_until_complete(hello_world()) - loop.close() - -.. seealso:: - - The :ref:`Hello World with call_soon() ` - example uses the :meth:`AbstractEventLoop.call_soon` method to schedule a - callback. - - -.. _asyncio-date-coroutine: - -Example: Coroutine displaying the current date -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of coroutine displaying the current date every second during 5 seconds -using the :meth:`sleep` function:: - - import asyncio - import datetime - - async def display_date(loop): - end_time = loop.time() + 5.0 - while True: - print(datetime.datetime.now()) - if (loop.time() + 1.0) >= end_time: - break - await asyncio.sleep(1) - - loop = asyncio.get_event_loop() - # Blocking call which returns when the display_date() coroutine is done - loop.run_until_complete(display_date(loop)) - loop.close() - -.. seealso:: - - The :ref:`display the current date with call_later() - ` example uses a callback with the - :meth:`AbstractEventLoop.call_later` method. - - -Example: Chain coroutines -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example chaining coroutines:: - - import asyncio - - async def compute(x, y): - print("Compute %s + %s ..." % (x, y)) - await asyncio.sleep(1.0) - return x + y - - async def print_sum(x, y): - result = await compute(x, y) - print("%s + %s = %s" % (x, y, result)) - - loop = asyncio.get_event_loop() - loop.run_until_complete(print_sum(1, 2)) - loop.close() - -``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits -until ``compute()`` is completed before returning its result. - -Sequence diagram of the example: - -.. image:: tulip_coro.png - :align: center - -The "Task" is created by the :meth:`AbstractEventLoop.run_until_complete` method -when it gets a coroutine object instead of a task. - -The diagram shows the control flow, it does not describe exactly how things -work internally. For example, the sleep coroutine creates an internal future -which uses :meth:`AbstractEventLoop.call_later` to wake up the task in 1 second. - - -InvalidStateError ------------------ - -.. exception:: InvalidStateError - - The operation is not allowed in this state. - - -TimeoutError ------------- - -.. exception:: TimeoutError - - The operation exceeded the given deadline. - -.. note:: - - This exception is different from the builtin :exc:`TimeoutError` exception! - - -Future ------- - -.. class:: Future(\*, loop=None) - - This class is *almost* compatible with :class:`concurrent.futures.Future`. - - Differences: - - - :meth:`result` and :meth:`exception` do not take a timeout argument and - raise an exception when the future isn't done yet. - - - Callbacks registered with :meth:`add_done_callback` are always called - via the event loop's :meth:`~AbstractEventLoop.call_soon`. - - - This class is not compatible with the :func:`~concurrent.futures.wait` and - :func:`~concurrent.futures.as_completed` functions in the - :mod:`concurrent.futures` package. - - This class is :ref:`not thread safe `. - - .. method:: cancel() - - Cancel the future and schedule callbacks. - - If the future is already done or cancelled, return ``False``. Otherwise, - change the future's state to cancelled, schedule the callbacks and return - ``True``. - - .. method:: cancelled() - - Return ``True`` if the future was cancelled. - - .. method:: done() - - Return ``True`` if the future is done. - - Done means either that a result / exception are available, or that the - future was cancelled. - - .. method:: result() - - Return the result this future represents. - - If the future has been cancelled, raises :exc:`CancelledError`. If the - future's result isn't yet available, raises :exc:`InvalidStateError`. If - the future is done and has an exception set, this exception is raised. - - .. method:: exception() - - Return the exception that was set on this future. - - The exception (or ``None`` if no exception was set) is returned only if - the future is done. If the future has been cancelled, raises - :exc:`CancelledError`. If the future isn't done yet, raises - :exc:`InvalidStateError`. - - .. method:: add_done_callback(fn) - - Add a callback to be run when the future becomes done. - - The callback is called with a single argument - the future object. If the - future is already done when this is called, the callback is scheduled - with :meth:`~AbstractEventLoop.call_soon`. - - :ref:`Use functools.partial to pass parameters to the callback - `. For example, - ``fut.add_done_callback(functools.partial(print, "Future:", - flush=True))`` will call ``print("Future:", fut, flush=True)``. - - .. method:: remove_done_callback(fn) - - Remove all instances of a callback from the "call when done" list. - - Returns the number of callbacks removed. - - .. method:: set_result(result) - - Mark the future done and set its result. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - .. method:: set_exception(exception) - - Mark the future done and set an exception. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - -Example: Future with run_until_complete() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example combining a :class:`Future` and a :ref:`coroutine function -`:: - - import asyncio - - async def slow_operation(future): - await asyncio.sleep(1) - future.set_result('Future is done!') - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - loop.run_until_complete(future) - print(future.result()) - loop.close() - -The coroutine function is responsible for the computation (which takes 1 second) -and it stores the result into the future. The -:meth:`~AbstractEventLoop.run_until_complete` method waits for the completion of -the future. - -.. note:: - The :meth:`~AbstractEventLoop.run_until_complete` method uses internally the - :meth:`~Future.add_done_callback` method to be notified when the future is - done. - - -Example: Future with run_forever() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The previous example can be written differently using the -:meth:`Future.add_done_callback` method to describe explicitly the control -flow:: - - import asyncio - - async def slow_operation(future): - await asyncio.sleep(1) - future.set_result('Future is done!') - - def got_result(future): - print(future.result()) - loop.stop() - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - future.add_done_callback(got_result) - try: - loop.run_forever() - finally: - loop.close() - -In this example, the future is used to link ``slow_operation()`` to -``got_result()``: when ``slow_operation()`` is done, ``got_result()`` is called -with the result. - - -Task ----- - -.. class:: Task(coro, \*, loop=None) - - Schedule the execution of a :ref:`coroutine `: wrap it in a - future. A task is a subclass of :class:`Future`. - - A task is responsible for executing a coroutine object in an event loop. If - the wrapped coroutine yields from a future, the task suspends the execution - of the wrapped coroutine and waits for the completion of the future. When - the future is done, the execution of the wrapped coroutine restarts with the - result or the exception of the future. - - Event loops use cooperative scheduling: an event loop only runs one task at - a time. Other tasks may run in parallel if other event loops are - running in different threads. While a task waits for the completion of a - future, the event loop executes a new task. - - The cancellation of a task is different from the cancelation of a - future. Calling :meth:`cancel` will throw a - :exc:`~concurrent.futures.CancelledError` to the wrapped - coroutine. :meth:`~Future.cancelled` only returns ``True`` if the - wrapped coroutine did not catch the - :exc:`~concurrent.futures.CancelledError` exception, or raised a - :exc:`~concurrent.futures.CancelledError` exception. - - If a pending task is destroyed, the execution of its wrapped :ref:`coroutine - ` did not complete. It is probably a bug and a warning is - logged: see :ref:`Pending task destroyed `. - - Don't directly create :class:`Task` instances: use the :func:`ensure_future` - function or the :meth:`AbstractEventLoop.create_task` method. - - This class is :ref:`not thread safe `. - - .. classmethod:: all_tasks(loop=None) - - Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - - .. classmethod:: current_task(loop=None) - - Return the currently running task in an event loop or ``None``. - - By default the current task for the current event loop is returned. - - ``None`` is returned when called not in the context of a :class:`Task`. - - .. method:: cancel() - - Request that this task cancel itself. - - This arranges for a :exc:`~concurrent.futures.CancelledError` to be - thrown into the wrapped coroutine on the next cycle through the event - loop. The coroutine then has a chance to clean up or even deny the - request using try/except/finally. - - Unlike :meth:`Future.cancel`, this does not guarantee that the task - will be cancelled: the exception might be caught and acted upon, delaying - cancellation of the task or preventing cancellation completely. The task - may also return a value or raise a different exception. - - Immediately after this method is called, :meth:`~Future.cancelled` will - not return ``True`` (unless the task was already cancelled). A task will - be marked as cancelled when the wrapped coroutine terminates with a - :exc:`~concurrent.futures.CancelledError` exception (even if - :meth:`cancel` was not called). - - .. method:: get_stack(\*, limit=None) - - Return the list of stack frames for this task's coroutine. - - If the coroutine is not done, this returns the stack where it is - suspended. If the coroutine has completed successfully or was - cancelled, this returns an empty list. If the coroutine was - terminated by an exception, this returns the list of traceback - frames. - - The frames are always ordered from oldest to newest. - - The optional limit gives the maximum number of frames to return; by - default all available frames are returned. Its meaning differs depending - on whether a stack or a traceback is returned: the newest frames of a - stack are returned, but the oldest frames of a traceback are returned. - (This matches the behavior of the traceback module.) - - For reasons beyond our control, only one stack frame is returned for a - suspended coroutine. - - .. method:: print_stack(\*, limit=None, file=None) - - Print the stack or traceback for this task's coroutine. - - This produces output similar to that of the traceback module, for the - frames retrieved by get_stack(). The limit argument is passed to - get_stack(). The file argument is an I/O stream to which the output - is written; by default output is written to sys.stderr. - - -Example: Parallel execution of tasks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example executing 3 tasks (A, B, C) in parallel:: - - import asyncio - - async def factorial(name, number): - f = 1 - for i in range(2, number+1): - print("Task %s: Compute factorial(%s)..." % (name, i)) - await asyncio.sleep(1) - f *= i - print("Task %s: factorial(%s) = %s" % (name, number, f)) - - loop = asyncio.get_event_loop() - loop.run_until_complete(asyncio.gather( - factorial("A", 2), - factorial("B", 3), - factorial("C", 4), - )) - loop.close() - -Output:: - - Task A: Compute factorial(2)... - Task B: Compute factorial(2)... - Task C: Compute factorial(2)... - Task A: factorial(2) = 2 - Task B: Compute factorial(3)... - Task C: Compute factorial(3)... - Task B: factorial(3) = 6 - Task C: Compute factorial(4)... - Task C: factorial(4) = 24 - -A task is automatically scheduled for execution when it is created. The event -loop stops when all tasks are done. - - -Task functions --------------- - -.. note:: - - In the functions below, the optional *loop* argument allows explicitly setting - the event loop object used by the underlying task or coroutine. If it's - not provided, the default event loop is used. - -.. function:: as_completed(fs, \*, loop=None, timeout=None) - - Return an iterator whose values, when waited for, are :class:`Future` - instances. - - Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures - are done. - - Example:: - - for f in as_completed(fs): - result = yield from f # The 'yield from' may raise - # Use result - - .. note:: - - The futures ``f`` are not necessarily members of fs. - -.. function:: ensure_future(coro_or_future, \*, loop=None) - - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. - - If the argument is a :class:`Future`, it is returned directly. - - .. versionadded:: 3.4.4 - - .. versionchanged:: 3.5.1 - The function accepts any :term:`awaitable` object. - - .. seealso:: - - The :meth:`AbstractEventLoop.create_task` method. - -.. function:: async(coro_or_future, \*, loop=None) - - A deprecated alias to :func:`ensure_future`. - - .. deprecated:: 3.4.4 - -.. function:: wrap_future(future, \*, loop=None) - - Wrap a :class:`concurrent.futures.Future` object in a :class:`Future` - object. - -.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) - - Return a future aggregating results from the given coroutine objects or - futures. - - All futures must share the same event loop. If all the tasks are done - successfully, the returned future's result is the list of results (in the - order of the original sequence, not necessarily the order of results - arrival). If *return_exceptions* is true, exceptions in the tasks are - treated the same as successful results, and gathered in the result list; - otherwise, the first raised exception will be immediately propagated to the - returned future. - - Cancellation: if the outer Future is cancelled, all children (that have not - completed yet) are also cancelled. If any child is cancelled, this is - treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the - outer Future is *not* cancelled in this case. (This is to prevent the - cancellation of one child to cause other children to be cancelled.) - - .. versionchanged:: 3.6.6 - If the *gather* itself is cancelled, the cancellation is propagated - regardless of *return_exceptions*. - -.. function:: iscoroutine(obj) - - Return ``True`` if *obj* is a :ref:`coroutine object `, - which may be based on a generator or an :keyword:`async def` coroutine. - -.. function:: iscoroutinefunction(func) - - Return ``True`` if *func* is determined to be a :ref:`coroutine function - `, which may be a decorated generator function or an - :keyword:`async def` function. - -.. function:: run_coroutine_threadsafe(coro, loop) - - Submit a :ref:`coroutine object ` to a given event loop. - - Return a :class:`concurrent.futures.Future` to access the result. - - This function is meant to be called from a different thread than the one - where the event loop is running. Usage:: - - # Create a coroutine - coro = asyncio.sleep(1, result=3) - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 - - If an exception is raised in the coroutine, the returned future will be - notified. It can also be used to cancel the task in the event loop:: - - try: - result = future.result(timeout) - except asyncio.TimeoutError: - print('The coroutine took too long, cancelling the task...') - future.cancel() - except Exception as exc: - print('The coroutine raised an exception: {!r}'.format(exc)) - else: - print('The coroutine returned: {!r}'.format(result)) - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - .. note:: - - Unlike other functions from the module, - :func:`run_coroutine_threadsafe` requires the *loop* argument to - be passed explicitly. - - .. versionadded:: 3.5.1 - -.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) - - Create a :ref:`coroutine ` that completes after a given - time (in seconds). If *result* is provided, it is produced to the caller - when the coroutine completes. - - The resolution of the sleep depends on the :ref:`granularity of the event - loop `. - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: shield(arg, \*, loop=None) - - Wait for a future, shielding it from cancellation. - - The statement:: - - res = yield from shield(something()) - - is exactly equivalent to the statement:: - - res = yield from something() - - *except* that if the coroutine containing it is cancelled, the task running - in ``something()`` is not cancelled. From the point of view of - ``something()``, the cancellation did not happen. But its caller is still - cancelled, so the yield-from expression still raises - :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is - cancelled by other means this will still cancel ``shield()``. - - If you want to completely ignore cancellation (not recommended) you can - combine ``shield()`` with a try/except clause, as follows:: - - try: - res = yield from shield(something()) - except CancelledError: - res = None - - -.. coroutinefunction:: wait(futures, \*, loop=None, timeout=None,\ - return_when=ALL_COMPLETED) - - Wait for the Futures and coroutine objects given by the sequence *futures* - to complete. Coroutines will be wrapped in Tasks. Returns two sets of - :class:`Future`: (done, pending). - - The sequence *futures* must not be empty. - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - - *return_when* indicates when this function should return. It must be one of - the following constants of the :mod:`concurrent.futures` module: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - - This function is a :ref:`coroutine `. - - Usage:: - - done, pending = yield from asyncio.wait(fs) - - .. note:: - - This does not raise :exc:`asyncio.TimeoutError`! Futures that aren't done - when the timeout occurs are returned in the second set. - - -.. coroutinefunction:: wait_for(fut, timeout, \*, loop=None) - - Wait for the single :class:`Future` or :ref:`coroutine object ` - to complete with timeout. If *timeout* is ``None``, block until the future - completes. - - Coroutine will be wrapped in :class:`Task`. - - Returns result of the Future or coroutine. When a timeout occurs, it - cancels the task and raises :exc:`asyncio.TimeoutError`. To avoid the task - cancellation, wrap it in :func:`shield`. - - If the wait is cancelled, the future *fut* is also cancelled. - - This function is a :ref:`coroutine `, usage:: - - result = yield from asyncio.wait_for(fut, 60.0) - - .. versionchanged:: 3.4.3 - If the wait is cancelled, the future *fut* is now also cancelled. diff --git a/third_party/python/Doc/library/asyncio.rst b/third_party/python/Doc/library/asyncio.rst deleted file mode 100644 index b076b7d00..000000000 --- a/third_party/python/Doc/library/asyncio.rst +++ /dev/null @@ -1,66 +0,0 @@ -:mod:`asyncio` --- Asynchronous I/O, event loop, coroutines and tasks -===================================================================== - -.. module:: asyncio - :synopsis: Asynchronous I/O, event loop, coroutines and tasks. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/asyncio/` - --------------- - -This module provides infrastructure for writing single-threaded concurrent -code using coroutines, multiplexing I/O access over sockets and other -resources, running network clients and servers, and other related primitives. -Here is a more detailed list of the package contents: - -* a pluggable :ref:`event loop ` with various system-specific - implementations; - -* :ref:`transport ` and :ref:`protocol ` abstractions - (similar to those in `Twisted `_); - -* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and - others (some may be system-dependent); - -* a :class:`Future` class that mimics the one in the :mod:`concurrent.futures` - module, but adapted for use with the event loop; - -* coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write - concurrent code in a sequential fashion; - -* cancellation support for :class:`Future`\s and coroutines; - -* :ref:`synchronization primitives ` for use between coroutines in - a single thread, mimicking those in the :mod:`threading` module; - -* an interface for passing work off to a threadpool, for times when - you absolutely, positively have to use a library that makes blocking - I/O calls. - -Asynchronous programming is more complex than classical "sequential" -programming: see the :ref:`Develop with asyncio ` page which lists -common traps and explains how to avoid them. :ref:`Enable the debug mode -` during development to detect common issues. - -Table of contents: - -.. toctree:: - :maxdepth: 3 - - asyncio-eventloop.rst - asyncio-eventloops.rst - asyncio-task.rst - asyncio-protocol.rst - asyncio-stream.rst - asyncio-subprocess.rst - asyncio-sync.rst - asyncio-queue.rst - asyncio-dev.rst - -.. seealso:: - - The :mod:`asyncio` module was designed in :PEP:`3156`. For a - motivational primer on transports and protocols, see :PEP:`3153`. - diff --git a/third_party/python/Doc/library/asyncore.rst b/third_party/python/Doc/library/asyncore.rst deleted file mode 100644 index 11d36163e..000000000 --- a/third_party/python/Doc/library/asyncore.rst +++ /dev/null @@ -1,356 +0,0 @@ -:mod:`asyncore` --- Asynchronous socket handler -=============================================== - -.. module:: asyncore - :synopsis: A base class for developing asynchronous socket handling - services. - -.. moduleauthor:: Sam Rushing -.. sectionauthor:: Christopher Petrilli -.. sectionauthor:: Steve Holden -.. heavily adapted from original documentation by Sam Rushing - -**Source code:** :source:`Lib/asyncore.py` - -.. deprecated:: 3.6 - Please use :mod:`asyncio` instead. - --------------- - -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - -This module provides the basic infrastructure for writing asynchronous socket -service clients and servers. - -There are only two ways to have a program on a single processor do "more than -one thing at a time." Multi-threaded programming is the simplest and most -popular way to do it, but there is another very different technique, that lets -you have nearly all the advantages of multi-threading, without actually using -multiple threads. It's really only practical if your program is largely I/O -bound. If your program is processor bound, then pre-emptive scheduled threads -are probably what you really need. Network servers are rarely processor -bound, however. - -If your operating system supports the :c:func:`select` system call in its I/O -library (and nearly all do), then you can use it to juggle multiple -communication channels at once; doing other work while your I/O is taking -place in the "background." Although this strategy can seem strange and -complex, especially at first, it is in many ways easier to understand and -control than multi-threaded programming. The :mod:`asyncore` module solves -many of the difficult problems for you, making the task of building -sophisticated high-performance network servers and clients a snap. For -"conversational" applications and protocols the companion :mod:`asynchat` -module is invaluable. - -The basic idea behind both modules is to create one or more network -*channels*, instances of class :class:`asyncore.dispatcher` and -:class:`asynchat.async_chat`. Creating the channels adds them to a global -map, used by the :func:`loop` function if you do not provide it with your own -*map*. - -Once the initial channel(s) is(are) created, calling the :func:`loop` function -activates channel service, which continues until the last channel (including -any that have been added to the map during asynchronous service) is closed. - - -.. function:: loop([timeout[, use_poll[, map[,count]]]]) - - Enter a polling loop that terminates after count passes or all open - channels have been closed. All arguments are optional. The *count* - parameter defaults to ``None``, resulting in the loop terminating only when all - channels have been closed. The *timeout* argument sets the timeout - parameter for the appropriate :func:`~select.select` or :func:`~select.poll` - call, measured in seconds; the default is 30 seconds. The *use_poll* - parameter, if true, indicates that :func:`~select.poll` should be used in - preference to :func:`~select.select` (the default is ``False``). - - The *map* parameter is a dictionary whose items are the channels to watch. - As channels are closed they are deleted from their map. If *map* is - omitted, a global map is used. Channels (instances of - :class:`asyncore.dispatcher`, :class:`asynchat.async_chat` and subclasses - thereof) can freely be mixed in the map. - - -.. class:: dispatcher() - - The :class:`dispatcher` class is a thin wrapper around a low-level socket - object. To make it more useful, it has a few methods for event-handling - which are called from the asynchronous loop. Otherwise, it can be treated - as a normal non-blocking socket object. - - The firing of low-level events at certain times or in certain connection - states tells the asynchronous loop that certain higher-level events have - taken place. For example, if we have asked for a socket to connect to - another host, we know that the connection has been made when the socket - becomes writable for the first time (at this point you know that you may - write to it with the expectation of success). The implied higher-level - events are: - - +----------------------+----------------------------------------+ - | Event | Description | - +======================+========================================+ - | ``handle_connect()`` | Implied by the first read or write | - | | event | - +----------------------+----------------------------------------+ - | ``handle_close()`` | Implied by a read event with no data | - | | available | - +----------------------+----------------------------------------+ - | ``handle_accepted()``| Implied by a read event on a listening | - | | socket | - +----------------------+----------------------------------------+ - - During asynchronous processing, each mapped channel's :meth:`readable` and - :meth:`writable` methods are used to determine whether the channel's socket - should be added to the list of channels :c:func:`select`\ ed or - :c:func:`poll`\ ed for read and write events. - - Thus, the set of channel events is larger than the basic socket events. The - full set of methods that can be overridden in your subclass follows: - - - .. method:: handle_read() - - Called when the asynchronous loop detects that a :meth:`read` call on the - channel's socket will succeed. - - - .. method:: handle_write() - - Called when the asynchronous loop detects that a writable socket can be - written. Often this method will implement the necessary buffering for - performance. For example:: - - def handle_write(self): - sent = self.send(self.buffer) - self.buffer = self.buffer[sent:] - - - .. method:: handle_expt() - - Called when there is out of band (OOB) data for a socket connection. This - will almost never happen, as OOB is tenuously supported and rarely used. - - - .. method:: handle_connect() - - Called when the active opener's socket actually makes a connection. Might - send a "welcome" banner, or initiate a protocol negotiation with the - remote endpoint, for example. - - - .. method:: handle_close() - - Called when the socket is closed. - - - .. method:: handle_error() - - Called when an exception is raised and not otherwise handled. The default - version prints a condensed traceback. - - - .. method:: handle_accept() - - Called on listening channels (passive openers) when a connection can be - established with a new remote endpoint that has issued a :meth:`connect` - call for the local endpoint. Deprecated in version 3.2; use - :meth:`handle_accepted` instead. - - .. deprecated:: 3.2 - - - .. method:: handle_accepted(sock, addr) - - Called on listening channels (passive openers) when a connection has been - established with a new remote endpoint that has issued a :meth:`connect` - call for the local endpoint. *sock* is a *new* socket object usable to - send and receive data on the connection, and *addr* is the address - bound to the socket on the other end of the connection. - - .. versionadded:: 3.2 - - - .. method:: readable() - - Called each time around the asynchronous loop to determine whether a - channel's socket should be added to the list on which read events can - occur. The default method simply returns ``True``, indicating that by - default, all channels will be interested in read events. - - - .. method:: writable() - - Called each time around the asynchronous loop to determine whether a - channel's socket should be added to the list on which write events can - occur. The default method simply returns ``True``, indicating that by - default, all channels will be interested in write events. - - - In addition, each channel delegates or extends many of the socket methods. - Most of these are nearly identical to their socket partners. - - - .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM) - - This is identical to the creation of a normal socket, and will use the - same options for creation. Refer to the :mod:`socket` documentation for - information on creating sockets. - - .. versionchanged:: 3.3 - *family* and *type* arguments can be omitted. - - - .. method:: connect(address) - - As with the normal socket object, *address* is a tuple with the first - element the host to connect to, and the second the port number. - - - .. method:: send(data) - - Send *data* to the remote end-point of the socket. - - - .. method:: recv(buffer_size) - - Read at most *buffer_size* bytes from the socket's remote end-point. An - empty bytes object implies that the channel has been closed from the - other end. - - Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though - :func:`select.select` or :func:`select.poll` has reported the socket - ready for reading. - - - .. method:: listen(backlog) - - Listen for connections made to the socket. The *backlog* argument - specifies the maximum number of queued connections and should be at least - 1; the maximum value is system-dependent (usually 5). - - - .. method:: bind(address) - - Bind the socket to *address*. The socket must not already be bound. (The - format of *address* depends on the address family --- refer to the - :mod:`socket` documentation for more information.) To mark - the socket as re-usable (setting the :const:`SO_REUSEADDR` option), call - the :class:`dispatcher` object's :meth:`set_reuse_addr` method. - - - .. method:: accept() - - Accept a connection. The socket must be bound to an address and listening - for connections. The return value can be either ``None`` or a pair - ``(conn, address)`` where *conn* is a *new* socket object usable to send - and receive data on the connection, and *address* is the address bound to - the socket on the other end of the connection. - When ``None`` is returned it means the connection didn't take place, in - which case the server should just ignore this event and keep listening - for further incoming connections. - - - .. method:: close() - - Close the socket. All future operations on the socket object will fail. - The remote end-point will receive no more data (after queued data is - flushed). Sockets are automatically closed when they are - garbage-collected. - - -.. class:: dispatcher_with_send() - - A :class:`dispatcher` subclass which adds simple buffered output capability, - useful for simple clients. For more sophisticated usage use - :class:`asynchat.async_chat`. - -.. class:: file_dispatcher() - - A file_dispatcher takes a file descriptor or :term:`file object` along - with an optional map argument and wraps it for use with the :c:func:`poll` - or :c:func:`loop` functions. If provided a file object or anything with a - :c:func:`fileno` method, that method will be called and passed to the - :class:`file_wrapper` constructor. Availability: UNIX. - -.. class:: file_wrapper() - - A file_wrapper takes an integer file descriptor and calls :func:`os.dup` to - duplicate the handle so that the original handle may be closed independently - of the file_wrapper. This class implements sufficient methods to emulate a - socket for use by the :class:`file_dispatcher` class. Availability: UNIX. - - -.. _asyncore-example-1: - -asyncore Example basic HTTP client ----------------------------------- - -Here is a very basic HTTP client that uses the :class:`dispatcher` class to -implement its socket handling:: - - import asyncore - - class HTTPClient(asyncore.dispatcher): - - def __init__(self, host, path): - asyncore.dispatcher.__init__(self) - self.create_socket() - self.connect( (host, 80) ) - self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' % - (path, host), 'ascii') - - def handle_connect(self): - pass - - def handle_close(self): - self.close() - - def handle_read(self): - print(self.recv(8192)) - - def writable(self): - return (len(self.buffer) > 0) - - def handle_write(self): - sent = self.send(self.buffer) - self.buffer = self.buffer[sent:] - - - client = HTTPClient('www.python.org', '/') - asyncore.loop() - -.. _asyncore-example-2: - -asyncore Example basic echo server ----------------------------------- - -Here is a basic echo server that uses the :class:`dispatcher` class to accept -connections and dispatches the incoming connections to a handler:: - - import asyncore - - class EchoHandler(asyncore.dispatcher_with_send): - - def handle_read(self): - data = self.recv(8192) - if data: - self.send(data) - - class EchoServer(asyncore.dispatcher): - - def __init__(self, host, port): - asyncore.dispatcher.__init__(self) - self.create_socket() - self.set_reuse_addr() - self.bind((host, port)) - self.listen(5) - - def handle_accepted(self, sock, addr): - print('Incoming connection from %s' % repr(addr)) - handler = EchoHandler(sock) - - server = EchoServer('localhost', 8080) - asyncore.loop() diff --git a/third_party/python/Doc/library/atexit.rst b/third_party/python/Doc/library/atexit.rst deleted file mode 100644 index 5c60b604c..000000000 --- a/third_party/python/Doc/library/atexit.rst +++ /dev/null @@ -1,109 +0,0 @@ -:mod:`atexit` --- Exit handlers -=============================== - -.. module:: atexit - :synopsis: Register and execute cleanup functions. - -.. moduleauthor:: Skip Montanaro -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`atexit` module defines functions to register and unregister cleanup -functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* -order in which they were registered; if you register ``A``, ``B``, and ``C``, -at interpreter termination time they will be run in the order ``C``, ``B``, -``A``. - -**Note:** The functions registered via this module are not called when the -program is killed by a signal not handled by Python, when a Python fatal -internal error is detected, or when :func:`os._exit` is called. - - -.. function:: register(func, *args, **kwargs) - - Register *func* as a function to be executed at termination. Any optional - arguments that are to be passed to *func* must be passed as arguments to - :func:`register`. It is possible to register the same function and arguments - more than once. - - At normal program termination (for instance, if :func:`sys.exit` is called or - the main module's execution completes), all functions registered are called in - last in, first out order. The assumption is that lower level modules will - normally be imported before higher level modules and thus must be cleaned up - later. - - If an exception is raised during execution of the exit handlers, a traceback is - printed (unless :exc:`SystemExit` is raised) and the exception information is - saved. After all exit handlers have had a chance to run the last exception to - be raised is re-raised. - - This function returns *func*, which makes it possible to use it as a - decorator. - - -.. function:: unregister(func) - - Remove *func* from the list of functions to be run at interpreter - shutdown. After calling :func:`unregister`, *func* is guaranteed not to be - called when the interpreter shuts down, even if it was registered more than - once. :func:`unregister` silently does nothing if *func* was not previously - registered. - - -.. seealso:: - - Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history - files. - - -.. _atexit-example: - -:mod:`atexit` Example ---------------------- - -The following simple example demonstrates how a module can initialize a counter -from a file when it is imported and save the counter's updated value -automatically when the program terminates without relying on the application -making an explicit call into this module at termination. :: - - try: - with open("counterfile") as infile: - _count = int(infile.read()) - except FileNotFoundError: - _count = 0 - - def incrcounter(n): - global _count - _count = _count + n - - def savecounter(): - with open("counterfile", "w") as outfile: - outfile.write("%d" % _count) - - import atexit - atexit.register(savecounter) - -Positional and keyword arguments may also be passed to :func:`register` to be -passed along to the registered function when it is called:: - - def goodbye(name, adjective): - print('Goodbye, %s, it was %s to meet you.' % (name, adjective)) - - import atexit - atexit.register(goodbye, 'Donny', 'nice') - - # or: - atexit.register(goodbye, adjective='nice', name='Donny') - -Usage as a :term:`decorator`:: - - import atexit - - @atexit.register - def goodbye(): - print("You are now leaving the Python sector.") - -This only works with functions that can be called without arguments. diff --git a/third_party/python/Doc/library/audioop.rst b/third_party/python/Doc/library/audioop.rst deleted file mode 100644 index bad9da2ec..000000000 --- a/third_party/python/Doc/library/audioop.rst +++ /dev/null @@ -1,282 +0,0 @@ -:mod:`audioop` --- Manipulate raw audio data -============================================ - -.. module:: audioop - :synopsis: Manipulate raw audio data. - --------------- - -The :mod:`audioop` module contains some useful operations on sound fragments. -It operates on sound fragments consisting of signed integer samples 8, 16, 24 -or 32 bits wide, stored in :term:`bytes-like objects `. All scalar items are -integers, unless specified otherwise. - -.. versionchanged:: 3.4 - Support for 24-bit samples was added. - All functions now accept any :term:`bytes-like object`. - String input now results in an immediate error. - -.. index:: - single: Intel/DVI ADPCM - single: ADPCM, Intel/DVI - single: a-LAW - single: u-LAW - -This module provides support for a-LAW, u-LAW and Intel/DVI ADPCM encodings. - -.. This para is mostly here to provide an excuse for the index entries... - -A few of the more complicated operations only take 16-bit samples, otherwise the -sample size (in bytes) is always a parameter of the operation. - -The module defines the following variables and functions: - - -.. exception:: error - - This exception is raised on all errors, such as unknown number of bytes per - sample, etc. - - -.. function:: add(fragment1, fragment2, width) - - Return a fragment which is the addition of the two samples passed as parameters. - *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both - fragments should have the same length. Samples are truncated in case of overflow. - - -.. function:: adpcm2lin(adpcmfragment, width, state) - - Decode an Intel/DVI ADPCM coded fragment to a linear fragment. See the - description of :func:`lin2adpcm` for details on ADPCM coding. Return a tuple - ``(sample, newstate)`` where the sample has the width specified in *width*. - - -.. function:: alaw2lin(fragment, width) - - Convert sound fragments in a-LAW encoding to linearly encoded sound fragments. - a-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - - -.. function:: avg(fragment, width) - - Return the average over all samples in the fragment. - - -.. function:: avgpp(fragment, width) - - Return the average peak-peak value over all samples in the fragment. No - filtering is done, so the usefulness of this routine is questionable. - - -.. function:: bias(fragment, width, bias) - - Return a fragment that is the original fragment with a bias added to each - sample. Samples wrap around in case of overflow. - - -.. function:: byteswap(fragment, width) - - "Byteswap" all samples in a fragment and returns the modified fragment. - Converts big-endian samples to little-endian and vice versa. - - .. versionadded:: 3.4 - - -.. function:: cross(fragment, width) - - Return the number of zero crossings in the fragment passed as an argument. - - -.. function:: findfactor(fragment, reference) - - Return a factor *F* such that ``rms(add(fragment, mul(reference, -F)))`` is - minimal, i.e., return the factor with which you should multiply *reference* to - make it match as well as possible to *fragment*. The fragments should both - contain 2-byte samples. - - The time taken by this routine is proportional to ``len(fragment)``. - - -.. function:: findfit(fragment, reference) - - Try to match *reference* as well as possible to a portion of *fragment* (which - should be the longer fragment). This is (conceptually) done by taking slices - out of *fragment*, using :func:`findfactor` to compute the best match, and - minimizing the result. The fragments should both contain 2-byte samples. - Return a tuple ``(offset, factor)`` where *offset* is the (integer) offset into - *fragment* where the optimal match started and *factor* is the (floating-point) - factor as per :func:`findfactor`. - - -.. function:: findmax(fragment, length) - - Search *fragment* for a slice of length *length* samples (not bytes!) with - maximum energy, i.e., return *i* for which ``rms(fragment[i*2:(i+length)*2])`` - is maximal. The fragments should both contain 2-byte samples. - - The routine takes time proportional to ``len(fragment)``. - - -.. function:: getsample(fragment, width, index) - - Return the value of sample *index* from the fragment. - - -.. function:: lin2adpcm(fragment, width, state) - - Convert samples to 4 bit Intel/DVI ADPCM encoding. ADPCM coding is an adaptive - coding scheme, whereby each 4 bit number is the difference between one sample - and the next, divided by a (varying) step. The Intel/DVI ADPCM algorithm has - been selected for use by the IMA, so it may well become a standard. - - *state* is a tuple containing the state of the coder. The coder returns a tuple - ``(adpcmfrag, newstate)``, and the *newstate* should be passed to the next call - of :func:`lin2adpcm`. In the initial call, ``None`` can be passed as the state. - *adpcmfrag* is the ADPCM coded fragment packed 2 4-bit values per byte. - - -.. function:: lin2alaw(fragment, width) - - Convert samples in the audio fragment to a-LAW encoding and return this as a - bytes object. a-LAW is an audio encoding format whereby you get a dynamic - range of about 13 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: lin2lin(fragment, width, newwidth) - - Convert samples between 1-, 2-, 3- and 4-byte formats. - - .. note:: - - In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are - signed, but 8 bit samples are unsigned. So when converting to 8 bit wide - samples for these formats, you need to also add 128 to the result:: - - new_frames = audioop.lin2lin(frames, old_width, 1) - new_frames = audioop.bias(new_frames, 1, 128) - - The same, in reverse, has to be applied when converting from 8 to 16, 24 - or 32 bit width samples. - - -.. function:: lin2ulaw(fragment, width) - - Convert samples in the audio fragment to u-LAW encoding and return this as a - bytes object. u-LAW is an audio encoding format whereby you get a dynamic - range of about 14 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: max(fragment, width) - - Return the maximum of the *absolute value* of all samples in a fragment. - - -.. function:: maxpp(fragment, width) - - Return the maximum peak-peak value in the sound fragment. - - -.. function:: minmax(fragment, width) - - Return a tuple consisting of the minimum and maximum values of all samples in - the sound fragment. - - -.. function:: mul(fragment, width, factor) - - Return a fragment that has all samples in the original fragment multiplied by - the floating-point value *factor*. Samples are truncated in case of overflow. - - -.. function:: ratecv(fragment, width, nchannels, inrate, outrate, state[, weightA[, weightB]]) - - Convert the frame rate of the input fragment. - - *state* is a tuple containing the state of the converter. The converter returns - a tuple ``(newfragment, newstate)``, and *newstate* should be passed to the next - call of :func:`ratecv`. The initial call should pass ``None`` as the state. - - The *weightA* and *weightB* arguments are parameters for a simple digital filter - and default to ``1`` and ``0`` respectively. - - -.. function:: reverse(fragment, width) - - Reverse the samples in a fragment and returns the modified fragment. - - -.. function:: rms(fragment, width) - - Return the root-mean-square of the fragment, i.e. ``sqrt(sum(S_i^2)/n)``. - - This is a measure of the power in an audio signal. - - -.. function:: tomono(fragment, width, lfactor, rfactor) - - Convert a stereo fragment to a mono fragment. The left channel is multiplied by - *lfactor* and the right channel by *rfactor* before adding the two channels to - give a mono signal. - - -.. function:: tostereo(fragment, width, lfactor, rfactor) - - Generate a stereo fragment from a mono fragment. Each pair of samples in the - stereo fragment are computed from the mono sample, whereby left channel samples - are multiplied by *lfactor* and right channel samples by *rfactor*. - - -.. function:: ulaw2lin(fragment, width) - - Convert sound fragments in u-LAW encoding to linearly encoded sound fragments. - u-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - -Note that operations such as :func:`.mul` or :func:`.max` make no distinction -between mono and stereo fragments, i.e. all samples are treated equal. If this -is a problem the stereo fragment should be split into two mono fragments first -and recombined later. Here is an example of how to do that:: - - def mul_stereo(sample, width, lfactor, rfactor): - lsample = audioop.tomono(sample, width, 1, 0) - rsample = audioop.tomono(sample, width, 0, 1) - lsample = audioop.mul(lsample, width, lfactor) - rsample = audioop.mul(rsample, width, rfactor) - lsample = audioop.tostereo(lsample, width, 1, 0) - rsample = audioop.tostereo(rsample, width, 0, 1) - return audioop.add(lsample, rsample, width) - -If you use the ADPCM coder to build network packets and you want your protocol -to be stateless (i.e. to be able to tolerate packet loss) you should not only -transmit the data but also the state. Note that you should send the *initial* -state (the one you passed to :func:`lin2adpcm`) along to the decoder, not the -final state (as returned by the coder). If you want to use -:class:`struct.Struct` to store the state in binary you can code the first -element (the predicted value) in 16 bits and the second (the delta index) in 8. - -The ADPCM coders have never been tried against other ADPCM coders, only against -themselves. It could well be that I misinterpreted the standards in which case -they will not be interoperable with the respective standards. - -The :func:`find\*` routines might look a bit funny at first sight. They are -primarily meant to do echo cancellation. A reasonably fast way to do this is to -pick the most energetic piece of the output sample, locate that in the input -sample and subtract the whole output sample from the input sample:: - - def echocancel(outputdata, inputdata): - pos = audioop.findmax(outputdata, 800) # one tenth second - out_test = outputdata[pos*2:] - in_test = inputdata[pos*2:] - ipos, factor = audioop.findfit(in_test, out_test) - # Optional (for better cancellation): - # factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)], - # out_test) - prefill = '\0'*(pos+ipos)*2 - postfill = '\0'*(len(inputdata)-len(prefill)-len(outputdata)) - outputdata = prefill + audioop.mul(outputdata, 2, -factor) + postfill - return audioop.add(inputdata, outputdata, 2) - diff --git a/third_party/python/Doc/library/base64.rst b/third_party/python/Doc/library/base64.rst deleted file mode 100644 index ad9f5f58b..000000000 --- a/third_party/python/Doc/library/base64.rst +++ /dev/null @@ -1,290 +0,0 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== - -.. module:: base64 - :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings; - Base85 and Ascii85 - -**Source code:** :source:`Lib/base64.py` - -.. index:: - pair: base64; encoding - single: MIME; base64 encoding - --------------- - -This module provides functions for encoding binary data to printable -ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`3548`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`3548` encodings are suitable for encoding binary data so that it can -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. - -There are two interfaces provided by this module. The modern interface -supports encoding :term:`bytes-like objects ` to ASCII -:class:`bytes`, and decoding :term:`bytes-like objects ` or -strings containing ASCII to :class:`bytes`. Both base-64 alphabets -defined in :rfc:`3548` (normal, and URL- and filesystem-safe) are supported. - -The legacy interface does not support decoding from strings, but it does -provide functions for encoding and decoding to and from :term:`file objects -`. It only supports the Base64 standard alphabet, and it adds -newlines every 76 characters as per :rfc:`2045`. Note that if you are looking -for :rfc:`2045` support you probably want to be looking at the :mod:`email` -package instead. - - -.. versionchanged:: 3.3 - ASCII-only Unicode strings are now accepted by the decoding functions of - the modern interface. - -.. versionchanged:: 3.4 - Any :term:`bytes-like objects ` are now accepted by all - encoding and decoding functions in this module. Ascii85/Base85 support added. - -The modern interface provides: - -.. function:: b64encode(s, altchars=None) - - Encode the :term:`bytes-like object` *s* using Base64 and return the encoded - :class:`bytes`. - - Optional *altchars* must be a :term:`bytes-like object` of at least - length 2 (additional characters are ignored) which specifies an alternative - alphabet for the ``+`` and ``/`` characters. This allows an application to e.g. - generate URL or filesystem safe Base64 strings. The default is ``None``, for - which the standard Base64 alphabet is used. - - -.. function:: b64decode(s, altchars=None, validate=False) - - Decode the Base64 encoded :term:`bytes-like object` or ASCII string - *s* and return the decoded :class:`bytes`. - - Optional *altchars* must be a :term:`bytes-like object` or ASCII string of - at least length 2 (additional characters are ignored) which specifies the - alternative alphabet used instead of the ``+`` and ``/`` characters. - - A :exc:`binascii.Error` exception is raised - if *s* is incorrectly padded. - - If *validate* is ``False`` (the default), characters that are neither - in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. - - -.. function:: standard_b64encode(s) - - Encode :term:`bytes-like object` *s* using the standard Base64 alphabet - and return the encoded :class:`bytes`. - - -.. function:: standard_b64decode(s) - - Decode :term:`bytes-like object` or ASCII string *s* using the standard - Base64 alphabet and return the decoded :class:`bytes`. - - -.. function:: urlsafe_b64encode(s) - - Encode :term:`bytes-like object` *s* using the - URL- and filesystem-safe alphabet, which - substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the - standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. - - -.. function:: urlsafe_b64decode(s) - - Decode :term:`bytes-like object` or ASCII string *s* - using the URL- and filesystem-safe - alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of - ``/`` in the standard Base64 alphabet, and return the decoded - :class:`bytes`. - - -.. function:: b32encode(s) - - Encode the :term:`bytes-like object` *s* using Base32 and return the - encoded :class:`bytes`. - - -.. function:: b32decode(s, casefold=False, map01=None) - - Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. - - Optional *casefold* is a flag specifying - whether a lowercase alphabet is acceptable as input. For security purposes, - the default is ``False``. - - :rfc:`3548` allows for optional mapping of the digit 0 (zero) to the letter O - (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) - or letter L (el). The optional argument *map01* when not ``None``, specifies - which letter the digit 1 should be mapped to (when *map01* is not ``None``, the - digit 0 is always mapped to the letter O). For security purposes the default is - ``None``, so that 0 and 1 are not allowed in the input. - - A :exc:`binascii.Error` is raised if *s* is - incorrectly padded or if there are non-alphabet characters present in the - input. - - -.. function:: b16encode(s) - - Encode the :term:`bytes-like object` *s* using Base16 and return the - encoded :class:`bytes`. - - -.. function:: b16decode(s, casefold=False) - - Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. - - Optional *casefold* is a flag specifying whether a - lowercase alphabet is acceptable as input. For security purposes, the default - is ``False``. - - A :exc:`binascii.Error` is raised if *s* is - incorrectly padded or if there are non-alphabet characters present in the - input. - - -.. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) - - Encode the :term:`bytes-like object` *b* using Ascii85 and return the - encoded :class:`bytes`. - - *foldspaces* is an optional flag that uses the special short sequence 'y' - instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. - - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long. - - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. - - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. - - .. versionadded:: 3.4 - - -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \\t\\n\\r\\v') - - Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. - - *foldspaces* is a flag that specifies whether the 'y' short sequence - should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. - - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). - - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by - default contains all whitespace characters in ASCII. - - .. versionadded:: 3.4 - - -.. function:: b85encode(b, pad=False) - - Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. - git-style binary diffs) and return the encoded :class:`bytes`. - - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - - .. versionadded:: 3.4 - - -.. function:: b85decode(b) - - Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. - - .. versionadded:: 3.4 - - -The legacy interface: - -.. function:: decode(input, output) - - Decode the contents of the binary *input* file and write the resulting binary - data to the *output* file. *input* and *output* must be :term:`file objects - `. *input* will be read until ``input.readline()`` returns an - empty bytes object. - - -.. function:: decodebytes(s) - - Decode the :term:`bytes-like object` *s*, which must contain one or more - lines of base64 encoded data, and return the decoded :class:`bytes`. - - .. versionadded:: 3.1 - -.. function:: decodestring(s) - - Deprecated alias of :func:`decodebytes`. - - .. deprecated:: 3.1 - - -.. function:: encode(input, output) - - Encode the contents of the binary *input* file and write the resulting base64 - encoded data to the *output* file. *input* and *output* must be :term:`file - objects `. *input* will be read until ``input.read()`` returns - an empty bytes object. :func:`encode` inserts a newline character (``b'\n'``) - after every 76 bytes of the output, as well as ensuring that the output - always ends with a newline, as per :rfc:`2045` (MIME). - - -.. function:: encodebytes(s) - - Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary - data, and return :class:`bytes` containing the base64-encoded data, with newlines - (``b'\n'``) inserted after every 76 bytes of output, and ensuring that - there is a trailing newline, as per :rfc:`2045` (MIME). - - .. versionadded:: 3.1 - -.. function:: encodestring(s) - - Deprecated alias of :func:`encodebytes`. - - .. deprecated:: 3.1 - - -An example usage of the module: - - >>> import base64 - >>> encoded = base64.b64encode(b'data to be encoded') - >>> encoded - b'ZGF0YSB0byBiZSBlbmNvZGVk' - >>> data = base64.b64decode(encoded) - >>> data - b'data to be encoded' - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. - - :rfc:`1521` - MIME (Multipurpose Internet Mail Extensions) Part One: Mechanisms for Specifying and Describing the Format of Internet Message Bodies - Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the - base64 encoding. - diff --git a/third_party/python/Doc/library/bdb.rst b/third_party/python/Doc/library/bdb.rst deleted file mode 100644 index 116ffcf88..000000000 --- a/third_party/python/Doc/library/bdb.rst +++ /dev/null @@ -1,372 +0,0 @@ -:mod:`bdb` --- Debugger framework -================================= - -.. module:: bdb - :synopsis: Debugger framework. - -**Source code:** :source:`Lib/bdb.py` - --------------- - -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints -or managing execution via the debugger. - -The following exception is defined: - -.. exception:: BdbQuit - - Exception raised by the :class:`Bdb` class for quitting the debugger. - - -The :mod:`bdb` module also defines two classes: - -.. class:: Breakpoint(self, file, line, temporary=0, cond=None, funcname=None) - - This class implements temporary breakpoints, ignore counts, disabling and - (re-)enabling, and conditionals. - - Breakpoints are indexed by number through a list called :attr:`bpbynumber` - and by ``(file, line)`` pairs through :attr:`bplist`. The former points to a - single instance of class :class:`Breakpoint`. The latter points to a list of - such instances since there may be more than one breakpoint per line. - - When creating a breakpoint, its associated filename should be in canonical - form. If a *funcname* is defined, a breakpoint hit will be counted when the - first line of that function is executed. A conditional breakpoint always - counts a hit. - - :class:`Breakpoint` instances have the following methods: - - .. method:: deleteMe() - - Delete the breakpoint from the list associated to a file/line. If it is - the last breakpoint in that position, it also deletes the entry for the - file/line. - - - .. method:: enable() - - Mark the breakpoint as enabled. - - - .. method:: disable() - - Mark the breakpoint as disabled. - - - .. method:: bpformat() - - Return a string with all the information about the breakpoint, nicely - formatted: - - * The breakpoint number. - * If it is temporary or not. - * Its file,line position. - * The condition that causes a break. - * If it must be ignored the next N times. - * The breakpoint hit count. - - .. versionadded:: 3.2 - - .. method:: bpprint(out=None) - - Print the output of :meth:`bpformat` to the file *out*, or if it is - ``None``, to standard output. - - -.. class:: Bdb(skip=None) - - The :class:`Bdb` class acts as a generic Python debugger base class. - - This class takes care of the details of the trace facility; a derived class - should implement user interaction. The standard debugger class - (:class:`pdb.Pdb`) is an example. - - The *skip* argument, if given, must be an iterable of glob-style - module name patterns. The debugger will not step into frames that - originate in a module that matches one of these patterns. Whether a - frame is considered to originate in a certain module is determined - by the ``__name__`` in the frame globals. - - .. versionadded:: 3.1 - The *skip* argument. - - The following methods of :class:`Bdb` normally don't need to be overridden. - - .. method:: canonic(filename) - - Auxiliary method for getting a filename in a canonical form, that is, as a - case-normalized (on case-insensitive filesystems) absolute path, stripped - of surrounding angle brackets. - - .. method:: reset() - - Set the :attr:`botframe`, :attr:`stopframe`, :attr:`returnframe` and - :attr:`quitting` attributes with values ready to start debugging. - - .. method:: trace_dispatch(frame, event, arg) - - This function is installed as the trace function of debugged frames. Its - return value is the new trace function (in most cases, that is, itself). - - The default implementation decides how to dispatch a frame, depending on - the type of event (passed as a string) that is about to be executed. - *event* can be one of the following: - - * ``"line"``: A new line of code is going to be executed. - * ``"call"``: A function is about to be called, or another code block - entered. - * ``"return"``: A function or other code block is about to return. - * ``"exception"``: An exception has occurred. - * ``"c_call"``: A C function is about to be called. - * ``"c_return"``: A C function has returned. - * ``"c_exception"``: A C function has raised an exception. - - For the Python events, specialized functions (see below) are called. For - the C events, no action is taken. - - The *arg* parameter depends on the previous event. - - See the documentation for :func:`sys.settrace` for more information on the - trace function. For more information on code and frame objects, refer to - :ref:`types`. - - .. method:: dispatch_line(frame) - - If the debugger should stop on the current line, invoke the - :meth:`user_line` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_line`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_call(frame, arg) - - If the debugger should stop on this function call, invoke the - :meth:`user_call` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_call`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_return(frame, arg) - - If the debugger should stop on this function return, invoke the - :meth:`user_return` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_return`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_exception(frame, arg) - - If the debugger should stop at this exception, invokes the - :meth:`user_exception` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_exception`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - Normally derived classes don't override the following methods, but they may - if they want to redefine the definition of stopping and breakpoints. - - .. method:: stop_here(frame) - - This method checks if the *frame* is somewhere below :attr:`botframe` in - the call stack. :attr:`botframe` is the frame in which debugging started. - - .. method:: break_here(frame) - - This method checks if there is a breakpoint in the filename and line - belonging to *frame* or, at least, in the current function. If the - breakpoint is a temporary one, this method deletes it. - - .. method:: break_anywhere(frame) - - This method checks if there is a breakpoint in the filename of the current - frame. - - Derived classes should override these methods to gain control over debugger - operation. - - .. method:: user_call(frame, argument_list) - - This method is called from :meth:`dispatch_call` when there is the - possibility that a break might be necessary anywhere inside the called - function. - - .. method:: user_line(frame) - - This method is called from :meth:`dispatch_line` when either - :meth:`stop_here` or :meth:`break_here` yields ``True``. - - .. method:: user_return(frame, return_value) - - This method is called from :meth:`dispatch_return` when :meth:`stop_here` - yields ``True``. - - .. method:: user_exception(frame, exc_info) - - This method is called from :meth:`dispatch_exception` when - :meth:`stop_here` yields ``True``. - - .. method:: do_clear(arg) - - Handle how a breakpoint must be removed when it is a temporary one. - - This method must be implemented by derived classes. - - - Derived classes and clients can call the following methods to affect the - stepping state. - - .. method:: set_step() - - Stop after one line of code. - - .. method:: set_next(frame) - - Stop on the next line in or below the given frame. - - .. method:: set_return(frame) - - Stop when returning from the given frame. - - .. method:: set_until(frame) - - Stop when the line with the line no greater than the current one is - reached or when returning from current frame. - - .. method:: set_trace([frame]) - - Start debugging from *frame*. If *frame* is not specified, debugging - starts from caller's frame. - - .. method:: set_continue() - - Stop only at breakpoints or when finished. If there are no breakpoints, - set the system trace function to ``None``. - - .. method:: set_quit() - - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in - the next call to one of the :meth:`dispatch_\*` methods. - - - Derived classes and clients can call the following methods to manipulate - breakpoints. These methods return a string containing an error message if - something went wrong, or ``None`` if all is well. - - .. method:: set_break(filename, lineno, temporary=0, cond, funcname) - - Set a new breakpoint. If the *lineno* line doesn't exist for the - *filename* passed as argument, return an error message. The *filename* - should be in canonical form, as described in the :meth:`canonic` method. - - .. method:: clear_break(filename, lineno) - - Delete the breakpoints in *filename* and *lineno*. If none were set, an - error message is returned. - - .. method:: clear_bpbynumber(arg) - - Delete the breakpoint which has the index *arg* in the - :attr:`Breakpoint.bpbynumber`. If *arg* is not numeric or out of range, - return an error message. - - .. method:: clear_all_file_breaks(filename) - - Delete all breakpoints in *filename*. If none were set, an error message - is returned. - - .. method:: clear_all_breaks() - - Delete all existing breakpoints. - - .. method:: get_bpbynumber(arg) - - Return a breakpoint specified by the given number. If *arg* is a string, - it will be converted to a number. If *arg* is a non-numeric string, if - the given breakpoint never existed or has been deleted, a - :exc:`ValueError` is raised. - - .. versionadded:: 3.2 - - .. method:: get_break(filename, lineno) - - Check if there is a breakpoint for *lineno* of *filename*. - - .. method:: get_breaks(filename, lineno) - - Return all breakpoints for *lineno* in *filename*, or an empty list if - none are set. - - .. method:: get_file_breaks(filename) - - Return all breakpoints in *filename*, or an empty list if none are set. - - .. method:: get_all_breaks() - - Return all breakpoints that are set. - - - Derived classes and clients can call the following methods to get a data - structure representing a stack trace. - - .. method:: get_stack(f, t) - - Get a list of records for a frame and all higher (calling) and lower - frames, and the size of the higher part. - - .. method:: format_stack_entry(frame_lineno, lprefix=': ') - - Return a string with information about a stack entry, identified by a - ``(frame, lineno)`` tuple: - - * The canonical form of the filename which contains the frame. - * The function name, or ``""``. - * The input arguments. - * The return value. - * The line of code (if it exists). - - - The following two methods can be called by clients to use a debugger to debug - a :term:`statement`, given as a string. - - .. method:: run(cmd, globals=None, locals=None) - - Debug a statement executed via the :func:`exec` function. *globals* - defaults to :attr:`__main__.__dict__`, *locals* defaults to *globals*. - - .. method:: runeval(expr, globals=None, locals=None) - - Debug an expression executed via the :func:`eval` function. *globals* and - *locals* have the same meaning as in :meth:`run`. - - .. method:: runctx(cmd, globals, locals) - - For backwards compatibility. Calls the :meth:`run` method. - - .. method:: runcall(func, *args, **kwds) - - Debug a single function call, and return its result. - - -Finally, the module defines the following functions: - -.. function:: checkfuncname(b, frame) - - Check whether we should break here, depending on the way the breakpoint *b* - was set. - - If it was set via line number, it checks if ``b.line`` is the same as the one - in the frame also passed as argument. If the breakpoint was set via function - name, we have to check we are in the right frame (the right function) and if - we are in its first executable line. - -.. function:: effective(file, line, frame) - - Determine if there is an effective (active) breakpoint at this line of code. - Return a tuple of the breakpoint and a boolean that indicates if it is ok - to delete a temporary breakpoint. Return ``(None, None)`` if there is no - matching breakpoint. - -.. function:: set_trace() - - Start debugging with a :class:`Bdb` instance from caller's frame. diff --git a/third_party/python/Doc/library/binary.rst b/third_party/python/Doc/library/binary.rst deleted file mode 100644 index 51fbdc1a4..000000000 --- a/third_party/python/Doc/library/binary.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _binaryservices: - -******************** -Binary Data Services -******************** - -The modules described in this chapter provide some basic services operations -for manipulation of binary data. Other operations on binary data, specifically -in relation to file formats and network protocols, are described in the -relevant sections. - -Some libraries described under :ref:`textservices` also work with either -ASCII-compatible binary formats (for example, :mod:`re`) or all binary data -(for example, :mod:`difflib`). - -In addition, see the documentation for Python's built-in binary data types in -:ref:`binaryseq`. - -.. toctree:: - - struct.rst - codecs.rst - diff --git a/third_party/python/Doc/library/binascii.rst b/third_party/python/Doc/library/binascii.rst deleted file mode 100644 index adb087ef8..000000000 --- a/third_party/python/Doc/library/binascii.rst +++ /dev/null @@ -1,186 +0,0 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== - -.. module:: binascii - :synopsis: Tools for converting between binary and various ASCII-encoded binary - representations. - -.. index:: - module: uu - module: base64 - module: binhex - --------------- - -The :mod:`binascii` module contains a number of methods to convert between -binary and various ASCII-encoded binary representations. Normally, you will not -use these functions directly but use wrapper modules like :mod:`uu`, -:mod:`base64`, or :mod:`binhex` instead. The :mod:`binascii` module contains -low-level functions written in C for greater speed that are used by the -higher-level modules. - -.. note:: - - ``a2b_*`` functions accept Unicode strings containing only ASCII characters. - Other functions only accept :term:`bytes-like objects ` (such as - :class:`bytes`, :class:`bytearray` and other objects that support the buffer - protocol). - - .. versionchanged:: 3.3 - ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. - - -The :mod:`binascii` module defines the following functions: - - -.. function:: a2b_uu(string) - - Convert a single line of uuencoded data back to binary and return the binary - data. Lines normally contain 45 (binary) bytes, except for the last line. Line - data may be followed by whitespace. - - -.. function:: b2a_uu(data) - - Convert binary data to a line of ASCII characters, the return value is the - converted line, including a newline char. The length of *data* should be at most - 45. - - -.. function:: a2b_base64(string) - - Convert a block of base64 data back to binary and return the binary data. More - than one line may be passed at a time. - - -.. function:: b2a_base64(data, \*, newline=True) - - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. - - .. versionchanged:: 3.6 - Added the *newline* parameter. - - -.. function:: a2b_qp(data, header=False) - - Convert a block of quoted-printable data back to binary and return the binary - data. More than one line may be passed at a time. If the optional argument - *header* is present and true, underscores will be decoded as spaces. - - -.. function:: b2a_qp(data, quotetabs=False, istext=True, header=False) - - Convert binary data to a line(s) of ASCII characters in quoted-printable - encoding. The return value is the converted line(s). If the optional argument - *quotetabs* is present and true, all tabs and spaces will be encoded. If the - optional argument *istext* is present and true, newlines are not encoded but - trailing whitespace will be encoded. If the optional argument *header* is - present and true, spaces will be encoded as underscores per :rfc:`1522`. If the - optional argument *header* is present and false, newline characters will be - encoded as well; otherwise linefeed conversion might corrupt the binary data - stream. - - -.. function:: a2b_hqx(string) - - Convert binhex4 formatted ASCII data to binary, without doing RLE-decompression. - The string should contain a complete number of binary bytes, or (in case of the - last portion of the binhex4 data) have the remaining bits zero. - - -.. function:: rledecode_hqx(data) - - Perform RLE-decompression on the data, as per the binhex4 standard. The - algorithm uses ``0x90`` after a byte as a repeat indicator, followed by a count. - A count of ``0`` specifies a byte value of ``0x90``. The routine returns the - decompressed data, unless data input data ends in an orphaned repeat indicator, - in which case the :exc:`Incomplete` exception is raised. - - .. versionchanged:: 3.2 - Accept only bytestring or bytearray objects as input. - - -.. function:: rlecode_hqx(data) - - Perform binhex4 style RLE-compression on *data* and return the result. - - -.. function:: b2a_hqx(data) - - Perform hexbin4 binary-to-ASCII translation and return the resulting string. The - argument should already be RLE-coded, and have a length divisible by 3 (except - possibly the last fragment). - - -.. function:: crc_hqx(data, value) - - Compute a 16-bit CRC value of *data*, starting with *value* as the - initial CRC, and return the result. This uses the CRC-CCITT polynomial - *x*:sup:`16` + *x*:sup:`12` + *x*:sup:`5` + 1, often represented as - 0x1021. This CRC is used in the binhex4 format. - - -.. function:: crc32(data[, value]) - - Compute CRC-32, the 32-bit checksum of *data*, starting with an - initial CRC of *value*. The default initial CRC is zero. The algorithm - is consistent with the ZIP file checksum. Since the algorithm is designed for - use as a checksum algorithm, it is not suitable for use as a general hash - algorithm. Use as follows:: - - print(binascii.crc32(b"hello world")) - # Or, in two pieces: - crc = binascii.crc32(b"hello") - crc = binascii.crc32(b" world", crc) - print('crc32 = {:#010x}'.format(crc)) - - .. versionchanged:: 3.0 - The result is always unsigned. - To generate the same numeric value across all Python versions and - platforms, use ``crc32(data) & 0xffffffff``. - - -.. function:: b2a_hex(data) - hexlify(data) - - Return the hexadecimal representation of the binary *data*. Every byte of - *data* is converted into the corresponding 2-digit hex representation. The - returned bytes object is therefore twice as long as the length of *data*. - - -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) - - Return the binary data represented by the hexadecimal string *hexstr*. This - function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number - of hexadecimal digits (which can be upper or lower case), otherwise an - :exc:`Error` exception is raised. - - -.. exception:: Error - - Exception raised on errors. These are usually programming errors. - - -.. exception:: Incomplete - - Exception raised on incomplete data. These are usually not programming errors, - but may be handled by reading a little more data and trying again. - - -.. seealso:: - - Module :mod:`base64` - Support for RFC compliant base64-style encoding in base 16, 32, 64, - and 85. - - Module :mod:`binhex` - Support for the binhex format used on the Macintosh. - - Module :mod:`uu` - Support for UU encoding used on Unix. - - Module :mod:`quopri` - Support for quoted-printable encoding used in MIME email messages. diff --git a/third_party/python/Doc/library/binhex.rst b/third_party/python/Doc/library/binhex.rst deleted file mode 100644 index 2966e0dbf..000000000 --- a/third_party/python/Doc/library/binhex.rst +++ /dev/null @@ -1,57 +0,0 @@ -:mod:`binhex` --- Encode and decode binhex4 files -================================================= - -.. module:: binhex - :synopsis: Encode and decode files in binhex4 format. - -**Source code:** :source:`Lib/binhex.py` - --------------- - -This module encodes and decodes files in binhex4 format, a format allowing -representation of Macintosh files in ASCII. Only the data fork is handled. - -The :mod:`binhex` module defines the following functions: - - -.. function:: binhex(input, output) - - Convert a binary file with filename *input* to binhex file *output*. The - *output* parameter can either be a filename or a file-like object (any object - supporting a :meth:`write` and :meth:`close` method). - - -.. function:: hexbin(input, output) - - Decode a binhex file *input*. *input* may be a filename or a file-like object - supporting :meth:`read` and :meth:`close` methods. The resulting file is written - to a file named *output*, unless the argument is ``None`` in which case the - output filename is read from the binhex file. - -The following exception is also defined: - - -.. exception:: Error - - Exception raised when something can't be encoded using the binhex format (for - example, a filename is too long to fit in the filename field), or when input is - not properly encoded binhex data. - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. - - -.. _binhex-notes: - -Notes ------ - -There is an alternative, more powerful interface to the coder and decoder, see -the source for details. - -If you code or decode textfiles on non-Macintosh platforms they will still use -the old Macintosh newline convention (carriage-return as end of line). - diff --git a/third_party/python/Doc/library/bisect.rst b/third_party/python/Doc/library/bisect.rst deleted file mode 100644 index 6bf7814b2..000000000 --- a/third_party/python/Doc/library/bisect.rst +++ /dev/null @@ -1,149 +0,0 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== - -.. module:: bisect - :synopsis: Array bisection algorithms for binary searching. -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger -.. example based on the PyModules FAQ entry by Aaron Watters - -**Source code:** :source:`Lib/bisect.py` - --------------- - -This module provides support for maintaining a list in sorted order without -having to sort the list after each insertion. For long lists of items with -expensive comparison operations, this can be an improvement over the more common -approach. The module is called :mod:`bisect` because it uses a basic bisection -algorithm to do its work. The source code may be most useful as a working -example of the algorithm (the boundary conditions are already right!). - -The following functions are provided: - - -.. function:: bisect_left(a, x, lo=0, hi=len(a)) - - Locate the insertion point for *x* in *a* to maintain sorted order. - The parameters *lo* and *hi* may be used to specify a subset of the list - which should be considered; by default the entire list is used. If *x* is - already present in *a*, the insertion point will be before (to the left of) - any existing entries. The return value is suitable for use as the first - parameter to ``list.insert()`` assuming that *a* is already sorted. - - The returned insertion point *i* partitions the array *a* into two halves so - that ``all(val < x for val in a[lo:i])`` for the left side and - ``all(val >= x for val in a[i:hi])`` for the right side. - -.. function:: bisect_right(a, x, lo=0, hi=len(a)) - bisect(a, x, lo=0, hi=len(a)) - - Similar to :func:`bisect_left`, but returns an insertion point which comes - after (to the right of) any existing entries of *x* in *a*. - - The returned insertion point *i* partitions the array *a* into two halves so - that ``all(val <= x for val in a[lo:i])`` for the left side and - ``all(val > x for val in a[i:hi])`` for the right side. - -.. function:: insort_left(a, x, lo=0, hi=len(a)) - - Insert *x* in *a* in sorted order. This is equivalent to - ``a.insert(bisect.bisect_left(a, x, lo, hi), x)`` assuming that *a* is - already sorted. Keep in mind that the O(log n) search is dominated by - the slow O(n) insertion step. - -.. function:: insort_right(a, x, lo=0, hi=len(a)) - insort(a, x, lo=0, hi=len(a)) - - Similar to :func:`insort_left`, but inserting *x* in *a* after any existing - entries of *x*. - -.. seealso:: - - `SortedCollection recipe - `_ that uses - bisect to build a full-featured collection class with straight-forward search - methods and support for a key-function. The keys are precomputed to save - unnecessary calls to the key function during searches. - - -Searching Sorted Lists ----------------------- - -The above :func:`bisect` functions are useful for finding insertion points but -can be tricky or awkward to use for common searching tasks. The following five -functions show how to transform them into the standard lookups for sorted -lists:: - - def index(a, x): - 'Locate the leftmost value exactly equal to x' - i = bisect_left(a, x) - if i != len(a) and a[i] == x: - return i - raise ValueError - - def find_lt(a, x): - 'Find rightmost value less than x' - i = bisect_left(a, x) - if i: - return a[i-1] - raise ValueError - - def find_le(a, x): - 'Find rightmost value less than or equal to x' - i = bisect_right(a, x) - if i: - return a[i-1] - raise ValueError - - def find_gt(a, x): - 'Find leftmost value greater than x' - i = bisect_right(a, x) - if i != len(a): - return a[i] - raise ValueError - - def find_ge(a, x): - 'Find leftmost item greater than or equal to x' - i = bisect_left(a, x) - if i != len(a): - return a[i] - raise ValueError - - -Other Examples --------------- - -.. _bisect-example: - -The :func:`bisect` function can be useful for numeric table lookups. This -example uses :func:`bisect` to look up a letter grade for an exam score (say) -based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is -a 'B', and so on:: - - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] - ... - >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] - ['F', 'A', 'C', 'C', 'B', 'A', 'A'] - -Unlike the :func:`sorted` function, it does not make sense for the :func:`bisect` -functions to have *key* or *reversed* arguments because that would lead to an -inefficient design (successive calls to bisect functions would not "remember" -all of the previous key lookups). - -Instead, it is better to search a list of precomputed keys to find the index -of the record in question:: - - >>> data = [('red', 5), ('blue', 1), ('yellow', 8), ('black', 0)] - >>> data.sort(key=lambda r: r[1]) - >>> keys = [r[1] for r in data] # precomputed list of keys - >>> data[bisect_left(keys, 0)] - ('black', 0) - >>> data[bisect_left(keys, 1)] - ('blue', 1) - >>> data[bisect_left(keys, 5)] - ('red', 5) - >>> data[bisect_left(keys, 8)] - ('yellow', 8) - diff --git a/third_party/python/Doc/library/builtins.rst b/third_party/python/Doc/library/builtins.rst deleted file mode 100644 index 8fb1fef6d..000000000 --- a/third_party/python/Doc/library/builtins.rst +++ /dev/null @@ -1,42 +0,0 @@ -:mod:`builtins` --- Built-in objects -==================================== - -.. module:: builtins - :synopsis: The module that provides the built-in namespace. - --------------- - -This module provides direct access to all 'built-in' identifiers of Python; for -example, ``builtins.open`` is the full name for the built-in function -:func:`open`. See :ref:`built-in-funcs` and :ref:`built-in-consts` for -documentation. - - -This module is not normally accessed explicitly by most applications, but can be -useful in modules that provide objects with the same name as a built-in value, -but in which the built-in of that name is also needed. For example, in a module -that wants to implement an :func:`open` function that wraps the built-in -:func:`open`, this module can be used directly:: - - import builtins - - def open(path): - f = builtins.open(path, 'r') - return UpperCaser(f) - - class UpperCaser: - '''Wrapper around a file that converts output to upper-case.''' - - def __init__(self, f): - self._f = f - - def read(self, count=-1): - return self._f.read(count).upper() - - # ... - -As an implementation detail, most modules have the name ``__builtins__`` made -available as part of their globals. The value of ``__builtins__`` is normally -either this module or the value of this module's :attr:`~object.__dict__` attribute. -Since this is an implementation detail, it may not be used by alternate -implementations of Python. diff --git a/third_party/python/Doc/library/bz2.rst b/third_party/python/Doc/library/bz2.rst deleted file mode 100644 index d5f622515..000000000 --- a/third_party/python/Doc/library/bz2.rst +++ /dev/null @@ -1,252 +0,0 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= - -.. module:: bz2 - :synopsis: Interfaces for bzip2 compression and decompression. - -.. moduleauthor:: Gustavo Niemeyer -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Gustavo Niemeyer -.. sectionauthor:: Nadeem Vawda - -**Source code:** :source:`Lib/bz2.py` - --------------- - -This module provides a comprehensive interface for compressing and -decompressing data using the bzip2 compression algorithm. - -The :mod:`bz2` module contains: - -* The :func:`.open` function and :class:`BZ2File` class for reading and - writing compressed files. -* The :class:`BZ2Compressor` and :class:`BZ2Decompressor` classes for - incremental (de)compression. -* The :func:`compress` and :func:`decompress` functions for one-shot - (de)compression. - -All of the classes in this module may safely be accessed from multiple threads. - - -(De)compression of files ------------------------- - -.. function:: open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None) - - Open a bzip2-compressed file in binary or text mode, returning a :term:`file - object`. - - As with the constructor for :class:`BZ2File`, the *filename* argument can be - an actual filename (a :class:`str` or :class:`bytes` object), or an existing - file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``, - ``'x'``, ``'xb'``, ``'a'`` or ``'ab'`` for binary mode, or ``'rt'``, - ``'wt'``, ``'xt'``, or ``'at'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 1 to 9, as for the - :class:`BZ2File` constructor. - - For binary mode, this function is equivalent to the :class:`BZ2File` - constructor: ``BZ2File(filename, mode, compresslevel=compresslevel)``. In - this case, the *encoding*, *errors* and *newline* arguments must not be - provided. - - For text mode, a :class:`BZ2File` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9) - - Open a bzip2-compressed file in binary mode. - - If *filename* is a :class:`str` or :class:`bytes` object, open the named file - directly. Otherwise, *filename* should be a :term:`file object`, which will - be used to read or write the compressed data. - - The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for - overwriting, ``'x'`` for exclusive creation, or ``'a'`` for appending. These - can equivalently be given as ``'rb'``, ``'wb'``, ``'xb'`` and ``'ab'`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of - ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. - - The *buffering* argument is ignored. Its use is deprecated. - - If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between - ``1`` and ``9`` specifying the level of compression: ``1`` produces the - least compression, and ``9`` (default) produces the most compression. - - If *mode* is ``'r'``, the input file may be the concatenation of multiple - compressed streams. - - :class:`BZ2File` provides all of the members specified by the - :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`. - Iteration and the :keyword:`with` statement are supported. - - :class:`BZ2File` also provides the following method: - - .. method:: peek([n]) - - Return buffered data without advancing the file position. At least one - byte of data will be returned (unless at EOF). The exact number of bytes - returned is unspecified. - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`BZ2File`, it may change the position of the underlying file - object (e.g. if the :class:`BZ2File` was constructed by passing a file - object for *filename*). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.1 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - The :meth:`fileno`, :meth:`readable`, :meth:`seekable`, :meth:`writable`, - :meth:`read1` and :meth:`readinto` methods were added. - - .. versionchanged:: 3.3 - Support was added for *filename* being a :term:`file object` instead of an - actual filename. - - .. versionchanged:: 3.3 - The ``'a'`` (append) mode was added, along with support for reading - multi-stream files. - - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -Incremental (de)compression ---------------------------- - -.. class:: BZ2Compressor(compresslevel=9) - - Create a new compressor object. This object may be used to compress data - incrementally. For one-shot compression, use the :func:`compress` function - instead. - - *compresslevel*, if given, must be a number between ``1`` and ``9``. The - default is ``9``. - - .. method:: compress(data) - - Provide data to the compressor object. Returns a chunk of compressed data - if possible, or an empty byte string otherwise. - - When you have finished providing data to the compressor, call the - :meth:`flush` method to finish the compression process. - - - .. method:: flush() - - Finish the compression process. Returns the compressed data left in - internal buffers. - - The compressor object may not be used after this method has been called. - - -.. class:: BZ2Decompressor() - - Create a new decompressor object. This object may be used to decompress data - incrementally. For one-shot compression, use the :func:`decompress` function - instead. - - .. note:: - This class does not transparently handle inputs containing multiple - compressed streams, unlike :func:`decompress` and :class:`BZ2File`. If - you need to decompress a multi-stream input with :class:`BZ2Decompressor`, - you must use a new decompressor for each stream. - - .. method:: decompress(data, max_length=-1) - - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. - - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. - - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an `EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. - - .. attribute:: eof - - ``True`` if the end-of-stream marker has been reached. - - .. versionadded:: 3.3 - - - .. attribute:: unused_data - - Data found after the end of the compressed stream. - - If this attribute is accessed before the end of the stream has been - reached, its value will be ``b''``. - - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 - - -One-shot (de)compression ------------------------- - -.. function:: compress(data, compresslevel=9) - - Compress *data*. - - *compresslevel*, if given, must be a number between ``1`` and ``9``. The - default is ``9``. - - For incremental compression, use a :class:`BZ2Compressor` instead. - - -.. function:: decompress(data) - - Decompress *data*. - - If *data* is the concatenation of multiple compressed streams, decompress - all of the streams. - - For incremental decompression, use a :class:`BZ2Decompressor` instead. - - .. versionchanged:: 3.3 - Support for multi-stream inputs was added. - diff --git a/third_party/python/Doc/library/calendar.rst b/third_party/python/Doc/library/calendar.rst deleted file mode 100644 index 41e9e28e0..000000000 --- a/third_party/python/Doc/library/calendar.rst +++ /dev/null @@ -1,316 +0,0 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== - -.. module:: calendar - :synopsis: Functions for working with calendars, including some emulation - of the Unix cal program. - -.. sectionauthor:: Drew Csillag - -**Source code:** :source:`Lib/calendar.py` - --------------- - -This module allows you to output calendars like the Unix :program:`cal` program, -and provides additional useful functions related to the calendar. By default, -these calendars have Monday as the first day of the week, and Sunday as the last -(the European convention). Use :func:`setfirstweekday` to set the first day of -the week to Sunday (6) or to any other weekday. Parameters that specify dates -are given as integers. For related -functionality, see also the :mod:`datetime` and :mod:`time` modules. - -Most of these functions and classes rely on the :mod:`datetime` module which -uses an idealized calendar, the current Gregorian calendar extended -in both directions. This matches the definition of the "proleptic Gregorian" -calendar in Dershowitz and Reingold's book "Calendrical Calculations", where -it's the base calendar for all computations. - - -.. class:: Calendar(firstweekday=0) - - Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the - first day of the week. ``0`` is Monday (the default), ``6`` is Sunday. - - A :class:`Calendar` object provides several methods that can be used for - preparing the calendar data for formatting. This class doesn't do any formatting - itself. This is the job of subclasses. - - - :class:`Calendar` instances have the following methods: - - .. method:: iterweekdays() - - Return an iterator for the week day numbers that will be used for one - week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. - - - .. method:: itermonthdates(year, month) - - Return an iterator for the month *month* (1--12) in the year *year*. This - iterator will return all days (as :class:`datetime.date` objects) for the - month and all days before the start of the month or after the end of the - month that are required to get a complete week. - - - .. method:: itermonthdays2(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`. Days returned will be tuples consisting of a day - number and a week day number. - - - .. method:: itermonthdays(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`. Days returned will simply be day numbers. - - - .. method:: monthdatescalendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven :class:`datetime.date` objects. - - - .. method:: monthdays2calendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven tuples of day numbers and weekday - numbers. - - - .. method:: monthdayscalendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven day numbers. - - - .. method:: yeardatescalendar(year, width=3) - - Return the data for the specified year ready for formatting. The return - value is a list of month rows. Each month row contains up to *width* - months (defaulting to 3). Each month contains between 4 and 6 weeks and - each week contains 1--7 days. Days are :class:`datetime.date` objects. - - - .. method:: yeardays2calendar(year, width=3) - - Return the data for the specified year ready for formatting (similar to - :meth:`yeardatescalendar`). Entries in the week lists are tuples of day - numbers and weekday numbers. Day numbers outside this month are zero. - - - .. method:: yeardayscalendar(year, width=3) - - Return the data for the specified year ready for formatting (similar to - :meth:`yeardatescalendar`). Entries in the week lists are day numbers. Day - numbers outside this month are zero. - - -.. class:: TextCalendar(firstweekday=0) - - This class can be used to generate plain text calendars. - - :class:`TextCalendar` instances have the following methods: - - .. method:: formatmonth(theyear, themonth, w=0, l=0) - - Return a month's calendar in a multi-line string. If *w* is provided, it - specifies the width of the date columns, which are centered. If *l* is - given, it specifies the number of lines that each week will use. Depends - on the first weekday as specified in the constructor or set by the - :meth:`setfirstweekday` method. - - - .. method:: prmonth(theyear, themonth, w=0, l=0) - - Print a month's calendar as returned by :meth:`formatmonth`. - - - .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) - - Return a *m*-column calendar for an entire year as a multi-line string. - Optional parameters *w*, *l*, and *c* are for date column width, lines per - week, and number of spaces between month columns, respectively. Depends on - the first weekday as specified in the constructor or set by the - :meth:`setfirstweekday` method. The earliest year for which a calendar - can be generated is platform-dependent. - - - .. method:: pryear(theyear, w=2, l=1, c=6, m=3) - - Print the calendar for an entire year as returned by :meth:`formatyear`. - - -.. class:: HTMLCalendar(firstweekday=0) - - This class can be used to generate HTML calendars. - - - :class:`HTMLCalendar` instances have the following methods: - - .. method:: formatmonth(theyear, themonth, withyear=True) - - Return a month's calendar as an HTML table. If *withyear* is true the year - will be included in the header, otherwise just the month name will be - used. - - - .. method:: formatyear(theyear, width=3) - - Return a year's calendar as an HTML table. *width* (defaulting to 3) - specifies the number of months per row. - - - .. method:: formatyearpage(theyear, width=3, css='calendar.css', encoding=None) - - Return a year's calendar as a complete HTML page. *width* (defaulting to - 3) specifies the number of months per row. *css* is the name for the - cascading style sheet to be used. :const:`None` can be passed if no style - sheet should be used. *encoding* specifies the encoding to be used for the - output (defaulting to the system default encoding). - - -.. class:: LocaleTextCalendar(firstweekday=0, locale=None) - - This subclass of :class:`TextCalendar` can be passed a locale name in the - constructor and will return month and weekday names in the specified locale. - If this locale includes an encoding all strings containing month and weekday - names will be returned as unicode. - - -.. class:: LocaleHTMLCalendar(firstweekday=0, locale=None) - - This subclass of :class:`HTMLCalendar` can be passed a locale name in the - constructor and will return month and weekday names in the specified - locale. If this locale includes an encoding all strings containing month and - weekday names will be returned as unicode. - -.. note:: - - The :meth:`formatweekday` and :meth:`formatmonthname` methods of these two - classes temporarily change the current locale to the given *locale*. Because - the current locale is a process-wide setting, they are not thread-safe. - - -For simple text calendars this module provides the following functions. - -.. function:: setfirstweekday(weekday) - - Sets the weekday (``0`` is Monday, ``6`` is Sunday) to start each week. The - values :const:`MONDAY`, :const:`TUESDAY`, :const:`WEDNESDAY`, :const:`THURSDAY`, - :const:`FRIDAY`, :const:`SATURDAY`, and :const:`SUNDAY` are provided for - convenience. For example, to set the first weekday to Sunday:: - - import calendar - calendar.setfirstweekday(calendar.SUNDAY) - - -.. function:: firstweekday() - - Returns the current setting for the weekday to start each week. - - -.. function:: isleap(year) - - Returns :const:`True` if *year* is a leap year, otherwise :const:`False`. - - -.. function:: leapdays(y1, y2) - - Returns the number of leap years in the range from *y1* to *y2* (exclusive), - where *y1* and *y2* are years. - - This function works for ranges spanning a century change. - - -.. function:: weekday(year, month, day) - - Returns the day of the week (``0`` is Monday) for *year* (``1970``--...), - *month* (``1``--``12``), *day* (``1``--``31``). - - -.. function:: weekheader(n) - - Return a header containing abbreviated weekday names. *n* specifies the width in - characters for one weekday. - - -.. function:: monthrange(year, month) - - Returns weekday of first day of the month and number of days in month, for the - specified *year* and *month*. - - -.. function:: monthcalendar(year, month) - - Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month a represented by zeros. Each week begins with Monday - unless set by :func:`setfirstweekday`. - - -.. function:: prmonth(theyear, themonth, w=0, l=0) - - Prints a month's calendar as returned by :func:`month`. - - -.. function:: month(theyear, themonth, w=0, l=0) - - Returns a month's calendar in a multi-line string using the :meth:`formatmonth` - of the :class:`TextCalendar` class. - - -.. function:: prcal(year, w=0, l=0, c=6, m=3) - - Prints the calendar for an entire year as returned by :func:`calendar`. - - -.. function:: calendar(year, w=2, l=1, c=6, m=3) - - Returns a 3-column calendar for an entire year as a multi-line string using - the :meth:`formatyear` of the :class:`TextCalendar` class. - - -.. function:: timegm(tuple) - - An unrelated but handy function that takes a time tuple such as returned by - the :func:`~time.gmtime` function in the :mod:`time` module, and returns the - corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' - inverse. - - -The :mod:`calendar` module exports the following data attributes: - -.. data:: day_name - - An array that represents the days of the week in the current locale. - - -.. data:: day_abbr - - An array that represents the abbreviated days of the week in the current locale. - - -.. data:: month_name - - An array that represents the months of the year in the current locale. This - follows normal convention of January being month number 1, so it has a length of - 13 and ``month_name[0]`` is the empty string. - - -.. data:: month_abbr - - An array that represents the abbreviated months of the year in the current - locale. This follows normal convention of January being month number 1, so it - has a length of 13 and ``month_abbr[0]`` is the empty string. - - -.. seealso:: - - Module :mod:`datetime` - Object-oriented interface to dates and times with similar functionality to the - :mod:`time` module. - - Module :mod:`time` - Low-level time related functions. diff --git a/third_party/python/Doc/library/cgi.rst b/third_party/python/Doc/library/cgi.rst deleted file mode 100644 index a3dad1b8b..000000000 --- a/third_party/python/Doc/library/cgi.rst +++ /dev/null @@ -1,540 +0,0 @@ -:mod:`cgi` --- Common Gateway Interface support -=============================================== - -.. module:: cgi - :synopsis: Helpers for running Python scripts via the Common Gateway Interface. - -**Source code:** :source:`Lib/cgi.py` - -.. index:: - pair: WWW; server - pair: CGI; protocol - pair: HTTP; protocol - pair: MIME; headers - single: URL - single: Common Gateway Interface - --------------- - -Support module for Common Gateway Interface (CGI) scripts. - -This module defines a number of utilities for use by CGI scripts written in -Python. - - -Introduction ------------- - -.. _cgi-intro: - -A CGI script is invoked by an HTTP server, usually to process user input -submitted through an HTML ``
    `` or ```` element. - -Most often, CGI scripts live in the server's special :file:`cgi-bin` directory. -The HTTP server places all sorts of information about the request (such as the -client's hostname, the requested URL, the query string, and lots of other -goodies) in the script's shell environment, executes the script, and sends the -script's output back to the client. - -The script's input is connected to the client too, and sometimes the form data -is read this way; at other times the form data is passed via the "query string" -part of the URL. This module is intended to take care of the different cases -and provide a simpler interface to the Python script. It also provides a number -of utilities that help in debugging scripts, and the latest addition is support -for file uploads from a form (if your browser supports it). - -The output of a CGI script should consist of two sections, separated by a blank -line. The first section contains a number of headers, telling the client what -kind of data is following. Python code to generate a minimal header section -looks like this:: - - print("Content-Type: text/html") # HTML is following - print() # blank line, end of headers - -The second section is usually HTML, which allows the client software to display -nicely formatted text with header, in-line images, etc. Here's Python code that -prints a simple piece of HTML:: - - print("CGI script output") - print("

    This is my first CGI script

    ") - print("Hello, world!") - - -.. _using-the-cgi-module: - -Using the cgi module --------------------- - -Begin by writing ``import cgi``. - -When you write a new script, consider adding these lines:: - - import cgitb - cgitb.enable() - -This activates a special exception handler that will display detailed reports in -the Web browser if any errors occur. If you'd rather not show the guts of your -program to users of your script, you can have the reports saved to files -instead, with code like this:: - - import cgitb - cgitb.enable(display=0, logdir="/path/to/logdir") - -It's very helpful to use this feature during script development. The reports -produced by :mod:`cgitb` provide information that can save you a lot of time in -tracking down bugs. You can always remove the ``cgitb`` line later when you -have tested your script and are confident that it works correctly. - -To get at submitted form data, use the :class:`FieldStorage` class. If the form -contains non-ASCII characters, use the *encoding* keyword parameter set to the -value of the encoding defined for the document. It is usually contained in the -META tag in the HEAD section of the HTML document or by the -:mailheader:`Content-Type` header). This reads the form contents from the -standard input or the environment (depending on the value of various -environment variables set according to the CGI standard). Since it may consume -standard input, it should be instantiated only once. - -The :class:`FieldStorage` instance can be indexed like a Python dictionary. -It allows membership testing with the :keyword:`in` operator, and also supports -the standard dictionary method :meth:`~dict.keys` and the built-in function -:func:`len`. Form fields containing empty strings are ignored and do not appear -in the dictionary; to keep such values, provide a true value for the optional -*keep_blank_values* keyword parameter when creating the :class:`FieldStorage` -instance. - -For instance, the following code (which assumes that the -:mailheader:`Content-Type` header and blank line have already been printed) -checks that the fields ``name`` and ``addr`` are both set to a non-empty -string:: - - form = cgi.FieldStorage() - if "name" not in form or "addr" not in form: - print("

    Error

    ") - print("Please fill in the name and addr fields.") - return - print("

    name:", form["name"].value) - print("

    addr:", form["addr"].value) - ...further form processing here... - -Here the fields, accessed through ``form[key]``, are themselves instances of -:class:`FieldStorage` (or :class:`MiniFieldStorage`, depending on the form -encoding). The :attr:`~FieldStorage.value` attribute of the instance yields -the string value of the field. The :meth:`~FieldStorage.getvalue` method -returns this string value directly; it also accepts an optional second argument -as a default to return if the requested key is not present. - -If the submitted form data contains more than one field with the same name, the -object retrieved by ``form[key]`` is not a :class:`FieldStorage` or -:class:`MiniFieldStorage` instance but a list of such instances. Similarly, in -this situation, ``form.getvalue(key)`` would return a list of strings. If you -expect this possibility (when your HTML form contains multiple fields with the -same name), use the :meth:`~FieldStorage.getlist` method, which always returns -a list of values (so that you do not need to special-case the single item -case). For example, this code concatenates any number of username fields, -separated by commas:: - - value = form.getlist("username") - usernames = ",".join(value) - -If a field represents an uploaded file, accessing the value via the -:attr:`~FieldStorage.value` attribute or the :meth:`~FieldStorage.getvalue` -method reads the entire file in memory as bytes. This may not be what you -want. You can test for an uploaded file by testing either the -:attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file` -attribute. You can then read the data from the :attr:`!file` -attribute before it is automatically closed as part of the garbage collection of -the :class:`FieldStorage` instance -(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will -return bytes):: - - fileitem = form["userfile"] - if fileitem.file: - # It's an uploaded file; count lines - linecount = 0 - while True: - line = fileitem.file.readline() - if not line: break - linecount = linecount + 1 - -:class:`FieldStorage` objects also support being used in a :keyword:`with` -statement, which will automatically close them when done. - -If an error is encountered when obtaining the contents of an uploaded file -(for example, when the user interrupts the form submission by clicking on -a Back or Cancel button) the :attr:`~FieldStorage.done` attribute of the -object for the field will be set to the value -1. - -The file upload draft standard entertains the possibility of uploading multiple -files from one field (using a recursive :mimetype:`multipart/\*` encoding). -When this occurs, the item will be a dictionary-like :class:`FieldStorage` item. -This can be determined by testing its :attr:`!type` attribute, which should be -:mimetype:`multipart/form-data` (or perhaps another MIME type matching -:mimetype:`multipart/\*`). In this case, it can be iterated over recursively -just like the top-level form object. - -When a form is submitted in the "old" format (as the query string or as a single -data part of type :mimetype:`application/x-www-form-urlencoded`), the items will -actually be instances of the class :class:`MiniFieldStorage`. In this case, the -:attr:`!list`, :attr:`!file`, and :attr:`filename` attributes are always ``None``. - -A form submitted via POST that also has a query string will contain both -:class:`FieldStorage` and :class:`MiniFieldStorage` items. - -.. versionchanged:: 3.4 - The :attr:`~FieldStorage.file` attribute is automatically closed upon the - garbage collection of the creating :class:`FieldStorage` instance. - -.. versionchanged:: 3.5 - Added support for the context management protocol to the - :class:`FieldStorage` class. - - -Higher Level Interface ----------------------- - -The previous section explains how to read CGI form data using the -:class:`FieldStorage` class. This section describes a higher level interface -which was added to this class to allow one to do it in a more readable and -intuitive way. The interface doesn't make the techniques described in previous -sections obsolete --- they are still useful to process file uploads efficiently, -for example. - -.. XXX: Is this true ? - -The interface consists of two simple methods. Using the methods you can process -form data in a generic way, without the need to worry whether only one or more -values were posted under one name. - -In the previous section, you learned to write following code anytime you -expected a user to post more than one value under one name:: - - item = form.getvalue("item") - if isinstance(item, list): - # The user is requesting more than one item. - else: - # The user is requesting only one item. - -This situation is common for example when a form contains a group of multiple -checkboxes with the same name:: - - - - -In most situations, however, there's only one form control with a particular -name in a form and then you expect and need only one value associated with this -name. So you write a script containing for example this code:: - - user = form.getvalue("user").upper() - -The problem with the code is that you should never expect that a client will -provide valid input to your scripts. For example, if a curious user appends -another ``user=foo`` pair to the query string, then the script would crash, -because in this situation the ``getvalue("user")`` method call returns a list -instead of a string. Calling the :meth:`~str.upper` method on a list is not valid -(since lists do not have a method of this name) and results in an -:exc:`AttributeError` exception. - -Therefore, the appropriate way to read form data values was to always use the -code which checks whether the obtained value is a single value or a list of -values. That's annoying and leads to less readable scripts. - -A more convenient approach is to use the methods :meth:`~FieldStorage.getfirst` -and :meth:`~FieldStorage.getlist` provided by this higher level interface. - - -.. method:: FieldStorage.getfirst(name, default=None) - - This method always returns only one value associated with form field *name*. - The method returns only the first value in case that more values were posted - under such name. Please note that the order in which the values are received - may vary from browser to browser and should not be counted on. [#]_ If no such - form field or value exists then the method returns the value specified by the - optional parameter *default*. This parameter defaults to ``None`` if not - specified. - - -.. method:: FieldStorage.getlist(name) - - This method always returns a list of values associated with form field *name*. - The method returns an empty list if no such form field or value exists for - *name*. It returns a list consisting of one item if only one such value exists. - -Using these methods you can write nice compact code:: - - import cgi - form = cgi.FieldStorage() - user = form.getfirst("user", "").upper() # This way it's safe. - for item in form.getlist("item"): - do_something(item) - - -.. _functions-in-cgi-module: - -Functions ---------- - -These are useful if you want more control, or if you want to employ some of the -algorithms implemented in this module in other circumstances. - - -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") - - Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are - passed to :func:`urllib.parse.parse_qs` unchanged. - -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False) - - This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` - instead. It is maintained here only for backward compatibility. - -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) - - This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` - instead. It is maintained here only for backward compatibility. - -.. function:: parse_multipart(fp, pdict) - - Parse input of type :mimetype:`multipart/form-data` (for file uploads). - Arguments are *fp* for the input file and *pdict* for a dictionary containing - other parameters in the :mailheader:`Content-Type` header. - - Returns a dictionary just like :func:`urllib.parse.parse_qs` keys are the field names, each - value is a list of values for that field. This is easy to use but not much good - if you are expecting megabytes to be uploaded --- in that case, use the - :class:`FieldStorage` class instead which is much more flexible. - - Note that this does not parse nested multipart parts --- use - :class:`FieldStorage` for that. - - .. versionchanged:: 3.6.13 - Added the *separator* parameter. - - -.. function:: parse_header(string) - - Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a - dictionary of parameters. - - -.. function:: test() - - Robust test CGI script, usable as main program. Writes minimal HTTP headers and - formats all information provided to the script in HTML form. - - -.. function:: print_environ() - - Format the shell environment in HTML. - - -.. function:: print_form(form) - - Format a form in HTML. - - -.. function:: print_directory() - - Format the current directory in HTML. - - -.. function:: print_environ_usage() - - Print a list of useful (used by CGI) environment variables in HTML. - - -.. function:: escape(s, quote=False) - - Convert the characters ``'&'``, ``'<'`` and ``'>'`` in string *s* to HTML-safe - sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the quotation mark - character (``"``) is also translated; this helps for inclusion in an HTML - attribute value delimited by double quotes, as in ````. Note - that single quotes are never translated. - - .. deprecated:: 3.2 - This function is unsafe because *quote* is false by default, and therefore - deprecated. Use :func:`html.escape` instead. - - -.. _cgi-security: - -Caring about security ---------------------- - -.. index:: pair: CGI; security - -There's one important rule: if you invoke an external program (via the -:func:`os.system` or :func:`os.popen` functions. or others with similar -functionality), make very sure you don't pass arbitrary strings received from -the client to the shell. This is a well-known security hole whereby clever -hackers anywhere on the Web can exploit a gullible CGI script to invoke -arbitrary shell commands. Even parts of the URL or field names cannot be -trusted, since the request doesn't have to come from your form! - -To be on the safe side, if you must pass a string gotten from a form to a shell -command, you should make sure the string contains only alphanumeric characters, -dashes, underscores, and periods. - - -Installing your CGI script on a Unix system -------------------------------------------- - -Read the documentation for your HTTP server and check with your local system -administrator to find the directory where CGI scripts should be installed; -usually this is in a directory :file:`cgi-bin` in the server tree. - -Make sure that your script is readable and executable by "others"; the Unix file -mode should be ``0o755`` octal (use ``chmod 0755 filename``). Make sure that the -first line of the script contains ``#!`` starting in column 1 followed by the -pathname of the Python interpreter, for instance:: - - #!/usr/local/bin/python - -Make sure the Python interpreter exists and is executable by "others". - -Make sure that any files your script needs to read or write are readable or -writable, respectively, by "others" --- their mode should be ``0o644`` for -readable and ``0o666`` for writable. This is because, for security reasons, the -HTTP server executes your script as user "nobody", without any special -privileges. It can only read (write, execute) files that everybody can read -(write, execute). The current directory at execution time is also different (it -is usually the server's cgi-bin directory) and the set of environment variables -is also different from what you get when you log in. In particular, don't count -on the shell's search path for executables (:envvar:`PATH`) or the Python module -search path (:envvar:`PYTHONPATH`) to be set to anything interesting. - -If you need to load modules from a directory which is not on Python's default -module search path, you can change the path in your script, before importing -other modules. For example:: - - import sys - sys.path.insert(0, "/usr/home/joe/lib/python") - sys.path.insert(0, "/usr/local/lib/python") - -(This way, the directory inserted last will be searched first!) - -Instructions for non-Unix systems will vary; check your HTTP server's -documentation (it will usually have a section on CGI scripts). - - -Testing your CGI script ------------------------ - -Unfortunately, a CGI script will generally not run when you try it from the -command line, and a script that works perfectly from the command line may fail -mysteriously when run from the server. There's one reason why you should still -test your script from the command line: if it contains a syntax error, the -Python interpreter won't execute it at all, and the HTTP server will most likely -send a cryptic error to the client. - -Assuming your script has no syntax errors, yet it does not work, you have no -choice but to read the next section. - - -Debugging CGI scripts ---------------------- - -.. index:: pair: CGI; debugging - -First of all, check for trivial installation errors --- reading the section -above on installing your CGI script carefully can save you a lot of time. If -you wonder whether you have understood the installation procedure correctly, try -installing a copy of this module file (:file:`cgi.py`) as a CGI script. When -invoked as a script, the file will dump its environment and the contents of the -form in HTML form. Give it the right mode etc, and send it a request. If it's -installed in the standard :file:`cgi-bin` directory, it should be possible to -send it a request by entering a URL into your browser of the form: - -.. code-block:: none - - http://yourhostname/cgi-bin/cgi.py?name=Joe+Blow&addr=At+Home - -If this gives an error of type 404, the server cannot find the script -- perhaps -you need to install it in a different directory. If it gives another error, -there's an installation problem that you should fix before trying to go any -further. If you get a nicely formatted listing of the environment and form -content (in this example, the fields should be listed as "addr" with value "At -Home" and "name" with value "Joe Blow"), the :file:`cgi.py` script has been -installed correctly. If you follow the same procedure for your own script, you -should now be able to debug it. - -The next step could be to call the :mod:`cgi` module's :func:`test` function -from your script: replace its main code with the single statement :: - - cgi.test() - -This should produce the same results as those gotten from installing the -:file:`cgi.py` file itself. - -When an ordinary Python script raises an unhandled exception (for whatever -reason: of a typo in a module name, a file that can't be opened, etc.), the -Python interpreter prints a nice traceback and exits. While the Python -interpreter will still do this when your CGI script raises an exception, most -likely the traceback will end up in one of the HTTP server's log files, or be -discarded altogether. - -Fortunately, once you have managed to get your script to execute *some* code, -you can easily send tracebacks to the Web browser using the :mod:`cgitb` module. -If you haven't done so already, just add the lines:: - - import cgitb - cgitb.enable() - -to the top of your script. Then try running it again; when a problem occurs, -you should see a detailed report that will likely make apparent the cause of the -crash. - -If you suspect that there may be a problem in importing the :mod:`cgitb` module, -you can use an even more robust approach (which only uses built-in modules):: - - import sys - sys.stderr = sys.stdout - print("Content-Type: text/plain") - print() - ...your code here... - -This relies on the Python interpreter to print the traceback. The content type -of the output is set to plain text, which disables all HTML processing. If your -script works, the raw HTML will be displayed by your client. If it raises an -exception, most likely after the first two lines have been printed, a traceback -will be displayed. Because no HTML interpretation is going on, the traceback -will be readable. - - -Common problems and solutions ------------------------------ - -* Most HTTP servers buffer the output from CGI scripts until the script is - completed. This means that it is not possible to display a progress report on - the client's display while the script is running. - -* Check the installation instructions above. - -* Check the HTTP server's log files. (``tail -f logfile`` in a separate window - may be useful!) - -* Always check a script for syntax errors first, by doing something like - ``python script.py``. - -* If your script does not have any syntax errors, try adding ``import cgitb; - cgitb.enable()`` to the top of the script. - -* When invoking external programs, make sure they can be found. Usually, this - means using absolute path names --- :envvar:`PATH` is usually not set to a very - useful value in a CGI script. - -* When reading or writing external files, make sure they can be read or written - by the userid under which your CGI script will be running: this is typically the - userid under which the web server is running, or some explicitly specified - userid for a web server's ``suexec`` feature. - -* Don't try to give a CGI script a set-uid mode. This doesn't work on most - systems, and is a security liability as well. - -.. rubric:: Footnotes - -.. [#] Note that some recent versions of the HTML specification do state what - order the field values should be supplied in, but knowing whether a request - was received from a conforming browser, or even from a browser at all, is - tedious and error-prone. diff --git a/third_party/python/Doc/library/cgitb.rst b/third_party/python/Doc/library/cgitb.rst deleted file mode 100644 index b65a6355f..000000000 --- a/third_party/python/Doc/library/cgitb.rst +++ /dev/null @@ -1,66 +0,0 @@ -:mod:`cgitb` --- Traceback manager for CGI scripts -================================================== - -.. module:: cgitb - :synopsis: Configurable traceback handler for CGI scripts. - -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/cgitb.py` - -.. index:: - single: CGI; exceptions - single: CGI; tracebacks - single: exceptions; in CGI scripts - single: tracebacks; in CGI scripts - --------------- - -The :mod:`cgitb` module provides a special exception handler for Python scripts. -(Its name is a bit misleading. It was originally designed to display extensive -traceback information in HTML for CGI scripts. It was later generalized to also -display this information in plain text.) After this module is activated, if an -uncaught exception occurs, a detailed, formatted report will be displayed. The -report includes a traceback showing excerpts of the source code for each level, -as well as the values of the arguments and local variables to currently running -functions, to help you debug the problem. Optionally, you can save this -information to a file instead of sending it to the browser. - -To enable this feature, simply add this to the top of your CGI script:: - - import cgitb - cgitb.enable() - -The options to the :func:`enable` function control whether the report is -displayed in the browser and whether the report is logged to a file for later -analysis. - - -.. function:: enable(display=1, logdir=None, context=5, format="html") - - .. index:: single: excepthook() (in module sys) - - This function causes the :mod:`cgitb` module to take over the interpreter's - default handling for exceptions by setting the value of :attr:`sys.excepthook`. - - The optional argument *display* defaults to ``1`` and can be set to ``0`` to - suppress sending the traceback to the browser. If the argument *logdir* is - present, the traceback reports are written to files. The value of *logdir* - should be a directory where these files will be placed. The optional argument - *context* is the number of lines of context to display around the current line - of source code in the traceback; this defaults to ``5``. If the optional - argument *format* is ``"html"``, the output is formatted as HTML. Any other - value forces plain text output. The default value is ``"html"``. - - -.. function:: handler(info=None) - - This function handles an exception using the default settings (that is, show a - report in the browser, but don't log to a file). This can be used when you've - caught an exception and want to report it using :mod:`cgitb`. The optional - *info* argument should be a 3-tuple containing an exception type, exception - value, and traceback object, exactly like the tuple returned by - :func:`sys.exc_info`. If the *info* argument is not supplied, the current - exception is obtained from :func:`sys.exc_info`. - diff --git a/third_party/python/Doc/library/chunk.rst b/third_party/python/Doc/library/chunk.rst deleted file mode 100644 index 5e24df923..000000000 --- a/third_party/python/Doc/library/chunk.rst +++ /dev/null @@ -1,137 +0,0 @@ -:mod:`chunk` --- Read IFF chunked data -====================================== - -.. module:: chunk - :synopsis: Module to read IFF chunks. - -.. moduleauthor:: Sjoerd Mullender -.. sectionauthor:: Sjoerd Mullender - -**Source code:** :source:`Lib/chunk.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - single: Real Media File Format - single: RMFF - --------------- - -This module provides an interface for reading files that use EA IFF 85 chunks. -[#]_ This format is used in at least the Audio Interchange File Format -(AIFF/AIFF-C) and the Real Media File Format (RMFF). The WAVE audio file format -is closely related and can also be read using this module. - -A chunk has the following structure: - -+---------+--------+-------------------------------+ -| Offset | Length | Contents | -+=========+========+===============================+ -| 0 | 4 | Chunk ID | -+---------+--------+-------------------------------+ -| 4 | 4 | Size of chunk in big-endian | -| | | byte order, not including the | -| | | header | -+---------+--------+-------------------------------+ -| 8 | *n* | Data bytes, where *n* is the | -| | | size given in the preceding | -| | | field | -+---------+--------+-------------------------------+ -| 8 + *n* | 0 or 1 | Pad byte needed if *n* is odd | -| | | and chunk alignment is used | -+---------+--------+-------------------------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) gives the -size of the chunk data, not including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed usage of -the :class:`Chunk` class defined here is to instantiate an instance at the start -of each chunk and read from the instance until it reaches the end, after which a -new instance can be instantiated. At the end of the file, creating a new -instance will fail with an :exc:`EOFError` exception. - - -.. class:: Chunk(file, align=True, bigendian=True, inclheader=False) - - Class which represents a chunk. The *file* argument is expected to be a - file-like object. An instance of this class is specifically allowed. The - only method that is needed is :meth:`~io.IOBase.read`. If the methods - :meth:`~io.IOBase.seek` and :meth:`~io.IOBase.tell` are present and don't - raise an exception, they are also used. - If these methods are present and raise an exception, they are expected to not - have altered the object. If the optional argument *align* is true, chunks - are assumed to be aligned on 2-byte boundaries. If *align* is false, no - alignment is assumed. The default value is true. If the optional argument - *bigendian* is false, the chunk size is assumed to be in little-endian order. - This is needed for WAVE audio files. The default value is true. If the - optional argument *inclheader* is true, the size given in the chunk header - includes the size of the header. The default value is false. - - A :class:`Chunk` object supports the following methods: - - - .. method:: getname() - - Returns the name (ID) of the chunk. This is the first 4 bytes of the - chunk. - - - .. method:: getsize() - - Returns the size of the chunk. - - - .. method:: close() - - Close and skip to the end of the chunk. This does not close the - underlying file. - - The remaining methods will raise :exc:`OSError` if called after the - :meth:`close` method has been called. Before Python 3.3, they used to - raise :exc:`IOError`, now an alias of :exc:`OSError`. - - - .. method:: isatty() - - Returns ``False``. - - - .. method:: seek(pos, whence=0) - - Set the chunk's current position. The *whence* argument is optional and - defaults to ``0`` (absolute file positioning); other values are ``1`` - (seek relative to the current position) and ``2`` (seek relative to the - file's end). There is no return value. If the underlying file does not - allow seek, only forward seeks are allowed. - - - .. method:: tell() - - Return the current position into the chunk. - - - .. method:: read(size=-1) - - Read at most *size* bytes from the chunk (less if the read hits the end of - the chunk before obtaining *size* bytes). If the *size* argument is - negative or omitted, read all data until the end of the chunk. An empty - bytes object is returned when the end of the chunk is encountered - immediately. - - - .. method:: skip() - - Skip to the end of the chunk. All further calls to :meth:`read` for the - chunk will return ``b''``. If you are not interested in the contents of - the chunk, this method should be called so that the file points to the - start of the next chunk. - - -.. rubric:: Footnotes - -.. [#] "EA IFF 85" Standard for Interchange Format Files, Jerry Morrison, Electronic - Arts, January 1985. - diff --git a/third_party/python/Doc/library/cmath.rst b/third_party/python/Doc/library/cmath.rst deleted file mode 100644 index c819a667e..000000000 --- a/third_party/python/Doc/library/cmath.rst +++ /dev/null @@ -1,312 +0,0 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== - -.. module:: cmath - :synopsis: Mathematical functions for complex numbers. - --------------- - -This module is always available. It provides access to mathematical functions -for complex numbers. The functions in this module accept integers, -floating-point numbers or complex numbers as arguments. They will also accept -any Python object that has either a :meth:`__complex__` or a :meth:`__float__` -method: these methods are used to convert the object to a complex or -floating-point number, respectively, and the function is then applied to the -result of the conversion. - -.. note:: - - On platforms with hardware and system-level support for signed - zeros, functions involving branch cuts are continuous on *both* - sides of the branch cut: the sign of the zero distinguishes one - side of the branch cut from the other. On platforms that do not - support signed zeros the continuity is as specified below. - - -Conversions to and from polar coordinates ------------------------------------------ - -A Python complex number ``z`` is stored internally using *rectangular* -or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j - -*Polar coordinates* give an alternative way to represent a complex -number. In polar coordinates, a complex number *z* is defined by the -modulus *r* and the phase angle *phi*. The modulus *r* is the distance -from *z* to the origin, while the phase *phi* is the counterclockwise -angle, measured in radians, from the positive x-axis to the line -segment that joins the origin to *z*. - -The following functions can be used to convert from the native -rectangular coordinates to polar coordinates and back. - -.. function:: phase(x) - - Return the phase of *x* (also known as the *argument* of *x*), as a - float. ``phase(x)`` is equivalent to ``math.atan2(x.imag, - x.real)``. The result lies in the range [-π, π], and the branch - cut for this operation lies along the negative real axis, - continuous from above. On systems with support for signed zeros - (which includes most systems in current use), this means that the - sign of the result is the same as the sign of ``x.imag``, even when - ``x.imag`` is zero:: - - >>> phase(complex(-1.0, 0.0)) - 3.141592653589793 - >>> phase(complex(-1.0, -0.0)) - -3.141592653589793 - - -.. note:: - - The modulus (absolute value) of a complex number *x* can be - computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. - - -.. function:: polar(x) - - Return the representation of *x* in polar coordinates. Returns a - pair ``(r, phi)`` where *r* is the modulus of *x* and phi is the - phase of *x*. ``polar(x)`` is equivalent to ``(abs(x), - phase(x))``. - - -.. function:: rect(r, phi) - - Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. - - -Power and logarithmic functions -------------------------------- - -.. function:: exp(x) - - Return the exponential value ``e**x``. - - -.. function:: log(x[, base]) - - Returns the logarithm of *x* to the given *base*. If the *base* is not - specified, returns the natural logarithm of *x*. There is one branch cut, from 0 - along the negative real axis to -∞, continuous from above. - - -.. function:: log10(x) - - Return the base-10 logarithm of *x*. This has the same branch cut as - :func:`log`. - - -.. function:: sqrt(x) - - Return the square root of *x*. This has the same branch cut as :func:`log`. - - -Trigonometric functions ------------------------ - -.. function:: acos(x) - - Return the arc cosine of *x*. There are two branch cuts: One extends right from - 1 along the real axis to ∞, continuous from below. The other extends left from - -1 along the real axis to -∞, continuous from above. - - -.. function:: asin(x) - - Return the arc sine of *x*. This has the same branch cuts as :func:`acos`. - - -.. function:: atan(x) - - Return the arc tangent of *x*. There are two branch cuts: One extends from - ``1j`` along the imaginary axis to ``∞j``, continuous from the right. The - other extends from ``-1j`` along the imaginary axis to ``-∞j``, continuous - from the left. - - -.. function:: cos(x) - - Return the cosine of *x*. - - -.. function:: sin(x) - - Return the sine of *x*. - - -.. function:: tan(x) - - Return the tangent of *x*. - - -Hyperbolic functions --------------------- - -.. function:: acosh(x) - - Return the inverse hyperbolic cosine of *x*. There is one branch cut, - extending left from 1 along the real axis to -∞, continuous from above. - - -.. function:: asinh(x) - - Return the inverse hyperbolic sine of *x*. There are two branch cuts: - One extends from ``1j`` along the imaginary axis to ``∞j``, - continuous from the right. The other extends from ``-1j`` along - the imaginary axis to ``-∞j``, continuous from the left. - - -.. function:: atanh(x) - - Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One - extends from ``1`` along the real axis to ``∞``, continuous from below. The - other extends from ``-1`` along the real axis to ``-∞``, continuous from - above. - - -.. function:: cosh(x) - - Return the hyperbolic cosine of *x*. - - -.. function:: sinh(x) - - Return the hyperbolic sine of *x*. - - -.. function:: tanh(x) - - Return the hyperbolic tangent of *x*. - - -Classification functions ------------------------- - -.. function:: isfinite(x) - - Return ``True`` if both the real and imaginary parts of *x* are finite, and - ``False`` otherwise. - - .. versionadded:: 3.2 - - -.. function:: isinf(x) - - Return ``True`` if either the real or the imaginary part of *x* is an - infinity, and ``False`` otherwise. - - -.. function:: isnan(x) - - Return ``True`` if either the real or the imaginary part of *x* is a NaN, - and ``False`` otherwise. - - -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - -Constants ---------- - - -.. data:: pi - - The mathematical constant *π*, as a float. - - -.. data:: e - - The mathematical constant *e*, as a float. - -.. data:: tau - - The mathematical constant *τ*, as a float. - - .. versionadded:: 3.6 - -.. data:: inf - - Floating-point positive infinity. Equivalent to ``float('inf')``. - - .. versionadded:: 3.6 - -.. data:: infj - - Complex number with zero real part and positive infinity imaginary - part. Equivalent to ``complex(0.0, float('inf'))``. - - .. versionadded:: 3.6 - -.. data:: nan - - A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. - - .. versionadded:: 3.6 - -.. data:: nanj - - Complex number with zero real part and NaN imaginary part. Equivalent to - ``complex(0.0, float('nan'))``. - - .. versionadded:: 3.6 - - -.. index:: module: math - -Note that the selection of functions is similar, but not identical, to that in -module :mod:`math`. The reason for having two modules is that some users aren't -interested in complex numbers, and perhaps don't even know what they are. They -would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a -complex number, even if the answer can be expressed as a real number (in which -case the complex number has an imaginary part of zero). - -A note on branch cuts: They are curves along which the given function fails to -be continuous. They are a necessary feature of many complex functions. It is -assumed that if you need to compute with complex functions, you will understand -about branch cuts. Consult almost any (not too elementary) book on complex -variables for enlightenment. For information of the proper choice of branch -cuts for numerical purposes, a good reference should be the following: - - -.. seealso:: - - Kahan, W: Branch cuts for complex elementary functions; or, Much ado about - nothing's sign bit. In Iserles, A., and Powell, M. (eds.), The state of the art - in numerical analysis. Clarendon Press (1987) pp165--211. diff --git a/third_party/python/Doc/library/cmd.rst b/third_party/python/Doc/library/cmd.rst deleted file mode 100644 index d57edb7eb..000000000 --- a/third_party/python/Doc/library/cmd.rst +++ /dev/null @@ -1,381 +0,0 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= - -.. module:: cmd - :synopsis: Build line-oriented command interpreters. - -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/cmd.py` - --------------- - -The :class:`Cmd` class provides a simple framework for writing line-oriented -command interpreters. These are often useful for test harnesses, administrative -tools, and prototypes that will later be wrapped in a more sophisticated -interface. - -.. class:: Cmd(completekey='tab', stdin=None, stdout=None) - - A :class:`Cmd` instance or subclass instance is a line-oriented interpreter - framework. There is no good reason to instantiate :class:`Cmd` itself; rather, - it's useful as a superclass of an interpreter class you define yourself in order - to inherit :class:`Cmd`'s methods and encapsulate action methods. - - The optional argument *completekey* is the :mod:`readline` name of a completion - key; it defaults to :kbd:`Tab`. If *completekey* is not :const:`None` and - :mod:`readline` is available, command completion is done automatically. - - The optional arguments *stdin* and *stdout* specify the input and output file - objects that the Cmd instance or subclass instance will use for input and - output. If not specified, they will default to :data:`sys.stdin` and - :data:`sys.stdout`. - - If you want a given *stdin* to be used, make sure to set the instance's - :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be - ignored. - - -.. _cmd-objects: - -Cmd Objects ------------ - -A :class:`Cmd` instance has the following methods: - - -.. method:: Cmd.cmdloop(intro=None) - - Repeatedly issue a prompt, accept input, parse an initial prefix off the - received input, and dispatch to action methods, passing them the remainder of - the line as argument. - - The optional argument is a banner or intro string to be issued before the first - prompt (this overrides the :attr:`intro` class attribute). - - If the :mod:`readline` module is loaded, input will automatically inherit - :program:`bash`\ -like history-list editing (e.g. :kbd:`Control-P` scrolls back - to the last command, :kbd:`Control-N` forward to the next one, :kbd:`Control-F` - moves the cursor to the right non-destructively, :kbd:`Control-B` moves the - cursor to the left non-destructively, etc.). - - An end-of-file on input is passed back as the string ``'EOF'``. - - .. index:: - single: ? (question mark); in a command interpreter - single: ! (exclamation); in a command interpreter - - An interpreter instance will recognize a command name ``foo`` if and only if it - has a method :meth:`do_foo`. As a special case, a line beginning with the - character ``'?'`` is dispatched to the method :meth:`do_help`. As another - special case, a line beginning with the character ``'!'`` is dispatched to the - method :meth:`do_shell` (if such a method is defined). - - This method will return when the :meth:`postcmd` method returns a true value. - The *stop* argument to :meth:`postcmd` is the return value from the command's - corresponding :meth:`do_\*` method. - - If completion is enabled, completing commands will be done automatically, and - completing of commands args is done by calling :meth:`complete_foo` with - arguments *text*, *line*, *begidx*, and *endidx*. *text* is the string prefix - we are attempting to match: all returned matches must begin with it. *line* is - the current input line with leading whitespace removed, *begidx* and *endidx* - are the beginning and ending indexes of the prefix text, which could be used to - provide different completion depending upon which position the argument is in. - - All subclasses of :class:`Cmd` inherit a predefined :meth:`do_help`. This - method, called with an argument ``'bar'``, invokes the corresponding method - :meth:`help_bar`, and if that is not present, prints the docstring of - :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all - available help topics (that is, all commands with corresponding - :meth:`help_\*` methods or commands that have docstrings), and also lists any - undocumented commands. - - -.. method:: Cmd.onecmd(str) - - Interpret the argument as though it had been typed in response to the prompt. - This may be overridden, but should not normally need to be; see the - :meth:`precmd` and :meth:`postcmd` methods for useful execution hooks. The - return value is a flag indicating whether interpretation of commands by the - interpreter should stop. If there is a :meth:`do_\*` method for the command - *str*, the return value of that method is returned, otherwise the return value - from the :meth:`default` method is returned. - - -.. method:: Cmd.emptyline() - - Method called when an empty line is entered in response to the prompt. If this - method is not overridden, it repeats the last nonempty command entered. - - -.. method:: Cmd.default(line) - - Method called on an input line when the command prefix is not recognized. If - this method is not overridden, it prints an error message and returns. - - -.. method:: Cmd.completedefault(text, line, begidx, endidx) - - Method called to complete an input line when no command-specific - :meth:`complete_\*` method is available. By default, it returns an empty list. - - -.. method:: Cmd.precmd(line) - - Hook method executed just before the command line *line* is interpreted, but - after the input prompt is generated and issued. This method is a stub in - :class:`Cmd`; it exists to be overridden by subclasses. The return value is - used as the command which will be executed by the :meth:`onecmd` method; the - :meth:`precmd` implementation may re-write the command or simply return *line* - unchanged. - - -.. method:: Cmd.postcmd(stop, line) - - Hook method executed just after a command dispatch is finished. This method is - a stub in :class:`Cmd`; it exists to be overridden by subclasses. *line* is the - command line which was executed, and *stop* is a flag which indicates whether - execution will be terminated after the call to :meth:`postcmd`; this will be the - return value of the :meth:`onecmd` method. The return value of this method will - be used as the new value for the internal flag which corresponds to *stop*; - returning false will cause interpretation to continue. - - -.. method:: Cmd.preloop() - - Hook method executed once when :meth:`cmdloop` is called. This method is a stub - in :class:`Cmd`; it exists to be overridden by subclasses. - - -.. method:: Cmd.postloop() - - Hook method executed once when :meth:`cmdloop` is about to return. This method - is a stub in :class:`Cmd`; it exists to be overridden by subclasses. - - -Instances of :class:`Cmd` subclasses have some public instance variables: - -.. attribute:: Cmd.prompt - - The prompt issued to solicit input. - - -.. attribute:: Cmd.identchars - - The string of characters accepted for the command prefix. - - -.. attribute:: Cmd.lastcmd - - The last nonempty command prefix seen. - - -.. attribute:: Cmd.cmdqueue - - A list of queued input lines. The cmdqueue list is checked in - :meth:`cmdloop` when new input is needed; if it is nonempty, its elements - will be processed in order, as if entered at the prompt. - - -.. attribute:: Cmd.intro - - A string to issue as an intro or banner. May be overridden by giving the - :meth:`cmdloop` method an argument. - - -.. attribute:: Cmd.doc_header - - The header to issue if the help output has a section for documented commands. - - -.. attribute:: Cmd.misc_header - - The header to issue if the help output has a section for miscellaneous help - topics (that is, there are :meth:`help_\*` methods without corresponding - :meth:`do_\*` methods). - - -.. attribute:: Cmd.undoc_header - - The header to issue if the help output has a section for undocumented commands - (that is, there are :meth:`do_\*` methods without corresponding :meth:`help_\*` - methods). - - -.. attribute:: Cmd.ruler - - The character used to draw separator lines under the help-message headers. If - empty, no ruler line is drawn. It defaults to ``'='``. - - -.. attribute:: Cmd.use_rawinput - - A flag, defaulting to true. If true, :meth:`cmdloop` uses :func:`input` to - display a prompt and read the next command; if false, :meth:`sys.stdout.write` - and :meth:`sys.stdin.readline` are used. (This means that by importing - :mod:`readline`, on systems that support it, the interpreter will automatically - support :program:`Emacs`\ -like line editing and command-history keystrokes.) - - -.. _cmd-example: - -Cmd Example ------------ - -.. sectionauthor:: Raymond Hettinger - -The :mod:`cmd` module is mainly useful for building custom shells that let a -user work with a program interactively. - -This section presents a simple example of how to build a shell around a few of -the commands in the :mod:`turtle` module. - -Basic turtle commands such as :meth:`~turtle.forward` are added to a -:class:`Cmd` subclass with method named :meth:`do_forward`. The argument is -converted to a number and dispatched to the turtle module. The docstring is -used in the help utility provided by the shell. - -The example also includes a basic record and playback facility implemented with -the :meth:`~Cmd.precmd` method which is responsible for converting the input to -lowercase and writing the commands to a file. The :meth:`do_playback` method -reads the file and adds the recorded commands to the :attr:`cmdqueue` for -immediate playback:: - - import cmd, sys - from turtle import * - - class TurtleShell(cmd.Cmd): - intro = 'Welcome to the turtle shell. Type help or ? to list commands.\n' - prompt = '(turtle) ' - file = None - - # ----- basic turtle commands ----- - def do_forward(self, arg): - 'Move the turtle forward by the specified distance: FORWARD 10' - forward(*parse(arg)) - def do_right(self, arg): - 'Turn turtle right by given number of degrees: RIGHT 20' - right(*parse(arg)) - def do_left(self, arg): - 'Turn turtle left by given number of degrees: LEFT 90' - left(*parse(arg)) - def do_goto(self, arg): - 'Move turtle to an absolute position with changing orientation. GOTO 100 200' - goto(*parse(arg)) - def do_home(self, arg): - 'Return turtle to the home position: HOME' - home() - def do_circle(self, arg): - 'Draw circle with given radius an options extent and steps: CIRCLE 50' - circle(*parse(arg)) - def do_position(self, arg): - 'Print the current turtle position: POSITION' - print('Current position is %d %d\n' % position()) - def do_heading(self, arg): - 'Print the current turtle heading in degrees: HEADING' - print('Current heading is %d\n' % (heading(),)) - def do_color(self, arg): - 'Set the color: COLOR BLUE' - color(arg.lower()) - def do_undo(self, arg): - 'Undo (repeatedly) the last turtle action(s): UNDO' - def do_reset(self, arg): - 'Clear the screen and return turtle to center: RESET' - reset() - def do_bye(self, arg): - 'Stop recording, close the turtle window, and exit: BYE' - print('Thank you for using Turtle') - self.close() - bye() - return True - - # ----- record and playback ----- - def do_record(self, arg): - 'Save future commands to filename: RECORD rose.cmd' - self.file = open(arg, 'w') - def do_playback(self, arg): - 'Playback commands from a file: PLAYBACK rose.cmd' - self.close() - with open(arg) as f: - self.cmdqueue.extend(f.read().splitlines()) - def precmd(self, line): - line = line.lower() - if self.file and 'playback' not in line: - print(line, file=self.file) - return line - def close(self): - if self.file: - self.file.close() - self.file = None - - def parse(arg): - 'Convert a series of zero or more numbers to an argument tuple' - return tuple(map(int, arg.split())) - - if __name__ == '__main__': - TurtleShell().cmdloop() - - -Here is a sample session with the turtle shell showing the help functions, using -blank lines to repeat commands, and the simple record and playback facility: - -.. code-block:: none - - Welcome to the turtle shell. Type help or ? to list commands. - - (turtle) ? - - Documented commands (type help ): - ======================================== - bye color goto home playback record right - circle forward heading left position reset undo - - (turtle) help forward - Move the turtle forward by the specified distance: FORWARD 10 - (turtle) record spiral.cmd - (turtle) position - Current position is 0 0 - - (turtle) heading - Current heading is 0 - - (turtle) reset - (turtle) circle 20 - (turtle) right 30 - (turtle) circle 40 - (turtle) right 30 - (turtle) circle 60 - (turtle) right 30 - (turtle) circle 80 - (turtle) right 30 - (turtle) circle 100 - (turtle) right 30 - (turtle) circle 120 - (turtle) right 30 - (turtle) circle 120 - (turtle) heading - Current heading is 180 - - (turtle) forward 100 - (turtle) - (turtle) right 90 - (turtle) forward 100 - (turtle) - (turtle) right 90 - (turtle) forward 400 - (turtle) right 90 - (turtle) forward 500 - (turtle) right 90 - (turtle) forward 400 - (turtle) right 90 - (turtle) forward 300 - (turtle) playback spiral.cmd - Current position is 0 0 - - Current heading is 0 - - Current heading is 180 - - (turtle) bye - Thank you for using Turtle diff --git a/third_party/python/Doc/library/code.rst b/third_party/python/Doc/library/code.rst deleted file mode 100644 index e2c47bab5..000000000 --- a/third_party/python/Doc/library/code.rst +++ /dev/null @@ -1,184 +0,0 @@ -:mod:`code` --- Interpreter base classes -======================================== - -.. module:: code - :synopsis: Facilities to implement read-eval-print loops. - -**Source code:** :source:`Lib/code.py` - --------------- - -The ``code`` module provides facilities to implement read-eval-print loops in -Python. Two classes and convenience functions are included which can be used to -build applications which provide an interactive interpreter prompt. - - -.. class:: InteractiveInterpreter(locals=None) - - This class deals with parsing and interpreter state (the user's namespace); it - does not deal with input buffering or prompting or input file naming (the - filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. - - -.. class:: InteractiveConsole(locals=None, filename="") - - Closely emulate the behavior of the interactive Python interpreter. This class - builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. - - -.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None) - - Convenience function to run a read-eval-print loop. This creates a new - instance of :class:`InteractiveConsole` and sets *readfunc* to be used as - the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is - provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` - method of the instance is then run with *banner* and *exitmsg* passed as the - banner and exit message to use, if provided. The console object is discarded - after use. - - .. versionchanged:: 3.6 - Added *exitmsg* parameter. - - -.. function:: compile_command(source, filename="", symbol="single") - - This function is useful for programs that want to emulate Python's interpreter - main loop (a.k.a. the read-eval-print loop). The tricky part is to determine - when the user has entered an incomplete command that can be completed by - entering more text (as opposed to a complete command or a syntax error). This - function *almost* always makes the same decision as the real interpreter main - loop. - - *source* is the source string; *filename* is the optional filename from which - source was read, defaulting to ``''``; and *symbol* is the optional - grammar start symbol, which should be either ``'single'`` (the default) or - ``'eval'``. - - Returns a code object (the same as ``compile(source, filename, symbol)``) if the - command is complete and valid; ``None`` if the command is incomplete; raises - :exc:`SyntaxError` if the command is complete and contains a syntax error, or - raises :exc:`OverflowError` or :exc:`ValueError` if the command contains an - invalid literal. - - -.. _interpreter-objects: - -Interactive Interpreter Objects -------------------------------- - - -.. method:: InteractiveInterpreter.runsource(source, filename="", symbol="single") - - Compile and run some source in the interpreter. Arguments are the same as for - :func:`compile_command`; the default for *filename* is ``''``, and for - *symbol* is ``'single'``. One several things can happen: - - * The input is incorrect; :func:`compile_command` raised an exception - (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be - printed by calling the :meth:`showsyntaxerror` method. :meth:`runsource` - returns ``False``. - - * The input is incomplete, and more input is required; :func:`compile_command` - returned ``None``. :meth:`runsource` returns ``True``. - - * The input is complete; :func:`compile_command` returned a code object. The - code is executed by calling the :meth:`runcode` (which also handles run-time - exceptions, except for :exc:`SystemExit`). :meth:`runsource` returns ``False``. - - The return value can be used to decide whether to use ``sys.ps1`` or ``sys.ps2`` - to prompt the next line. - - -.. method:: InteractiveInterpreter.runcode(code) - - Execute a code object. When an exception occurs, :meth:`showtraceback` is called - to display a traceback. All exceptions are caught except :exc:`SystemExit`, - which is allowed to propagate. - - A note about :exc:`KeyboardInterrupt`: this exception may occur elsewhere in - this code, and may not always be caught. The caller should be prepared to deal - with it. - - -.. method:: InteractiveInterpreter.showsyntaxerror(filename=None) - - Display the syntax error that just occurred. This does not display a stack - trace because there isn't one for syntax errors. If *filename* is given, it is - stuffed into the exception instead of the default filename provided by Python's - parser, because it always uses ``''`` when reading from a string. The - output is written by the :meth:`write` method. - - -.. method:: InteractiveInterpreter.showtraceback() - - Display the exception that just occurred. We remove the first stack item - because it is within the interpreter object implementation. The output is - written by the :meth:`write` method. - - .. versionchanged:: 3.5 The full chained traceback is displayed instead - of just the primary traceback. - - -.. method:: InteractiveInterpreter.write(data) - - Write a string to the standard error stream (``sys.stderr``). Derived classes - should override this to provide the appropriate output handling as needed. - - -.. _console-objects: - -Interactive Console Objects ---------------------------- - -The :class:`InteractiveConsole` class is a subclass of -:class:`InteractiveInterpreter`, and so offers all the methods of the -interpreter objects as well as the following additions. - - -.. method:: InteractiveConsole.interact(banner=None, exitmsg=None) - - Closely emulate the interactive Python console. The optional *banner* argument - specify the banner to print before the first interaction; by default it prints a - banner similar to the one printed by the standard Python interpreter, followed - by the class name of the console object in parentheses (so as not to confuse - this with the real interpreter -- since it's so close!). - - The optional *exitmsg* argument specifies an exit message printed when exiting. - Pass the empty string to suppress the exit message. If *exitmsg* is not given or - ``None``, a default message is printed. - - .. versionchanged:: 3.4 - To suppress printing any banner, pass an empty string. - - .. versionchanged:: 3.6 - Print an exit message when exiting. - - -.. method:: InteractiveConsole.push(line) - - Push a line of source text to the interpreter. The line should not have a - trailing newline; it may have internal newlines. The line is appended to a - buffer and the interpreter's :meth:`runsource` method is called with the - concatenated contents of the buffer as source. If this indicates that the - command was executed or invalid, the buffer is reset; otherwise, the command is - incomplete, and the buffer is left as it was after the line was appended. The - return value is ``True`` if more input is required, ``False`` if the line was - dealt with in some way (this is the same as :meth:`runsource`). - - -.. method:: InteractiveConsole.resetbuffer() - - Remove any unhandled source text from the input buffer. - - -.. method:: InteractiveConsole.raw_input(prompt="") - - Write a prompt and read a line. The returned line does not include the trailing - newline. When the user enters the EOF key sequence, :exc:`EOFError` is raised. - The base implementation reads from ``sys.stdin``; a subclass may replace this - with a different implementation. diff --git a/third_party/python/Doc/library/codecs.rst b/third_party/python/Doc/library/codecs.rst deleted file mode 100644 index 992135154..000000000 --- a/third_party/python/Doc/library/codecs.rst +++ /dev/null @@ -1,1502 +0,0 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= - -.. module:: codecs - :synopsis: Encode and decode data and streams. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/codecs.py` - -.. index:: - single: Unicode - single: Codecs - pair: Codecs; encode - pair: Codecs; decode - single: streams - pair: stackable; streams - --------------- - -This module defines base classes for standard Python codecs (encoders and -decoders) and provides access to the internal Python codec registry, which -manages the codec and error handling lookup process. Most standard codecs -are :term:`text encodings `, which encode text to bytes, -but there are also codecs provided that encode text to text, and bytes to -bytes. Custom codecs may encode and decode between arbitrary types, but some -module features are restricted to use specifically with -:term:`text encodings `, or with codecs that encode to -:class:`bytes`. - -The module defines the following functions for encoding and decoding with -any codec: - -.. function:: encode(obj, encoding='utf-8', errors='strict') - - Encodes *obj* using the codec registered for *encoding*. - - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that encoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeEncodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. - -.. function:: decode(obj, encoding='utf-8', errors='strict') - - Decodes *obj* using the codec registered for *encoding*. - - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that decoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. - -The full details for each codec can also be looked up directly: - -.. function:: lookup(encoding) - - Looks up the codec info in the Python codec registry and returns a - :class:`CodecInfo` object as defined below. - - Encodings are first looked up in the registry's cache. If not found, the list of - registered search functions is scanned. If no :class:`CodecInfo` object is - found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object - is stored in the cache and returned to the caller. - -.. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) - - Codec details when looking up the codec registry. The constructor - arguments are stored in attributes of the same name: - - - .. attribute:: name - - The name of the encoding. - - - .. attribute:: encode - decode - - The stateless encoding and decoding functions. These must be - functions or methods which have the same interface as - the :meth:`~Codec.encode` and :meth:`~Codec.decode` methods of Codec - instances (see :ref:`Codec Interface `). - The functions or methods are expected to work in a stateless mode. - - - .. attribute:: incrementalencoder - incrementaldecoder - - Incremental encoder and decoder classes or factory functions. - These have to provide the interface defined by the base classes - :class:`IncrementalEncoder` and :class:`IncrementalDecoder`, - respectively. Incremental codecs can maintain state. - - - .. attribute:: streamwriter - streamreader - - Stream writer and reader classes or factory functions. These have to - provide the interface defined by the base classes - :class:`StreamWriter` and :class:`StreamReader`, respectively. - Stream codecs can maintain state. - -To simplify access to the various codec components, the module provides -these additional functions which use :func:`lookup` for the codec lookup: - -.. function:: getencoder(encoding) - - Look up the codec for the given encoding and return its encoder function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getdecoder(encoding) - - Look up the codec for the given encoding and return its decoder function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getincrementalencoder(encoding) - - Look up the codec for the given encoding and return its incremental encoder - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found or the codec - doesn't support an incremental encoder. - - -.. function:: getincrementaldecoder(encoding) - - Look up the codec for the given encoding and return its incremental decoder - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found or the codec - doesn't support an incremental decoder. - - -.. function:: getreader(encoding) - - Look up the codec for the given encoding and return its :class:`StreamReader` - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getwriter(encoding) - - Look up the codec for the given encoding and return its :class:`StreamWriter` - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - -Custom codecs are made available by registering a suitable codec search -function: - -.. function:: register(search_function) - - Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters, and return a - :class:`CodecInfo` object. In case a search function cannot find - a given encoding, it should return ``None``. - - .. note:: - - Search function registration is not currently reversible, - which may cause problems in some cases, such as unit testing or - module reloading. - -While the builtin :func:`open` and the associated :mod:`io` module are the -recommended approach for working with encoded text files, this module -provides additional utility functions and classes that allow the use of a -wider range of codecs when working with binary files: - -.. function:: open(filename, mode='r', encoding=None, errors='strict', buffering=1) - - Open an encoded file using the given *mode* and return an instance of - :class:`StreamReaderWriter`, providing transparent encoding/decoding. - The default file mode is ``'r'``, meaning to open the file in read mode. - - .. note:: - - Underlying encoded files are always opened in binary mode. - No automatic conversion of ``'\n'`` is done on reading and writing. - The *mode* argument may be any binary mode acceptable to the built-in - :func:`open` function; the ``'b'`` is automatically added. - - *encoding* specifies the encoding which is to be used for the file. - Any encoding that encodes to and decodes from bytes is allowed, and - the data types supported by the file methods depend on the codec used. - - *errors* may be given to define the error handling. It defaults to ``'strict'`` - which causes a :exc:`ValueError` to be raised in case an encoding error occurs. - - *buffering* has the same meaning as for the built-in :func:`open` function. It - defaults to line buffered. - - -.. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict') - - Return a :class:`StreamRecoder` instance, a wrapped version of *file* - which provides transparent transcoding. The original file is closed - when the wrapped version is closed. - - Data written to the wrapped file is decoded according to the given - *data_encoding* and then written to the original file as bytes using - *file_encoding*. Bytes read from the original file are decoded - according to *file_encoding*, and the result is encoded - using *data_encoding*. - - If *file_encoding* is not given, it defaults to *data_encoding*. - - *errors* may be given to define the error handling. It defaults to - ``'strict'``, which causes :exc:`ValueError` to be raised in case an encoding - error occurs. - - -.. function:: iterencode(iterator, encoding, errors='strict', **kwargs) - - Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any - other keyword argument) is passed through to the incremental encoder. - - This function requires that the codec accept text :class:`str` objects - to encode. Therefore it does not support bytes-to-bytes encoders such as - ``base64_codec``. - - -.. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) - - Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any - other keyword argument) is passed through to the incremental decoder. - - This function requires that the codec accept :class:`bytes` objects - to decode. Therefore it does not support text-to-text encoders such as - ``rot_13``, although ``rot_13`` may be used equivalently with - :func:`iterencode`. - - -The module also provides the following constants which are useful for reading -and writing to platform dependent files: - - -.. data:: BOM - BOM_BE - BOM_LE - BOM_UTF8 - BOM_UTF16 - BOM_UTF16_BE - BOM_UTF16_LE - BOM_UTF32 - BOM_UTF32_BE - BOM_UTF32_LE - - These constants define various byte sequences, - being Unicode byte order marks (BOMs) for several encodings. They are - used in UTF-16 and UTF-32 data streams to indicate the byte order used, - and in UTF-8 as a Unicode signature. :const:`BOM_UTF16` is either - :const:`BOM_UTF16_BE` or :const:`BOM_UTF16_LE` depending on the platform's - native byte order, :const:`BOM` is an alias for :const:`BOM_UTF16`, - :const:`BOM_LE` for :const:`BOM_UTF16_LE` and :const:`BOM_BE` for - :const:`BOM_UTF16_BE`. The others represent the BOM in UTF-8 and UTF-32 - encodings. - - -.. _codec-base-classes: - -Codec Base Classes ------------------- - -The :mod:`codecs` module defines a set of base classes which define the -interfaces for working with codec objects, and can also be used as the basis -for custom codec implementations. - -Each codec has to define four interfaces to make it usable as codec in Python: -stateless encoder, stateless decoder, stream reader and stream writer. The -stream reader and writers typically reuse the stateless encoder/decoder to -implement the file protocols. Codec authors also need to define how the -codec will handle encoding and decoding errors. - - -.. _surrogateescape: -.. _error-handlers: - -Error Handlers -^^^^^^^^^^^^^^ - -To simplify and standardize error handling, -codecs may implement different error handling schemes by -accepting the *errors* string argument. The following string values are -defined and implemented by all standard Python codecs: - -.. tabularcolumns:: |l|L| - -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass); | -| | this is the default. Implemented in | -| | :func:`strict_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'ignore'`` | Ignore the malformed data and continue | -| | without further notice. Implemented in | -| | :func:`ignore_errors`. | -+-------------------------+-----------------------------------------------+ - -The following error handlers are only applicable to -:term:`text encodings `: - -.. index:: - single: ? (question mark); replacement character - single: \ (backslash); escape sequence - single: \x; escape sequence - single: \u; escape sequence - single: \U; escape sequence - single: \N; escape sequence - -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'replace'`` | Replace with a suitable replacement | -| | marker; Python will use the official | -| | ``U+FFFD`` REPLACEMENT CHARACTER for the | -| | built-in codecs on decoding, and '?' on | -| | encoding. Implemented in | -| | :func:`replace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | -| | reference (only for encoding). Implemented | -| | in :func:`xmlcharrefreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'backslashreplace'`` | Replace with backslashed escape sequences. | -| | Implemented in | -| | :func:`backslashreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences | -| | (only for encoding). Implemented in | -| | :func:`namereplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'surrogateescape'`` | On decoding, replace byte with individual | -| | surrogate code ranging from ``U+DC80`` to | -| | ``U+DCFF``. This code will then be turned | -| | back into the same byte when the | -| | ``'surrogateescape'`` error handler is used | -| | when encoding the data. (See :pep:`383` for | -| | more.) | -+-------------------------+-----------------------------------------------+ - -In addition, the following error handler is specific to the given codecs: - -+-------------------+------------------------+-------------------------------------------+ -| Value | Codecs | Meaning | -+===================+========================+===========================================+ -|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | -| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | -| | utf-32-be, utf-32-le | presence of surrogates as an error. | -+-------------------+------------------------+-------------------------------------------+ - -.. versionadded:: 3.1 - The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. - -.. versionchanged:: 3.4 - The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs. - -.. versionadded:: 3.5 - The ``'namereplace'`` error handler. - -.. versionchanged:: 3.5 - The ``'backslashreplace'`` error handlers now works with decoding and - translating. - -The set of allowed values can be extended by registering a new named error -handler: - -.. function:: register_error(name, error_handler) - - Register the error handling function *error_handler* under the name *name*. - The *error_handler* argument will be called during encoding and decoding - in case of an error, when *name* is specified as the errors parameter. - - For encoding, *error_handler* will be called with a :exc:`UnicodeEncodeError` - instance, which contains information about the location of the error. The - error handler must either raise this or a different exception, or return a - tuple with a replacement for the unencodable part of the input and a position - where encoding should continue. The replacement may be either :class:`str` or - :class:`bytes`. If the replacement is bytes, the encoder will simply copy - them into the output buffer. If the replacement is a string, the encoder will - encode the replacement. Encoding continues on original input at the - specified position. Negative position values will be treated as being - relative to the end of the input string. If the resulting position is out of - bound an :exc:`IndexError` will be raised. - - Decoding and translating works similarly, except :exc:`UnicodeDecodeError` or - :exc:`UnicodeTranslateError` will be passed to the handler and that the - replacement from the error handler will be put into the output directly. - - -Previously registered error handlers (including the standard error handlers) -can be looked up by name: - -.. function:: lookup_error(name) - - Return the error handler previously registered under the name *name*. - - Raises a :exc:`LookupError` in case the handler cannot be found. - -The following standard error handlers are also made available as module level -functions: - -.. function:: strict_errors(exception) - - Implements the ``'strict'`` error handling: each encoding or - decoding error raises a :exc:`UnicodeError`. - - -.. function:: replace_errors(exception) - - Implements the ``'replace'`` error handling (for :term:`text encodings - ` only): substitutes ``'?'`` for encoding errors - (to be encoded by the codec), and ``'\ufffd'`` (the Unicode replacement - character) for decoding errors. - - -.. function:: ignore_errors(exception) - - Implements the ``'ignore'`` error handling: malformed data is ignored and - encoding or decoding is continued without further notice. - - -.. function:: xmlcharrefreplace_errors(exception) - - Implements the ``'xmlcharrefreplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by an appropriate XML character reference. - - -.. function:: backslashreplace_errors(exception) - - Implements the ``'backslashreplace'`` error handling (for - :term:`text encodings ` only): malformed data is - replaced by a backslashed escape sequence. - -.. function:: namereplace_errors(exception) - - Implements the ``'namereplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by a ``\N{...}`` escape sequence. - - .. versionadded:: 3.5 - - -.. _codec-objects: - -Stateless Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The base :class:`Codec` class defines these methods which also define the -function interfaces of the stateless encoder and decoder: - - -.. method:: Codec.encode(input[, errors]) - - Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular - character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. - - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. - - The encoder must be able to handle zero length input and return an empty object - of the output object type in this situation. - - -.. method:: Codec.decode(input[, errors]) - - Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular - character set encoding to a string object. - - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only - buffer interface -- for example, buffer objects and memory mapped files. - - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. - - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. - - The decoder must be able to handle zero length input and return an empty object - of the output object type in this situation. - - -Incremental Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`IncrementalEncoder` and :class:`IncrementalDecoder` classes provide -the basic interface for incremental encoding and decoding. Encoding/decoding the -input isn't done with one call to the stateless encoder/decoder function, but -with multiple calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method of -the incremental encoder/decoder. The incremental encoder/decoder keeps track of -the encoding/decoding process during method calls. - -The joined output of calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method is -the same as if all the single inputs were joined into one, and this input was -encoded/decoded with the stateless encoder/decoder. - - -.. _incremental-encoder-objects: - -IncrementalEncoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`IncrementalEncoder` class is used for encoding an input in multiple -steps. It defines the following methods which every incremental encoder must -define in order to be compatible with the Python codec registry. - - -.. class:: IncrementalEncoder(errors='strict') - - Constructor for an :class:`IncrementalEncoder` instance. - - All incremental encoders must provide this constructor interface. They are free - to add additional keyword arguments, but only the ones defined here are used by - the Python codec registry. - - The :class:`IncrementalEncoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`IncrementalEncoder` - object. - - - .. method:: encode(object[, final]) - - Encodes *object* (taking the current state of the encoder into account) - and returns the resulting encoded object. If this is the last call to - :meth:`encode` *final* must be true (the default is false). - - - .. method:: reset() - - Reset the encoder to the initial state. The output is discarded: call - ``.encode(object, final=True)``, passing an empty byte or text string - if necessary, to reset the encoder and to get the output. - - - .. method:: getstate() - - Return the current state of the encoder which must be an integer. The - implementation should make sure that ``0`` is the most common - state. (States that are more complicated than integers can be converted - into an integer by marshaling/pickling the state and encoding the bytes - of the resulting string into an integer). - - - .. method:: setstate(state) - - Set the state of the encoder to *state*. *state* must be an encoder state - returned by :meth:`getstate`. - - -.. _incremental-decoder-objects: - -IncrementalDecoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`IncrementalDecoder` class is used for decoding an input in multiple -steps. It defines the following methods which every incremental decoder must -define in order to be compatible with the Python codec registry. - - -.. class:: IncrementalDecoder(errors='strict') - - Constructor for an :class:`IncrementalDecoder` instance. - - All incremental decoders must provide this constructor interface. They are free - to add additional keyword arguments, but only the ones defined here are used by - the Python codec registry. - - The :class:`IncrementalDecoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`IncrementalDecoder` - object. - - - .. method:: decode(object[, final]) - - Decodes *object* (taking the current state of the decoder into account) - and returns the resulting decoded object. If this is the last call to - :meth:`decode` *final* must be true (the default is false). If *final* is - true the decoder must decode the input completely and must flush all - buffers. If this isn't possible (e.g. because of incomplete byte sequences - at the end of the input) it must initiate error handling just like in the - stateless case (which might raise an exception). - - - .. method:: reset() - - Reset the decoder to the initial state. - - - .. method:: getstate() - - Return the current state of the decoder. This must be a tuple with two - items, the first must be the buffer containing the still undecoded - input. The second must be an integer and can be additional state - info. (The implementation should make sure that ``0`` is the most common - additional state info.) If this additional state info is ``0`` it must be - possible to set the decoder to the state which has no input buffered and - ``0`` as the additional state info, so that feeding the previously - buffered input to the decoder returns it to the previous state without - producing any output. (Additional state info that is more complicated than - integers can be converted into an integer by marshaling/pickling the info - and encoding the bytes of the resulting string into an integer.) - - - .. method:: setstate(state) - - Set the state of the encoder to *state*. *state* must be a decoder state - returned by :meth:`getstate`. - - -Stream Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - -The :class:`StreamWriter` and :class:`StreamReader` classes provide generic -working interfaces which can be used to implement new encoding submodules very -easily. See :mod:`encodings.utf_8` for an example of how this is done. - - -.. _stream-writer-objects: - -StreamWriter Objects -~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamWriter` class is a subclass of :class:`Codec` and defines the -following methods which every stream writer must define in order to be -compatible with the Python codec registry. - - -.. class:: StreamWriter(stream, errors='strict') - - Constructor for a :class:`StreamWriter` instance. - - All stream writers must provide this constructor interface. They are free to add - additional keyword arguments, but only the ones defined here are used by the - Python codec registry. - - The *stream* argument must be a file-like object open for writing - text or binary data, as appropriate for the specific codec. - - The :class:`StreamWriter` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`StreamWriter` object. - - .. method:: write(object) - - Writes the object's contents encoded to the stream. - - - .. method:: writelines(list) - - Writes the concatenated list of strings to the stream (possibly by reusing - the :meth:`write` method). The standard bytes-to-bytes codecs - do not support this method. - - - .. method:: reset() - - Flushes and resets the codec buffers used for keeping state. - - Calling this method should ensure that the data on the output is put into - a clean state that allows appending of new fresh data without having to - rescan the whole stream to recover state. - - -In addition to the above methods, the :class:`StreamWriter` must also inherit -all other methods and attributes from the underlying stream. - - -.. _stream-reader-objects: - -StreamReader Objects -~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamReader` class is a subclass of :class:`Codec` and defines the -following methods which every stream reader must define in order to be -compatible with the Python codec registry. - - -.. class:: StreamReader(stream, errors='strict') - - Constructor for a :class:`StreamReader` instance. - - All stream readers must provide this constructor interface. They are free to add - additional keyword arguments, but only the ones defined here are used by the - Python codec registry. - - The *stream* argument must be a file-like object open for reading - text or binary data, as appropriate for the specific codec. - - The :class:`StreamReader` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`StreamReader` object. - - The set of allowed values for the *errors* argument can be extended with - :func:`register_error`. - - - .. method:: read([size[, chars, [firstline]]]) - - Decodes data from the stream and returns the resulting object. - - The *chars* argument indicates the number of decoded - code points or bytes to return. The :func:`read` method will - never return more data than requested, but it might return less, - if there is not enough available. - - The *size* argument indicates the approximate maximum - number of encoded bytes or code points to read - for decoding. The decoder can modify this setting as - appropriate. The default value -1 indicates to read and decode as much as - possible. This parameter is intended to - prevent having to decode huge files in one step. - - The *firstline* flag indicates that - it would be sufficient to only return the first - line, if there are decoding errors on later lines. - - The method should use a greedy read strategy meaning that it should read - as much data as is allowed within the definition of the encoding and the - given size, e.g. if optional encoding endings or state markers are - available on the stream, these should be read too. - - - .. method:: readline([size[, keepends]]) - - Read one line from the input stream and return the decoded data. - - *size*, if given, is passed as size argument to the stream's - :meth:`read` method. - - If *keepends* is false line-endings will be stripped from the lines - returned. - - - .. method:: readlines([sizehint[, keepends]]) - - Read all lines available on the input stream and return them as a list of - lines. - - Line-endings are implemented using the codec's decoder method and are - included in the list entries if *keepends* is true. - - *sizehint*, if given, is passed as the *size* argument to the stream's - :meth:`read` method. - - - .. method:: reset() - - Resets the codec buffers used for keeping state. - - Note that no stream repositioning should take place. This method is - primarily intended to be able to recover from decoding errors. - - -In addition to the above methods, the :class:`StreamReader` must also inherit -all other methods and attributes from the underlying stream. - -.. _stream-reader-writer: - -StreamReaderWriter Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamReaderWriter` is a convenience class that allows wrapping -streams which work in both read and write modes. - -The design is such that one can use the factory functions returned by the -:func:`lookup` function to construct the instance. - - -.. class:: StreamReaderWriter(stream, Reader, Writer, errors='strict') - - Creates a :class:`StreamReaderWriter` instance. *stream* must be a file-like - object. *Reader* and *Writer* must be factory functions or classes providing the - :class:`StreamReader` and :class:`StreamWriter` interface resp. Error handling - is done in the same way as defined for the stream readers and writers. - -:class:`StreamReaderWriter` instances define the combined interfaces of -:class:`StreamReader` and :class:`StreamWriter` classes. They inherit all other -methods and attributes from the underlying stream. - - -.. _stream-recoder-objects: - -StreamRecoder Objects -~~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamRecoder` translates data from one encoding to another, -which is sometimes useful when dealing with different encoding environments. - -The design is such that one can use the factory functions returned by the -:func:`lookup` function to construct the instance. - - -.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') - - Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* - work on the backend — the data in *stream*. - - You can use these objects to do transparent transcodings from e.g. Latin-1 - to UTF-8 and back. - - The *stream* argument must be a file-like object. - - The *encode* and *decode* arguments must - adhere to the :class:`Codec` interface. *Reader* and - *Writer* must be factory functions or classes providing objects of the - :class:`StreamReader` and :class:`StreamWriter` interface respectively. - - Error handling is done in the same way as defined for the stream readers and - writers. - - -:class:`StreamRecoder` instances define the combined interfaces of -:class:`StreamReader` and :class:`StreamWriter` classes. They inherit all other -methods and attributes from the underlying stream. - - -.. _encodings-overview: - -Encodings and Unicode ---------------------- - -Strings are stored internally as sequences of code points in -range ``0x0``--``0x10FFFF``. (See :pep:`393` for -more details about the implementation.) -Once a string object is used outside of CPU and memory, endianness -and how these arrays are stored as bytes become an issue. As with other -codecs, serialising a string into a sequence of bytes is known as *encoding*, -and recreating the string from the sequence of bytes is known as *decoding*. -There are a variety of different text serialisation codecs, which are -collectivity referred to as :term:`text encodings `. - -The simplest text encoding (called ``'latin-1'`` or ``'iso-8859-1'``) maps -the code points 0--255 to the bytes ``0x0``--``0xff``, which means that a string -object that contains code points above ``U+00FF`` can't be encoded with this -codec. Doing so will raise a :exc:`UnicodeEncodeError` that looks -like the following (although the details of the error message may differ): -``UnicodeEncodeError: 'latin-1' codec can't encode character '\u1234' in -position 3: ordinal not in range(256)``. - -There's another group of encodings (the so called charmap encodings) that choose -a different subset of all Unicode code points and how these code points are -mapped to the bytes ``0x0``--``0xff``. To see how this is done simply open -e.g. :file:`encodings/cp1252.py` (which is an encoding that is used primarily on -Windows). There's a string constant with 256 characters that shows you which -character is mapped to which byte value. - -All of these encodings can only encode 256 of the 1114112 code points -defined in Unicode. A simple and straightforward way that can store each Unicode -code point, is to store each code point as four consecutive bytes. There are two -possibilities: store the bytes in big endian or in little endian order. These -two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in an ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. -Unfortunately the character ``U+FEFF`` had a second purpose as -a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow -a word to be split. It can e.g. be used to give hints to a ligature algorithm. -With Unicode 4.0 using ``U+FEFF`` as a ``ZERO WIDTH NO-BREAK SPACE`` has been -deprecated (with ``U+2060`` (``WORD JOINER``) assuming this role). Nevertheless -Unicode software still must be able to handle ``U+FEFF`` in both roles: as a BOM -it's a device to determine the storage layout of the encoded bytes, and vanishes -once the byte sequence has been decoded into a string; as a ``ZERO WIDTH -NO-BREAK SPACE`` it's a normal character that will be decoded like any other. - -There's another encoding that is able to encoding the full range of Unicode -characters: UTF-8. UTF-8 is an 8-bit encoding, which means there are no issues -with byte order in UTF-8. Each byte in a UTF-8 byte sequence consists of two -parts: marker bits (the most significant bits) and payload bits. The marker bits -are a sequence of zero to four ``1`` bits followed by a ``0`` bit. Unicode characters are -encoded like this (with x being payload bits, which when concatenated give the -Unicode character): - -+-----------------------------------+----------------------------------------------+ -| Range | Encoding | -+===================================+==============================================+ -| ``U-00000000`` ... ``U-0000007F`` | 0xxxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00000080`` ... ``U-000007FF`` | 110xxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00000800`` ... ``U-0000FFFF`` | 1110xxxx 10xxxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00010000`` ... ``U-0010FFFF`` | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ - -The least significant bit of the Unicode character is the rightmost x bit. - -As UTF-8 is an 8-bit encoding no BOM is required and any ``U+FEFF`` character in -the decoded string (even if it's the first character) is treated as a ``ZERO -WIDTH NO-BREAK SPACE``. - -Without external information it's impossible to reliably determine which -encoding was used for encoding a string. Each charmap encoding can -decode any random byte sequence. However that's not possible with UTF-8, as -UTF-8 byte sequences have a structure that doesn't allow arbitrary byte -sequences. To increase the reliability with which a UTF-8 encoding can be -detected, Microsoft invented a variant of UTF-8 (that Python 2.5 calls -``"utf-8-sig"``) for its Notepad program: Before any of the Unicode characters -is written to the file, a UTF-8 encoded BOM (which looks like this as a byte -sequence: ``0xef``, ``0xbb``, ``0xbf``) is written. As it's rather improbable -that any charmap encoded file starts with these byte values (which would e.g. -map to - - | LATIN SMALL LETTER I WITH DIAERESIS - | RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - | INVERTED QUESTION MARK - -in iso-8859-1), this increases the probability that a ``utf-8-sig`` encoding can be -correctly guessed from the byte sequence. So here the BOM is not used to be able -to determine the byte order used for generating the byte sequence, but as a -signature that helps in guessing the encoding. On encoding the utf-8-sig codec -will write ``0xef``, ``0xbb``, ``0xbf`` as the first three bytes to the file. On -decoding ``utf-8-sig`` will skip those three bytes if they appear as the first -three bytes in the file. In UTF-8, the use of the BOM is discouraged and -should generally be avoided. - - -.. _standard-encodings: - -Standard Encodings ------------------- - -Python comes with a number of codecs built-in, either implemented as C functions -or with dictionaries as mapping tables. The following table lists the codecs by -name, together with a few common aliases, and the languages for which the -encoding is likely used. Neither the list of aliases nor the list of languages -is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. - -.. impl-detail:: - - Some common encodings can bypass the codecs lookup machinery to - improve performance. These optimization opportunities are only - recognized by CPython for a limited set of (case insensitive) - aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs - (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and - the same using underscores instead of dashes. Using alternative - aliases for these encodings may result in slower execution. - - .. versionchanged:: 3.6 - Optimization opportunity recognized for us-ascii. - -Many of the character sets support the same languages. They vary in individual -characters (e.g. whether the EURO SIGN is supported or not), and in the -assignment of characters to code positions. For the European languages in -particular, the following variants typically exist: - -* an ISO 8859 codeset - -* a Microsoft Windows code page, which is typically derived from an 8859 codeset, - but replaces control characters with additional graphic characters - -* an IBM EBCDIC code page - -* an IBM PC code page, which is ASCII compatible - -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| - -+-----------------+--------------------------------+--------------------------------+ -| Codec | Aliases | Languages | -+=================+================================+================================+ -| ascii | 646, us-ascii | English | -+-----------------+--------------------------------+--------------------------------+ -| big5 | big5-tw, csbig5 | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| big5hkscs | big5-hkscs, hkscs | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| cp037 | IBM037, IBM039 | English | -+-----------------+--------------------------------+--------------------------------+ -| cp273 | 273, IBM273, csIBM273 | German | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ -| cp424 | EBCDIC-CP-HE, IBM424 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp437 | 437, IBM437 | English | -+-----------------+--------------------------------+--------------------------------+ -| cp500 | EBCDIC-CP-BE, EBCDIC-CP-CH, | Western Europe | -| | IBM500 | | -+-----------------+--------------------------------+--------------------------------+ -| cp720 | | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp737 | | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp775 | IBM775 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| cp850 | 850, IBM850 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp852 | 852, IBM852 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp855 | 855, IBM855 | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| cp856 | | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp857 | 857, IBM857 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp860 | 860, IBM860 | Portuguese | -+-----------------+--------------------------------+--------------------------------+ -| cp861 | 861, CP-IS, IBM861 | Icelandic | -+-----------------+--------------------------------+--------------------------------+ -| cp862 | 862, IBM862 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp863 | 863, IBM863 | Canadian | -+-----------------+--------------------------------+--------------------------------+ -| cp864 | IBM864 | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp865 | 865, IBM865 | Danish, Norwegian | -+-----------------+--------------------------------+--------------------------------+ -| cp866 | 866, IBM866 | Russian | -+-----------------+--------------------------------+--------------------------------+ -| cp869 | 869, CP-GR, IBM869 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp874 | | Thai | -+-----------------+--------------------------------+--------------------------------+ -| cp875 | | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| cp949 | 949, ms949, uhc | Korean | -+-----------------+--------------------------------+--------------------------------+ -| cp950 | 950, ms950 | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| cp1006 | | Urdu | -+-----------------+--------------------------------+--------------------------------+ -| cp1026 | ibm1026 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp1125 | 1125, ibm1125, cp866u, ruscii | Ukrainian | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1250 | windows-1250 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1251 | windows-1251 | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| cp1252 | windows-1252 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1253 | windows-1253 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp1254 | windows-1254 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp1255 | windows-1255 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp1256 | windows-1256 | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp1257 | windows-1257 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| cp1258 | windows-1258 | Vietnamese | -+-----------------+--------------------------------+--------------------------------+ -| cp65001 | | Windows only: Windows UTF-8 | -| | | (``CP_UTF8``) | -| | | | -| | | .. versionadded:: 3.3 | -+-----------------+--------------------------------+--------------------------------+ -| euc_jp | eucjp, ujis, u-jis | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_jis_2004 | jisx0213, eucjis2004 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_jisx0213 | eucjisx0213 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_kr | euckr, korean, ksc5601, | Korean | -| | ks_c-5601, ks_c-5601-1987, | | -| | ksx1001, ks_x-1001 | | -+-----------------+--------------------------------+--------------------------------+ -| gb2312 | chinese, csiso58gb231280, | Simplified Chinese | -| | euc-cn, euccn, eucgb2312-cn, | | -| | gb2312-1980, gb2312-80, | | -| | iso-ir-58 | | -+-----------------+--------------------------------+--------------------------------+ -| gbk | 936, cp936, ms936 | Unified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| gb18030 | gb18030-2000 | Unified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| hz | hzgb, hz-gb, hz-gb-2312 | Simplified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp | csiso2022jp, iso2022jp, | Japanese | -| | iso-2022-jp | | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_1 | iso2022jp-1, iso-2022-jp-1 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_2 | iso2022jp-2, iso-2022-jp-2 | Japanese, Korean, Simplified | -| | | Chinese, Western Europe, Greek | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_2004 | iso2022jp-2004, | Japanese | -| | iso-2022-jp-2004 | | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_3 | iso2022jp-3, iso-2022-jp-3 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_ext | iso2022jp-ext, iso-2022-jp-ext | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_kr | csiso2022kr, iso2022kr, | Korean | -| | iso-2022-kr | | -+-----------------+--------------------------------+--------------------------------+ -| latin_1 | iso-8859-1, iso8859-1, 8859, | West Europe | -| | cp819, latin, latin1, L1 | | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_2 | iso-8859-2, latin2, L2 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_5 | iso-8859-5, cyrillic | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_6 | iso-8859-6, arabic | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_7 | iso-8859-7, greek, greek8 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_8 | iso-8859-8, hebrew | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_9 | iso-8859-9, latin5, L5 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_10 | iso-8859-10, latin6, L6 | Nordic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_11 | iso-8859-11, thai | Thai languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_13 | iso-8859-13, latin7, L7 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_14 | iso-8859-14, latin8, L8 | Celtic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_15 | iso-8859-15, latin9, L9 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_16 | iso-8859-16, latin10, L10 | South-Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| johab | cp1361, ms1361 | Korean | -+-----------------+--------------------------------+--------------------------------+ -| koi8_r | | Russian | -+-----------------+--------------------------------+--------------------------------+ -| koi8_t | | Tajik | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ -| koi8_u | | Ukrainian | -+-----------------+--------------------------------+--------------------------------+ -| kz1048 | kz_1048, strk1048_2002, rk1048 | Kazakh | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ -| mac_cyrillic | maccyrillic | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| mac_greek | macgreek | Greek | -+-----------------+--------------------------------+--------------------------------+ -| mac_iceland | maciceland | Icelandic | -+-----------------+--------------------------------+--------------------------------+ -| mac_latin2 | maclatin2, maccentraleurope | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| mac_roman | macroman, macintosh | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| mac_turkish | macturkish | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| ptcp154 | csptcp154, pt154, cp154, | Kazakh | -| | cyrillic-asian | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jis | csshiftjis, shiftjis, sjis, | Japanese | -| | s_jis | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jis_2004 | shiftjis2004, sjis_2004, | Japanese | -| | sjis2004 | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jisx0213 | shiftjisx0213, sjisx0213, | Japanese | -| | s_jisx0213 | | -+-----------------+--------------------------------+--------------------------------+ -| utf_32 | U32, utf32 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_32_be | UTF-32BE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_32_le | UTF-32LE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16 | U16, utf16 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16_be | UTF-16BE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16_le | UTF-16LE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_7 | U7, unicode-1-1-utf-7 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_8 | U8, UTF, utf8 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | -+-----------------+--------------------------------+--------------------------------+ - -.. versionchanged:: 3.4 - The utf-16\* and utf-32\* encoders no longer allow surrogate code points - (``U+D800``--``U+DFFF``) to be encoded. - The utf-32\* decoders no longer decode - byte sequences that correspond to surrogate code points. - - -Python Specific Encodings -------------------------- - -A number of predefined codecs are specific to Python, so their codec names have -no meaning outside Python. These are listed in the tables below based on the -expected input and output types (note that while text encodings are the most -common use case for codecs, the underlying codec infrastructure supports -arbitrary data transforms rather than just text encodings). For asymmetric -codecs, the stated purpose describes the encoding direction. - -Text Encodings -^^^^^^^^^^^^^^ - -The following codecs provide :class:`str` to :class:`bytes` encoding and -:term:`bytes-like object` to :class:`str` decoding, similar to the Unicode text -encodings. - -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| - -+--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | -+====================+=========+===========================+ -| idna | | Implements :rfc:`3490`, | -| | | see also | -| | | :mod:`encodings.idna`. | -| | | Only ``errors='strict'`` | -| | | is supported. | -+--------------------+---------+---------------------------+ -| mbcs | ansi, | Windows only: Encode | -| | dbcs | operand according to the | -| | | ANSI codepage (CP_ACP) | -+--------------------+---------+---------------------------+ -| oem | | Windows only: Encode | -| | | operand according to the | -| | | OEM codepage (CP_OEMCP) | -| | | | -| | | .. versionadded:: 3.6 | -+--------------------+---------+---------------------------+ -| palmos | | Encoding of PalmOS 3.5 | -+--------------------+---------+---------------------------+ -| punycode | | Implements :rfc:`3492`. | -| | | Stateful codecs are not | -| | | supported. | -+--------------------+---------+---------------------------+ -| raw_unicode_escape | | Latin-1 encoding with | -| | | ``\uXXXX`` and | -| | | ``\UXXXXXXXX`` for other | -| | | code points. Existing | -| | | backslashes are not | -| | | escaped in any way. | -| | | It is used in the Python | -| | | pickle protocol. | -+--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | -| | | all conversions, even | -| | | empty strings. The error | -| | | handler is ignored. | -+--------------------+---------+---------------------------+ -| unicode_escape | | Encoding suitable as the | -| | | contents of a Unicode | -| | | literal in ASCII-encoded | -| | | Python source code, | -| | | except that quotes are | -| | | not escaped. Decodes from | -| | | Latin-1 source code. | -| | | Beware that Python source | -| | | code actually uses UTF-8 | -| | | by default. | -+--------------------+---------+---------------------------+ -| unicode_internal | | Return the internal | -| | | representation of the | -| | | operand. Stateful codecs | -| | | are not supported. | -| | | | -| | | .. deprecated:: 3.3 | -| | | This representation is | -| | | obsoleted by | -| | | :pep:`393`. | -+--------------------+---------+---------------------------+ - -.. _binary-transforms: - -Binary Transforms -^^^^^^^^^^^^^^^^^ - -The following codecs provide binary transforms: :term:`bytes-like object` -to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` -(which only produces :class:`str` output). - - -.. tabularcolumns:: |l|L|L|L| - -+----------------------+------------------+------------------------------+------------------------------+ -| Codec | Aliases | Purpose | Encoder / decoder | -+======================+==================+==============================+==============================+ -| base64_codec [#b64]_ | base64, base_64 | Convert operand to multiline | :meth:`base64.encodebytes` / | -| | | MIME base64 (the result | :meth:`base64.decodebytes` | -| | | always includes a trailing | | -| | | ``'\n'``) | | -| | | | | -| | | .. versionchanged:: 3.4 | | -| | | accepts any | | -| | | :term:`bytes-like object` | | -| | | as input for encoding and | | -| | | decoding | | -+----------------------+------------------+------------------------------+------------------------------+ -| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / | -| | | using bz2 | :meth:`bz2.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ -| hex_codec | hex | Convert operand to | :meth:`binascii.b2a_hex` / | -| | | hexadecimal | :meth:`binascii.a2b_hex` | -| | | representation, with two | | -| | | digits per byte | | -+----------------------+------------------+------------------------------+------------------------------+ -| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encode` with | -| | quotedprintable, | quoted printable | ``quotetabs=True`` / | -| | quoted_printable | | :meth:`quopri.decode` | -+----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode | :meth:`uu.decode` | -+----------------------+------------------+------------------------------+------------------------------+ -| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / | -| | | using gzip | :meth:`zlib.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ - -.. [#b64] In addition to :term:`bytes-like objects `, - ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for - decoding - -.. versionadded:: 3.2 - Restoration of the binary transforms. - -.. versionchanged:: 3.4 - Restoration of the aliases for the binary transforms. - - -.. _text-transforms: - -Text Transforms -^^^^^^^^^^^^^^^ - -The following codec provides a text transform: a :class:`str` to :class:`str` -mapping. It is not supported by :meth:`str.encode` (which only produces -:class:`bytes` output). - -.. tabularcolumns:: |l|l|L| - -+--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | -+====================+=========+===========================+ -| rot_13 | rot13 | Returns the Caesar-cypher | -| | | encryption of the operand | -+--------------------+---------+---------------------------+ - -.. versionadded:: 3.2 - Restoration of the ``rot_13`` text transform. - -.. versionchanged:: 3.4 - Restoration of the ``rot13`` alias. - - -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- - -.. module:: encodings.idna - :synopsis: Internationalized Domain Names implementation -.. moduleauthor:: Martin v. Löwis - -This module implements :rfc:`3490` (Internationalized Domain Names in -Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for -Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding -and :mod:`stringprep`. - -These RFCs together define a protocol to support non-ASCII characters in domain -names. A domain name containing non-ASCII characters (such as -``www.Alliancefrançaise.nu``) is converted into an ASCII-compatible encoding -(ACE, such as ``www.xn--alliancefranaise-npb.nu``). The ACE form of the domain -name is then used in all places where arbitrary characters are not allowed by -the protocol, such as DNS queries, HTTP :mailheader:`Host` fields, and so -on. This conversion is carried out in the application; if possible invisible to -the user: The application should transparently convert Unicode domain labels to -IDNA on the wire, and convert back ACE labels to Unicode before presenting them -to the user. - -Python supports this conversion in several ways: the ``idna`` codec performs -conversion between Unicode and ACE, separating an input string into labels -based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>` -and converting each label to ACE as required, and conversely separating an input -byte string into labels based on the ``.`` separator and converting any ACE -labels found into unicode. Furthermore, the :mod:`socket` module -transparently converts Unicode host names to ACE, so that applications need not -be concerned about converting host names themselves when they pass them to the -socket module. On top of that, modules that have host names as function -parameters, such as :mod:`http.client` and :mod:`ftplib`, accept Unicode host -names (:mod:`http.client` then also transparently sends an IDNA hostname in the -:mailheader:`Host` field if it sends that field at all). - -When receiving host names from the wire (such as in reverse name lookup), no -automatic conversion to Unicode is performed: Applications wishing to present -such host names to the user should decode them to Unicode. - -The module :mod:`encodings.idna` also implements the nameprep procedure, which -performs certain normalizations on host names, to achieve case-insensitivity of -international domain names, and to unify similar characters. The nameprep -functions can be used directly if desired. - - -.. function:: nameprep(label) - - Return the nameprepped version of *label*. The implementation currently assumes - query strings, so ``AllowUnassigned`` is true. - - -.. function:: ToASCII(label) - - Convert a label to ASCII, as specified in :rfc:`3490`. ``UseSTD3ASCIIRules`` is - assumed to be false. - - -.. function:: ToUnicode(label) - - Convert a label to Unicode, as specified in :rfc:`3490`. - - -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ - -.. module:: encodings.mbcs - :synopsis: Windows ANSI codepage - -Encode operand according to the ANSI codepage (CP_ACP). - -Availability: Windows only. - -.. versionchanged:: 3.3 - Support any error handler. - -.. versionchanged:: 3.2 - Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used - to encode, and ``'ignore'`` to decode. - - -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- - -.. module:: encodings.utf_8_sig - :synopsis: UTF-8 codec with BOM signature -.. moduleauthor:: Walter Dörwald - -This module implements a variant of the UTF-8 codec: On encoding a UTF-8 encoded -BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this -is only done once (on the first write to the byte stream). For decoding an -optional UTF-8 encoded BOM at the start of the data will be skipped. diff --git a/third_party/python/Doc/library/codeop.rst b/third_party/python/Doc/library/codeop.rst deleted file mode 100644 index a52d2c62c..000000000 --- a/third_party/python/Doc/library/codeop.rst +++ /dev/null @@ -1,72 +0,0 @@ -:mod:`codeop` --- Compile Python code -===================================== - -.. module:: codeop - :synopsis: Compile (possibly incomplete) Python code. - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Michael Hudson - -**Source code:** :source:`Lib/codeop.py` - --------------- - -The :mod:`codeop` module provides utilities upon which the Python -read-eval-print loop can be emulated, as is done in the :mod:`code` module. As -a result, you probably don't want to use the module directly; if you want to -include such a loop in your program you probably want to use the :mod:`code` -module instead. - -There are two parts to this job: - -#. Being able to tell if a line of input completes a Python statement: in - short, telling whether to print '``>>>``' or '``...``' next. - -#. Remembering which future statements the user has entered, so subsequent - input can be compiled with these in effect. - -The :mod:`codeop` module provides a way of doing each of these things, and a way -of doing them both. - -To do just the former: - -.. function:: compile_command(source, filename="", symbol="single") - - Tries to compile *source*, which should be a string of Python code and return a - code object if *source* is valid Python code. In that case, the filename - attribute of the code object will be *filename*, which defaults to - ``''``. Returns ``None`` if *source* is *not* valid Python code, but is a - prefix of valid Python code. - - If there is a problem with *source*, an exception will be raised. - :exc:`SyntaxError` is raised if there is invalid Python syntax, and - :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. - - The *symbol* argument determines whether *source* is compiled as a statement - (``'single'``, the default) or as an :term:`expression` (``'eval'``). Any - other value will cause :exc:`ValueError` to be raised. - - .. note:: - - It is possible (but not likely) that the parser stops parsing with a - successful outcome before reaching the end of the source; in this case, - trailing symbols may be ignored instead of causing an error. For example, - a backslash followed by two newlines may be followed by arbitrary garbage. - This will be fixed once the API for the parser is better. - - -.. class:: Compile() - - Instances of this class have :meth:`__call__` methods identical in signature to - the built-in function :func:`compile`, but with the difference that if the - instance compiles program text containing a :mod:`__future__` statement, the - instance 'remembers' and compiles all subsequent program texts with the - statement in force. - - -.. class:: CommandCompiler() - - Instances of this class have :meth:`__call__` methods identical in signature to - :func:`compile_command`; the difference is that if the instance compiles program - text containing a ``__future__`` statement, the instance 'remembers' and - compiles all subsequent program texts with the statement in force. diff --git a/third_party/python/Doc/library/collections.abc.rst b/third_party/python/Doc/library/collections.abc.rst deleted file mode 100644 index 601545320..000000000 --- a/third_party/python/Doc/library/collections.abc.rst +++ /dev/null @@ -1,305 +0,0 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== - -.. module:: collections.abc - :synopsis: Abstract base classes for containers - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -.. versionadded:: 3.3 - Formerly, this module was part of the :mod:`collections` module. - -**Source code:** :source:`Lib/_collections_abc.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module provides :term:`abstract base classes ` that -can be used to test whether a class provides a particular interface; for -example, whether it is hashable or whether it is a mapping. - - -.. _collections-abstract-base-classes: - -Collections Abstract Base Classes ---------------------------------- - -The collections module offers the following :term:`ABCs `: - -.. tabularcolumns:: |l|L|L|L| - -========================== ====================== ======================= ==================================================== -ABC Inherits from Abstract Methods Mixin Methods -========================== ====================== ======================= ==================================================== -:class:`Container` ``__contains__`` -:class:`Hashable` ``__hash__`` -:class:`Iterable` ``__iter__`` -:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` -:class:`Reversible` :class:`Iterable` ``__reversed__`` -:class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` -:class:`Sized` ``__len__`` -:class:`Callable` ``__call__`` -:class:`Collection` :class:`Sized`, ``__contains__``, - :class:`Iterable`, ``__iter__``, - :class:`Container` ``__len__`` - -:class:`Sequence` :class:`Reversible`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Collection` ``__len__`` ``index``, and ``count`` - -:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` - ``__len__``, - ``insert`` - -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - -:class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, - ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` - -:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and - ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, - ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` - ``add``, - ``discard`` - -:class:`Mapping` :class:`Collection` ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, - ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` - ``__len__`` - -:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and - ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, - ``__delitem__``, and ``setdefault`` - ``__iter__``, - ``__len__`` - - -:class:`MappingView` :class:`Sized` ``__len__`` -:class:`ItemsView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`KeysView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__`` -:class:`Awaitable` ``__await__`` -:class:`Coroutine` :class:`Awaitable` ``send``, ``throw`` ``close`` -:class:`AsyncIterable` ``__aiter__`` -:class:`AsyncIterator` :class:`AsyncIterable` ``__anext__`` ``__aiter__`` -:class:`AsyncGenerator` :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` -========================== ====================== ======================= ==================================================== - - -.. class:: Container - Hashable - Sized - Callable - - ABCs for classes that provide respectively the methods :meth:`__contains__`, - :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`. - -.. class:: Iterable - - ABC for classes that provide the :meth:`__iter__` method. - - Checking ``isinstance(obj, Iterable)`` detects classes that are registered - as :class:`Iterable` or that have an :meth:`__iter__` method, but it does - not detect classes that iterate with the :meth:`__getitem__` method. - The only reliable way to determine whether an object is :term:`iterable` - is to call ``iter(obj)``. - -.. class:: Collection - - ABC for sized iterable container classes. - - .. versionadded:: 3.6 - -.. class:: Iterator - - ABC for classes that provide the :meth:`~iterator.__iter__` and - :meth:`~iterator.__next__` methods. See also the definition of - :term:`iterator`. - -.. class:: Reversible - - ABC for iterable classes that also provide the :meth:`__reversed__` - method. - - .. versionadded:: 3.6 - -.. class:: Generator - - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, - :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. - - .. versionadded:: 3.5 - -.. class:: Sequence - MutableSequence - ByteString - - ABCs for read-only and mutable :term:`sequences `. - - Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`__getitem__` method. - Consequently, if :meth:`__getitem__` is implemented with constant - access speed, the mixin methods will have linear performance; - however, if the underlying method is linear (as it would be with a - linked list), the mixins will have quadratic performance and will - likely need to be overridden. - - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. - -.. class:: Set - MutableSet - - ABCs for read-only and mutable sets. - -.. class:: Mapping - MutableMapping - - ABCs for read-only and mutable :term:`mappings `. - -.. class:: MappingView - ItemsView - KeysView - ValuesView - - ABCs for mapping, items, keys, and values :term:`views `. - -.. class:: Awaitable - - ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. - - :term:`Coroutine` objects and instances of the - :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: Coroutine - - ABC for coroutine compatible classes. These implement the - following methods, defined in :ref:`coroutine-objects`: - :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and - :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: AsyncIterable - - ABC for classes that provide ``__aiter__`` method. See also the - definition of :term:`asynchronous iterable`. - - .. versionadded:: 3.5 - -.. class:: AsyncIterator - - ABC for classes that provide ``__aiter__`` and ``__anext__`` - methods. See also the definition of :term:`asynchronous iterator`. - - .. versionadded:: 3.5 - -.. class:: AsyncGenerator - - ABC for asynchronous generator classes that implement the protocol - defined in :pep:`525` and :pep:`492`. - - .. versionadded:: 3.6 - - -These ABCs allow us to ask classes or instances if they provide -particular functionality, for example:: - - size = None - if isinstance(myvar, collections.abc.Sized): - size = len(myvar) - -Several of the ABCs are also useful as mixins that make it easier to develop -classes supporting container APIs. For example, to write a class supporting -the full :class:`Set` API, it is only necessary to supply the three underlying -abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. -The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: - - class ListBasedSet(collections.abc.Set): - ''' Alternate set implementation favoring space over speed - and not requiring the set elements to be hashable. ''' - def __init__(self, iterable): - self.elements = lst = [] - for value in iterable: - if value not in lst: - lst.append(value) - - def __iter__(self): - return iter(self.elements) - - def __contains__(self, value): - return value in self.elements - - def __len__(self): - return len(self.elements) - - s1 = ListBasedSet('abcdef') - s2 = ListBasedSet('defghi') - overlap = s1 & s2 # The __and__() method is supported automatically - -Notes on using :class:`Set` and :class:`MutableSet` as a mixin: - -(1) - Since some set operations create new sets, the default mixin methods need - a way to create new instances from an iterable. The class constructor is - assumed to have a signature in the form ``ClassName(iterable)``. - That assumption is factored-out to an internal classmethod called - :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. - If the :class:`Set` mixin is being used in a class with a different - constructor signature, you will need to override :meth:`_from_iterable` - with a classmethod that can construct new instances from - an iterable argument. - -(2) - To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, - then the other operations will automatically follow suit. - -(3) - The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value - for the set; however, :meth:`__hash__` is not defined because not all sets - are hashable or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define - ``__hash__ = Set._hash``. - -.. seealso:: - - * `OrderedSet recipe `_ for an - example built on :class:`MutableSet`. - - * For more about ABCs, see the :mod:`abc` module and :pep:`3119`. diff --git a/third_party/python/Doc/library/collections.rst b/third_party/python/Doc/library/collections.rst deleted file mode 100644 index 25687348d..000000000 --- a/third_party/python/Doc/library/collections.rst +++ /dev/null @@ -1,1184 +0,0 @@ -:mod:`collections` --- Container datatypes -========================================== - -.. module:: collections - :synopsis: Container datatypes - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/collections/__init__.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module implements specialized container datatypes providing alternatives to -Python's general purpose built-in containers, :class:`dict`, :class:`list`, -:class:`set`, and :class:`tuple`. - -===================== ==================================================================== -:func:`namedtuple` factory function for creating tuple subclasses with named fields -:class:`deque` list-like container with fast appends and pops on either end -:class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects -:class:`OrderedDict` dict subclass that remembers the order entries were added -:class:`defaultdict` dict subclass that calls a factory function to supply missing values -:class:`UserDict` wrapper around dictionary objects for easier dict subclassing -:class:`UserList` wrapper around list objects for easier list subclassing -:class:`UserString` wrapper around string objects for easier string subclassing -===================== ==================================================================== - -.. versionchanged:: 3.3 - Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. - For backwards compatibility, they continue to be visible in this module - as well. - - -:class:`ChainMap` objects -------------------------- - -.. versionadded:: 3.3 - -A :class:`ChainMap` class is provided for quickly linking a number of mappings -so they can be treated as a single unit. It is often much faster than creating -a new dictionary and running multiple :meth:`~dict.update` calls. - -The class can be used to simulate nested scopes and is useful in templating. - -.. class:: ChainMap(*maps) - - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. - - The underlying mappings are stored in a list. That list is public and can - be accessed or updated using the *maps* attribute. There is no other state. - - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. - - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. - - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: - - .. attribute:: maps - - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. - - .. method:: new_child(m=None) - - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. - - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. - - .. attribute:: parents - - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. - - -.. seealso:: - - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. - - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :meth:`~collections.ChainMap.parents` property. - - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. - - * A `greatly simplified read-only version of Chainmap - `_. - - -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with chained maps. - - -Example of simulating Python's internal lookup chain:: - - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k:v for k, v in vars(namespace).items() if v} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] # Get first key in the chain of contexts - d['x'] = 1 # Set value in current context - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - >>> d # display result - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) - - -:class:`Counter` objects ------------------------- - -A counter tool is provided to support convenient and rapid tallies. -For example:: - - >>> # Tally occurrences of words in a list - >>> cnt = Counter() - >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: - ... cnt[word] += 1 - >>> cnt - Counter({'blue': 3, 'red': 2, 'green': 1}) - - >>> # Find the ten most common words in Hamlet - >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) - >>> Counter(words).most_common(10) - [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), - ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] - -.. class:: Counter([iterable-or-mapping]) - - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. - It is an unordered collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. - - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping - >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: - - >>> c = Counter(['eggs', 'ham']) - >>> c['bacon'] # count of a missing element is zero - 0 - - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: - - >>> c['sausage'] = 0 # counter entry with a zero count - >>> del c['sausage'] # del actually removes the entry - - .. versionadded:: 3.1 - - - Counter objects support three methods beyond those available for all - dictionaries: - - .. method:: elements() - - Return an iterator over elements repeating each as many times as its - count. Elements are returned in arbitrary order. If an element's count - is less than one, :meth:`elements` will ignore it. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> sorted(c.elements()) - ['a', 'a', 'a', 'a', 'b', 'b'] - - .. method:: most_common([n]) - - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :func:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered arbitrarily: - - >>> Counter('abracadabra').most_common(3) # doctest: +SKIP - [('a', 5), ('r', 2), ('b', 2)] - - .. method:: subtract([iterable-or-mapping]) - - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> d = Counter(a=1, b=2, c=3, d=4) - >>> c.subtract(d) - >>> c - Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - - .. versionadded:: 3.2 - - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. - - .. method:: fromkeys(iterable) - - This class method is not implemented for :class:`Counter` objects. - - .. method:: update([iterable-or-mapping]) - - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. - -Common patterns for working with :class:`Counter` objects:: - - sum(c.values()) # total of all counts - c.clear() # reset all counts - list(c) # list unique elements - set(c) # convert to a set - dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs - Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements - +c # remove zero and negative counts - -Several mathematical operations are provided for combining :class:`Counter` -objects to produce multisets (counters that have counts greater than zero). -Addition and subtraction combine counters by adding or subtracting the counts -of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. - - >>> c = Counter(a=3, b=1) - >>> d = Counter(a=1, b=2) - >>> c + d # add two counters together: c[x] + d[x] - Counter({'a': 4, 'b': 3}) - >>> c - d # subtract (keeping only positive counts) - Counter({'a': 2}) - >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP - Counter({'a': 1, 'b': 1}) - >>> c | d # union: max(c[x], d[x]) - Counter({'a': 3, 'b': 2}) - -Unary addition and subtraction are shortcuts for adding an empty counter -or subtracting from an empty counter. - - >>> c = Counter(a=2, b=-4) - >>> +c - Counter({'a': 2}) - >>> -c - Counter({'b': 4}) - -.. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. - -.. note:: - - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. - - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. - - * The :meth:`most_common` method requires only that the values be orderable. - - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`update` and :meth:`subtract` which allow negative and zero values - for both inputs and outputs. - - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. - - * The :meth:`elements` method requires integer counts. It ignores zero and - negative counts. - -.. seealso:: - - * `Bag class `_ - in Smalltalk. - - * Wikipedia entry for `Multisets `_. - - * `C++ multisets `_ - tutorial with examples. - - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. - - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`: - - map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC - - -:class:`deque` objects ----------------------- - -.. class:: deque([iterable, [maxlen]]) - - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. - - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. - - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. - - - If *maxlen* is not specified or is ``None``, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. - - - Deque objects support the following methods: - - .. method:: append(x) - - Add *x* to the right side of the deque. - - - .. method:: appendleft(x) - - Add *x* to the left side of the deque. - - - .. method:: clear() - - Remove all elements from the deque leaving it with length 0. - - - .. method:: copy() - - Create a shallow copy of the deque. - - .. versionadded:: 3.5 - - - .. method:: count(x) - - Count the number of deque elements equal to *x*. - - .. versionadded:: 3.2 - - - .. method:: extend(iterable) - - Extend the right side of the deque by appending elements from the iterable - argument. - - - .. method:: extendleft(iterable) - - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. - - - .. method:: index(x[, start[, stop]]) - - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - - .. versionadded:: 3.5 - - - .. method:: insert(i, x) - - Insert *x* into the deque at position *i*. - - If the insertion would cause a bounded deque to grow beyond *maxlen*, - an :exc:`IndexError` is raised. - - .. versionadded:: 3.5 - - - .. method:: pop() - - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: popleft() - - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: remove(value) - - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n=1) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate - to the left. - - When the deque is not empty, rotating one step to the right is equivalent - to ``d.appendleft(d.pop())``, and rotating one step to the left is - equivalent to ``d.append(d.popleft())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or ``None`` if unbounded. - - .. versionadded:: 3.1 - - -In addition to the above, deques support iteration, pickling, ``len(d)``, -``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with -the :keyword:`in` operator, and subscript references such as ``d[-1]``. Indexed -access is O(1) at both ends but slows to O(n) in the middle. For fast random -access, use lists instead. - -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - -Example: - -.. doctest:: - - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I - - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) - - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' - - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque - - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) - - -:class:`deque` Recipes -^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with deques. - -Bounded length deques provide functionality similar to the ``tail`` filter -in Unix:: - - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) - -Another approach to using deques is to maintain a sequence of recently -added elements by appending to the right and popping to the left:: - - def moving_average(iterable, n=3): - # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 - # http://en.wikipedia.org/wiki/Moving_average - it = iter(iterable) - d = deque(itertools.islice(it, n-1)) - d.appendleft(0) - s = sum(d) - for elem in it: - s += elem - d.popleft() - d.append(elem) - yield s / n - -The :meth:`rotate` method provides a way to implement :class:`deque` slicing and -deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the :meth:`rotate` method to position elements to be popped:: - - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) - -To implement :class:`deque` slicing, use a similar approach applying -:meth:`rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`popleft`, add new entries with :meth:`extend`, and then -reverse the rotation. -With minor variations on that approach, it is easy to implement Forth style -stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, -``rot``, and ``roll``. - - -:class:`defaultdict` objects ----------------------------- - -.. class:: defaultdict([default_factory[, ...]]) - - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. - - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. - - - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: - - .. method:: __missing__(key) - - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. - - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. - - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. - - This method is called by the :meth:`__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. - - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. - - - :class:`defaultdict` objects support the following instance variable: - - - .. attribute:: default_factory - - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. - - -:class:`defaultdict` Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using :class:`list` as the :attr:`default_factory`, it is easy to group a -sequence of key-value pairs into a dictionary of lists: - - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` -operation then attaches the value to the new list. When keys are encountered -again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is -simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -Setting the :attr:`default_factory` to :class:`int` makes the -:class:`defaultdict` useful for counting (like a bag or multiset in other -languages): - - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> sorted(d.items()) - [('i', 4), ('m', 1), ('p', 2), ('s', 4)] - -When a letter is first encountered, it is missing from the mapping, so the -:attr:`default_factory` function calls :func:`int` to supply a default count of -zero. The increment operation then builds up the count for each letter. - -The function :func:`int` which always returns zero is just a special case of -constant functions. A faster and more flexible way to create constant functions -is to use a lambda function which can supply any constant value (not just -zero): - - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' - -Setting the :attr:`default_factory` to :class:`set` makes the -:class:`defaultdict` useful for building a dictionary of sets: - - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> sorted(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] - - -:func:`namedtuple` Factory Function for Tuples with Named Fields ----------------------------------------------------------------- - -Named tuples assign meaning to each position in a tuple and allow for more readable, -self-documenting code. They can be used wherever regular tuples are used, and -they add the ability to access fields by name instead of position index. - -.. function:: namedtuple(typename, field_names, *, verbose=False, rename=False, module=None) - - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. - - The *field_names* are a sequence of strings such as ``['x', 'y']``. - Alternatively, *field_names* can be a single string with each fieldname - separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. - - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. - - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. - - If *verbose* is true, the class definition is printed after it is - built. This option is outdated; instead, it is simpler to print the - :attr:`_source` attribute. - - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. - - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. - - .. versionchanged:: 3.1 - Added support for *rename*. - - .. versionchanged:: 3.6 - The *verbose* and *rename* parameters became - :ref:`keyword-only arguments `. - - .. versionchanged:: 3.6 - Added the *module* parameter. - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) - -Named tuples are especially useful for assigning field names to result tuples returned -by the :mod:`csv` or :mod:`sqlite3` modules:: - - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) - - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) - -In addition to the methods inherited from tuples, named tuples support -three additional methods and two attributes. To prevent conflicts with -field names, the method and attribute names start with an underscore. - -.. classmethod:: somenamedtuple._make(iterable) - - Class method that makes a new instance from an existing sequence or iterable. - - .. doctest:: - - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) - -.. method:: somenamedtuple._asdict() - - Return a new :class:`OrderedDict` which maps field names to their corresponding - values: - - .. doctest:: - - >>> p = Point(x=11, y=22) - >>> p._asdict() - OrderedDict([('x', 11), ('y', 22)]) - - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. - -.. method:: somenamedtuple._replace(**kwargs) - - Return a new instance of the named tuple replacing specified fields with new - values:: - - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) - - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) - -.. attribute:: somenamedtuple._source - - A string with the pure Python source code used to create the named - tuple class. The source makes the named tuple self-documenting. - It can be printed, executed using :func:`exec`, or saved to a file - and imported. - - .. versionadded:: 3.3 - -.. attribute:: somenamedtuple._fields - - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. - - .. doctest:: - - >>> p._fields # view the field names - ('x', 'y') - - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) - -To retrieve a field whose name is stored in a string, use the :func:`getattr` -function: - - >>> getattr(p, 'x') - 11 - -To convert a dictionary to a named tuple, use the double-star-operator -(as described in :ref:`tut-unpacking-arguments`): - - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) - -Since a named tuple is a regular Python class, it is easy to add or change -functionality with a subclass. Here is how to add a calculated field and -a fixed-width print format: - -.. doctest:: - - >>> class Point(namedtuple('Point', ['x', 'y'])): - ... __slots__ = () - ... @property - ... def hypot(self): - ... return (self.x ** 2 + self.y ** 2) ** 0.5 - ... def __str__(self): - ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - - >>> for p in Point(3, 4), Point(14, 5/7): - ... print(p) - Point: x= 3.000 y= 4.000 hypot= 5.000 - Point: x=14.000 y= 0.714 hypot=14.018 - -The subclass shown above sets ``__slots__`` to an empty tuple. This helps -keep memory requirements low by preventing the creation of instance dictionaries. - -Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`_fields` attribute: - - >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) - -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - -Default values can be implemented by using :meth:`_replace` to -customize a prototype instance: - - >>> Account = namedtuple('Account', 'owner balance transaction_count') - >>> default_account = Account('', 0.0, 0) - >>> johns_account = default_account._replace(owner='John') - >>> janes_account = default_account._replace(owner='Jane') - - -.. seealso:: - - * `Recipe for named tuple abstract base class with a metaclass mix-in - `_ - by Jan Kaliszewski. Besides providing an :term:`abstract base class` for - named tuples, it also supports an alternate :term:`metaclass`-based - constructor that is convenient for use cases where named tuples are being - subclassed. - - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. - - * See :meth:`typing.NamedTuple` for a way to add type hints for named tuples. - - -:class:`OrderedDict` objects ----------------------------- - -Ordered dictionaries are just like regular dictionaries but they remember the -order that items were inserted. When iterating over an ordered dictionary, -the items are returned in the order their keys were first added. - -.. class:: OrderedDict([items]) - - Return an instance of a dict subclass, supporting the usual :class:`dict` - methods. An *OrderedDict* is a dict that remembers the order that keys - were first inserted. If a new entry overwrites an existing entry, the - original insertion position is left unchanged. Deleting an entry and - reinserting it will move it to the end. - - .. versionadded:: 3.1 - - .. method:: popitem(last=True) - - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in - :abbr:`LIFO (last-in, first-out)` order if *last* is true - or :abbr:`FIFO (first-in, first-out)` order if false. - - .. method:: move_to_end(key, last=True) - - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist:: - - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d.keys()) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d.keys()) - 'bacde' - - .. versionadded:: 3.2 - -In addition to the usual mapping methods, ordered dictionaries also support -reverse iteration using :func:`reversed`. - -Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. -Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. - -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. - -.. versionchanged:: 3.6 - With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` - method. - -:class:`OrderedDict` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Since an ordered dictionary remembers its insertion order, it can be used -in conjunction with sorting to make a sorted dictionary:: - - >>> # regular unsorted dictionary - >>> d = {'banana': 3, 'apple': 4, 'pear': 1, 'orange': 2} - - >>> # dictionary sorted by key - >>> OrderedDict(sorted(d.items(), key=lambda t: t[0])) - OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)]) - - >>> # dictionary sorted by value - >>> OrderedDict(sorted(d.items(), key=lambda t: t[1])) - OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)]) - - >>> # dictionary sorted by length of the key string - >>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0]))) - OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)]) - -The new sorted dictionaries maintain their sort order when entries -are deleted. But when new keys are added, the keys are appended -to the end and the sort is not maintained. - -It is also straight-forward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. -If a new entry overwrites an existing entry, the -original insertion position is changed and moved to the end:: - - class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' - - def __setitem__(self, key, value): - if key in self: - del self[key] - OrderedDict.__setitem__(self, key, value) - -An ordered dictionary can be combined with the :class:`Counter` class -so that the counter remembers the order elements are first encountered:: - - class OrderedCounter(Counter, OrderedDict): - 'Counter that remembers the order elements are first encountered' - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, OrderedDict(self)) - - def __reduce__(self): - return self.__class__, (OrderedDict(self),) - - -:class:`UserDict` objects -------------------------- - -The class, :class:`UserDict` acts as a wrapper around dictionary objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`dict`; however, this class can be easier -to work with because the underlying dictionary is accessible as an -attribute. - -.. class:: UserDict([initialdata]) - - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. - - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: - - .. attribute:: data - - A real dictionary used to store the contents of the :class:`UserDict` - class. - - - -:class:`UserList` objects -------------------------- - -This class acts as a wrapper around list objects. It is a useful base class -for your own list-like classes which can inherit from them and override -existing methods or add new ones. In this way, one can add new behaviors to -lists. - -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`list`; however, this class can be easier -to work with because the underlying list is accessible as an attribute. - -.. class:: UserList([list]) - - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. - - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: - - .. attribute:: data - - A real :class:`list` object used to store the contents of the - :class:`UserList` class. - -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to -offer a constructor which can be called with either no arguments or one -argument. List operations which return a new sequence attempt to create an -instance of the actual implementation class. To do so, it assumes that the -constructor can be called with a single parameter, which is a sequence object -used as a data source. - -If a derived class does not wish to comply with this requirement, all of the -special methods supported by this class will need to be overridden; please -consult the sources for information about the methods which need to be provided -in that case. - -:class:`UserString` objects ---------------------------- - -The class, :class:`UserString` acts as a wrapper around string objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`str`; however, this class can be easier -to work with because the underlying string is accessible as an -attribute. - -.. class:: UserString([sequence]) - - Class that simulates a string or a Unicode string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *sequence*. The *sequence* can - be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a - subclass) or an arbitrary sequence which can be converted into a string using - the built-in :func:`str` function. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/third_party/python/Doc/library/colorsys.rst b/third_party/python/Doc/library/colorsys.rst deleted file mode 100644 index c33f531cc..000000000 --- a/third_party/python/Doc/library/colorsys.rst +++ /dev/null @@ -1,65 +0,0 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== - -.. module:: colorsys - :synopsis: Conversion functions between RGB and other color systems. - -.. sectionauthor:: David Ascher - -**Source code:** :source:`Lib/colorsys.py` - --------------- - -The :mod:`colorsys` module defines bidirectional conversions of color values -between colors expressed in the RGB (Red Green Blue) color space used in -computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness -Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color -spaces are floating point values. In the YIQ space, the Y coordinate is between -0 and 1, but the I and Q coordinates can be positive or negative. In all other -spaces, the coordinates are all between 0 and 1. - -.. seealso:: - - More information about color spaces can be found at - http://www.poynton.com/ColorFAQ.html and - https://www.cambridgeincolour.com/tutorials/color-spaces.htm. - -The :mod:`colorsys` module defines the following functions: - - -.. function:: rgb_to_yiq(r, g, b) - - Convert the color from RGB coordinates to YIQ coordinates. - - -.. function:: yiq_to_rgb(y, i, q) - - Convert the color from YIQ coordinates to RGB coordinates. - - -.. function:: rgb_to_hls(r, g, b) - - Convert the color from RGB coordinates to HLS coordinates. - - -.. function:: hls_to_rgb(h, l, s) - - Convert the color from HLS coordinates to RGB coordinates. - - -.. function:: rgb_to_hsv(r, g, b) - - Convert the color from RGB coordinates to HSV coordinates. - - -.. function:: hsv_to_rgb(h, s, v) - - Convert the color from HSV coordinates to RGB coordinates. - -Example:: - - >>> import colorsys - >>> colorsys.rgb_to_hsv(0.2, 0.4, 0.4) - (0.5, 0.5, 0.4) - >>> colorsys.hsv_to_rgb(0.5, 0.5, 0.4) - (0.2, 0.4, 0.4) diff --git a/third_party/python/Doc/library/compileall.rst b/third_party/python/Doc/library/compileall.rst deleted file mode 100644 index c1af02b0d..000000000 --- a/third_party/python/Doc/library/compileall.rst +++ /dev/null @@ -1,234 +0,0 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== - -.. module:: compileall - :synopsis: Tools for byte-compiling all Python source files in a directory tree. - -**Source code:** :source:`Lib/compileall.py` - --------------- - -This module provides some utility functions to support installing Python -libraries. These functions compile Python source files in a directory tree. -This module can be used to create the cached byte-code files at library -installation time, which makes them available for use even by users who don't -have write permission to the library directories. - - -Command-line use ----------------- - -This module can work as a script (using :program:`python -m compileall`) to -compile Python sources. - -.. program:: compileall - -.. cmdoption:: directory ... - file ... - - Positional arguments are files to compile or directories that contain - source files, traversed recursively. If no argument is given, behave as if - the command line was ``-l ``. - -.. cmdoption:: -l - - Do not recurse into subdirectories, only compile source code files directly - contained in the named or implied directories. - -.. cmdoption:: -f - - Force rebuild even if timestamps are up-to-date. - -.. cmdoption:: -q - - Do not print the list of files compiled. If passed once, error messages will - still be printed. If passed twice (``-qq``), all output is suppressed. - -.. cmdoption:: -d destdir - - Directory prepended to the path to each file being compiled. This will - appear in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - -.. cmdoption:: -x regex - - regex is used to search the full path to each file considered for - compilation, and if the regex produces a match, the file is skipped. - -.. cmdoption:: -i list - - Read the file ``list`` and add each line that it contains to the list of - files and directories to compile. If ``list`` is ``-``, read lines from - ``stdin``. - -.. cmdoption:: -b - - Write the byte-code files to their legacy locations and names, which may - overwrite byte-code files created by another version of Python. The default - is to write files to their :pep:`3147` locations and names, which allows - byte-code files from multiple versions of Python to coexist. - -.. cmdoption:: -r - - Control the maximum recursion level for subdirectories. - If this is given, then ``-l`` option will not be taken into account. - :program:`python -m compileall -r 0` is equivalent to - :program:`python -m compileall -l`. - -.. cmdoption:: -j N - - Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.cpu_count()` - will be used. - -.. versionchanged:: 3.2 - Added the ``-i``, ``-b`` and ``-h`` options. - -.. versionchanged:: 3.5 - Added the ``-j``, ``-r``, and ``-qq`` options. ``-q`` option - was changed to a multilevel value. ``-b`` will always produce a - byte-code file ending in ``.pyc``, never ``.pyo``. - - -There is no command-line option to control the optimization level used by the -:func:`compile` function, because the Python interpreter itself already -provides the option: :program:`python -O -m compileall`. - -Public functions ----------------- - -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1) - - Recursively descend the directory tree named by *dir*, compiling all :file:`.py` - files along the way. Return a true value if all the files compiled successfully, - and a false value otherwise. - - The *maxlevels* parameter is used to limit the depth of the recursion; it - defaults to ``10``. - - If *ddir* is given, it is prepended to the path to each file being compiled - for use in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - - If *force* is true, modules are re-compiled even if the timestamps are up to - date. - - If *rx* is given, its search method is called on the complete path to each - file considered for compilation, and if it returns a true value, the file - is skipped. - - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. - - If *legacy* is true, byte-code files are written to their legacy locations - and names, which may overwrite byte-code files created by another version of - Python. The default is to write files to their :pep:`3147` locations and - names, which allows byte-code files from multiple versions of Python to - coexist. - - *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. - - The argument *workers* specifies how many workers are used to - compile files in parallel. The default is to not use multiple workers. - If the platform can't use multiple workers and *workers* argument is given, - then sequential compilation will be used as a fallback. If *workers* is - lower than ``0``, a :exc:`ValueError` will be raised. - - .. versionchanged:: 3.2 - Added the *legacy* and *optimize* parameter. - - .. versionchanged:: 3.5 - Added the *workers* parameter. - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1) - - Compile the file with path *fullname*. Return a true value if the file - compiled successfully, and a false value otherwise. - - If *ddir* is given, it is prepended to the path to the file being compiled - for use in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - - If *rx* is given, its search method is passed the full path name to the - file being compiled, and if it returns a true value, the file is not - compiled and ``True`` is returned. - - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. - - If *legacy* is true, byte-code files are written to their legacy locations - and names, which may overwrite byte-code files created by another version of - Python. The default is to write files to their :pep:`3147` locations and - names, which allows byte-code files from multiple versions of Python to - coexist. - - *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1) - - Byte-compile all the :file:`.py` files found along ``sys.path``. Return a - true value if all the files compiled successfully, and a false value otherwise. - - If *skip_curdir* is true (the default), the current directory is not included - in the search. All other parameters are passed to the :func:`compile_dir` - function. Note that unlike the other compile functions, ``maxlevels`` - defaults to ``0``. - - .. versionchanged:: 3.2 - Added the *legacy* and *optimize* parameter. - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - -To force a recompile of all the :file:`.py` files in the :file:`Lib/` -subdirectory and all its subdirectories:: - - import compileall - - compileall.compile_dir('Lib/', force=True) - - # Perform same compilation, excluding files in .svn directories. - import re - compileall.compile_dir('Lib/', rx=re.compile(r'[/\\][.]svn'), force=True) - - # pathlib.Path objects can also be used. - import pathlib - compileall.compile_dir(pathlib.Path('Lib/'), force=True) - -.. seealso:: - - Module :mod:`py_compile` - Byte-compile a single source file. diff --git a/third_party/python/Doc/library/concurrency.rst b/third_party/python/Doc/library/concurrency.rst deleted file mode 100644 index 0de281bd1..000000000 --- a/third_party/python/Doc/library/concurrency.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _concurrency: - -******************** -Concurrent Execution -******************** - -The modules described in this chapter provide support for concurrent -execution of code. The appropriate choice of tool will depend on the -task to be executed (CPU bound vs IO bound) and preferred style of -development (event driven cooperative multitasking vs preemptive -multitasking). Here's an overview: - - -.. toctree:: - - threading.rst - multiprocessing.rst - concurrent.rst - concurrent.futures.rst - subprocess.rst - sched.rst - queue.rst - - -The following are support modules for some of the above services: - -.. toctree:: - - dummy_threading.rst - _thread.rst - _dummy_thread.rst diff --git a/third_party/python/Doc/library/concurrent.futures.rst b/third_party/python/Doc/library/concurrent.futures.rst deleted file mode 100644 index 319f757aa..000000000 --- a/third_party/python/Doc/library/concurrent.futures.rst +++ /dev/null @@ -1,445 +0,0 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== - -.. module:: concurrent.futures - :synopsis: Execute computations concurrently using threads or processes. - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` - --------------- - -The :mod:`concurrent.futures` module provides a high-level interface for -asynchronously executing callables. - -The asynchronous execution can be performed with threads, using -:class:`ThreadPoolExecutor`, or separate processes, using -:class:`ProcessPoolExecutor`. Both implement the same interface, which is -defined by the abstract :class:`Executor` class. - - -Executor Objects ----------------- - -.. class:: Executor - - An abstract class that provides methods to execute calls asynchronously. It - should not be used directly, but through its concrete subclasses. - - .. method:: submit(fn, *args, **kwargs) - - Schedules the callable, *fn*, to be executed as ``fn(*args **kwargs)`` - and returns a :class:`Future` object representing the execution of the - callable. :: - - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(pow, 323, 1235) - print(future.result()) - - .. method:: map(func, *iterables, timeout=None, chunksize=1) - - Similar to :func:`map(func, *iterables) ` except: - - * the *iterables* are collected immediately rather than lazily; - - * *func* is executed asynchronously and several calls to - *func* may be made concurrently. - - The returned iterator raises a :exc:`concurrent.futures.TimeoutError` - if :meth:`~iterator.__next__` is called and the result isn't available - after *timeout* seconds from the original call to :meth:`Executor.map`. - *timeout* can be an int or a float. If *timeout* is not specified or - ``None``, there is no limit to the wait time. - - If a *func* call raises an exception, then that exception will be - raised when its value is retrieved from the iterator. - - When using :class:`ProcessPoolExecutor`, this method chops *iterables* - into a number of chunks which it submits to the pool as separate - tasks. The (approximate) size of these chunks can be specified by - setting *chunksize* to a positive integer. For very long iterables, - using a large value for *chunksize* can significantly improve - performance compared to the default size of 1. With - :class:`ThreadPoolExecutor`, *chunksize* has no effect. - - .. versionchanged:: 3.5 - Added the *chunksize* argument. - - .. method:: shutdown(wait=True) - - Signal the executor that it should free any resources that it is using - when the currently pending futures are done executing. Calls to - :meth:`Executor.submit` and :meth:`Executor.map` made after shutdown will - raise :exc:`RuntimeError`. - - If *wait* is ``True`` then this method will not return until all the - pending futures are done executing and the resources associated with the - executor have been freed. If *wait* is ``False`` then this method will - return immediately and the resources associated with the executor will be - freed when all pending futures are done executing. Regardless of the - value of *wait*, the entire Python program will not exit until all - pending futures are done executing. - - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: - - import shutil - with ThreadPoolExecutor(max_workers=4) as e: - e.submit(shutil.copy, 'src1.txt', 'dest1.txt') - e.submit(shutil.copy, 'src2.txt', 'dest2.txt') - e.submit(shutil.copy, 'src3.txt', 'dest3.txt') - e.submit(shutil.copy, 'src4.txt', 'dest4.txt') - - -ThreadPoolExecutor ------------------- - -:class:`ThreadPoolExecutor` is an :class:`Executor` subclass that uses a pool of -threads to execute calls asynchronously. - -Deadlocks can occur when the callable associated with a :class:`Future` waits on -the results of another :class:`Future`. For example:: - - import time - def wait_on_b(): - time.sleep(5) - print(b.result()) # b will never complete because it is waiting on a. - return 5 - - def wait_on_a(): - time.sleep(5) - print(a.result()) # a will never complete because it is waiting on b. - return 6 - - - executor = ThreadPoolExecutor(max_workers=2) - a = executor.submit(wait_on_b) - b = executor.submit(wait_on_a) - -And:: - - def wait_on_future(): - f = executor.submit(pow, 5, 2) - # This will never complete because there is only one worker thread and - # it is executing this function. - print(f.result()) - - executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) - - -.. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='') - - An :class:`Executor` subclass that uses a pool of at most *max_workers* - threads to execute calls asynchronously. - - .. versionchanged:: 3.5 - If *max_workers* is ``None`` or - not given, it will default to the number of processors on the machine, - multiplied by ``5``, assuming that :class:`ThreadPoolExecutor` is often - used to overlap I/O instead of CPU work and the number of workers - should be higher than the number of workers - for :class:`ProcessPoolExecutor`. - - .. versionadded:: 3.6 - The *thread_name_prefix* argument was added to allow users to - control the :class:`threading.Thread` names for worker threads created by - the pool for easier debugging. - -.. _threadpoolexecutor-example: - -ThreadPoolExecutor Example -~~~~~~~~~~~~~~~~~~~~~~~~~~ -:: - - import concurrent.futures - import urllib.request - - URLS = ['http://www.foxnews.com/', - 'http://www.cnn.com/', - 'http://europe.wsj.com/', - 'http://www.bbc.co.uk/', - 'http://some-made-up-domain.com/'] - - # Retrieve a single page and report the URL and contents - def load_url(url, timeout): - with urllib.request.urlopen(url, timeout=timeout) as conn: - return conn.read() - - # We can use a with statement to ensure threads are cleaned up promptly - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Start the load operations and mark each future with its URL - future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} - for future in concurrent.futures.as_completed(future_to_url): - url = future_to_url[future] - try: - data = future.result() - except Exception as exc: - print('%r generated an exception: %s' % (url, exc)) - else: - print('%r page is %d bytes' % (url, len(data))) - - -ProcessPoolExecutor -------------------- - -The :class:`ProcessPoolExecutor` class is an :class:`Executor` subclass that -uses a pool of processes to execute calls asynchronously. -:class:`ProcessPoolExecutor` uses the :mod:`multiprocessing` module, which -allows it to side-step the :term:`Global Interpreter Lock` but also means that -only picklable objects can be executed and returned. - -The ``__main__`` module must be importable by worker subprocesses. This means -that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. - -Calling :class:`Executor` or :class:`Future` methods from a callable submitted -to a :class:`ProcessPoolExecutor` will result in deadlock. - -.. class:: ProcessPoolExecutor(max_workers=None) - - An :class:`Executor` subclass that executes calls asynchronously using a pool - of at most *max_workers* processes. If *max_workers* is ``None`` or not - given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` - will be raised. - - .. versionchanged:: 3.3 - When one of the worker processes terminates abruptly, a - :exc:`BrokenProcessPool` error is now raised. Previously, behaviour - was undefined but operations on the executor or its futures would often - freeze or deadlock. - - -.. _processpoolexecutor-example: - -ProcessPoolExecutor Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:: - - import concurrent.futures - import math - - PRIMES = [ - 112272535095293, - 112582705942171, - 112272535095293, - 115280095190773, - 115797848077099, - 1099726899285419] - - def is_prime(n): - if n % 2 == 0: - return False - - sqrt_n = int(math.floor(math.sqrt(n))) - for i in range(3, sqrt_n + 1, 2): - if n % i == 0: - return False - return True - - def main(): - with concurrent.futures.ProcessPoolExecutor() as executor: - for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)): - print('%d is prime: %s' % (number, prime)) - - if __name__ == '__main__': - main() - - -Future Objects --------------- - -The :class:`Future` class encapsulates the asynchronous execution of a callable. -:class:`Future` instances are created by :meth:`Executor.submit`. - -.. class:: Future - - Encapsulates the asynchronous execution of a callable. :class:`Future` - instances are created by :meth:`Executor.submit` and should not be created - directly except for testing. - - .. method:: cancel() - - Attempt to cancel the call. If the call is currently being executed and - cannot be cancelled then the method will return ``False``, otherwise the - call will be cancelled and the method will return ``True``. - - .. method:: cancelled() - - Return ``True`` if the call was successfully cancelled. - - .. method:: running() - - Return ``True`` if the call is currently being executed and cannot be - cancelled. - - .. method:: done() - - Return ``True`` if the call was successfully cancelled or finished - running. - - .. method:: result(timeout=None) - - Return the value returned by the call. If the call hasn't yet completed - then this method will wait up to *timeout* seconds. If the call hasn't - completed in *timeout* seconds, then a - :exc:`concurrent.futures.TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. - - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. - - If the call raised, this method will raise the same exception. - - .. method:: exception(timeout=None) - - Return the exception raised by the call. If the call hasn't yet - completed then this method will wait up to *timeout* seconds. If the - call hasn't completed in *timeout* seconds, then a - :exc:`concurrent.futures.TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. - - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. - - If the call completed without raising, ``None`` is returned. - - .. method:: add_done_callback(fn) - - Attaches the callable *fn* to the future. *fn* will be called, with the - future as its only argument, when the future is cancelled or finishes - running. - - Added callables are called in the order that they were added and are - always called in a thread belonging to the process that added them. If - the callable raises an :exc:`Exception` subclass, it will be logged and - ignored. If the callable raises a :exc:`BaseException` subclass, the - behavior is undefined. - - If the future has already completed or been cancelled, *fn* will be - called immediately. - - The following :class:`Future` methods are meant for use in unit tests and - :class:`Executor` implementations. - - .. method:: set_running_or_notify_cancel() - - This method should only be called by :class:`Executor` implementations - before executing the work associated with the :class:`Future` and by unit - tests. - - If the method returns ``False`` then the :class:`Future` was cancelled, - i.e. :meth:`Future.cancel` was called and returned `True`. Any threads - waiting on the :class:`Future` completing (i.e. through - :func:`as_completed` or :func:`wait`) will be woken up. - - If the method returns ``True`` then the :class:`Future` was not cancelled - and has been put in the running state, i.e. calls to - :meth:`Future.running` will return `True`. - - This method can only be called once and cannot be called after - :meth:`Future.set_result` or :meth:`Future.set_exception` have been - called. - - .. method:: set_result(result) - - Sets the result of the work associated with the :class:`Future` to - *result*. - - This method should only be used by :class:`Executor` implementations and - unit tests. - - .. method:: set_exception(exception) - - Sets the result of the work associated with the :class:`Future` to the - :class:`Exception` *exception*. - - This method should only be used by :class:`Executor` implementations and - unit tests. - - -Module Functions ----------------- - -.. function:: wait(fs, timeout=None, return_when=ALL_COMPLETED) - - Wait for the :class:`Future` instances (possibly created by different - :class:`Executor` instances) given by *fs* to complete. Returns a named - 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or were cancelled) before the wait completed. The second - set, named ``not_done``, contains uncompleted futures. - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - - *return_when* indicates when this function should return. It must be one of - the following constants: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - -.. function:: as_completed(fs, timeout=None) - - Returns an iterator over the :class:`Future` instances (possibly created by - different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that - are duplicated will be returned once. Any futures that completed before - :func:`as_completed` is called will be yielded first. The returned iterator - raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` - is called and the result isn't available after *timeout* seconds from the - original call to :func:`as_completed`. *timeout* can be an int or float. If - *timeout* is not specified or ``None``, there is no limit to the wait time. - - -.. seealso:: - - :pep:`3148` -- futures - execute computations asynchronously - The proposal which described this feature for inclusion in the Python - standard library. - - -Exception classes ------------------ - -.. currentmodule:: concurrent.futures - -.. exception:: CancelledError - - Raised when a future is cancelled. - -.. exception:: TimeoutError - - Raised when a future operation exceeds the given timeout. - -.. currentmodule:: concurrent.futures.process - -.. exception:: BrokenProcessPool - - Derived from :exc:`RuntimeError`, this exception class is raised when - one of the workers of a :class:`ProcessPoolExecutor` has terminated - in a non-clean fashion (for example, if it was killed from the outside). - - .. versionadded:: 3.3 - diff --git a/third_party/python/Doc/library/concurrent.rst b/third_party/python/Doc/library/concurrent.rst deleted file mode 100644 index 2eba53651..000000000 --- a/third_party/python/Doc/library/concurrent.rst +++ /dev/null @@ -1,6 +0,0 @@ -The :mod:`concurrent` package -============================= - -Currently, there is only one module in this package: - -* :mod:`concurrent.futures` -- Launching parallel tasks diff --git a/third_party/python/Doc/library/configparser.rst b/third_party/python/Doc/library/configparser.rst deleted file mode 100644 index 79e189687..000000000 --- a/third_party/python/Doc/library/configparser.rst +++ /dev/null @@ -1,1324 +0,0 @@ -:mod:`configparser` --- Configuration file parser -================================================= - -.. module:: configparser - :synopsis: Configuration file parser. - -.. moduleauthor:: Ken Manheimer -.. moduleauthor:: Barry Warsaw -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Christopher G. Petrilli -.. sectionauthor:: Łukasz Langa - -**Source code:** :source:`Lib/configparser.py` - -.. index:: - pair: .ini; file - pair: configuration; file - single: ini file - single: Windows ini file - --------------- - -This module provides the :class:`ConfigParser` class which implements a basic -configuration language which provides a structure similar to what's found in -Microsoft Windows INI files. You can use this to write Python programs which -can be customized by end users easily. - -.. note:: - - This library does *not* interpret or write the value-type prefixes used in - the Windows Registry extended version of INI syntax. - -.. seealso:: - - Module :mod:`shlex` - Support for creating Unix shell-like mini-languages which can be used as - an alternate format for application configuration files. - - Module :mod:`json` - The json module implements a subset of JavaScript syntax which can also - be used for this purpose. - - -Quick Start ------------ - -Let's take a very basic configuration file that looks like this: - -.. code-block:: ini - - [DEFAULT] - ServerAliveInterval = 45 - Compression = yes - CompressionLevel = 9 - ForwardX11 = yes - - [bitbucket.org] - User = hg - - [topsecret.server.com] - Port = 50022 - ForwardX11 = no - -The structure of INI files is described `in the following section -<#supported-ini-file-structure>`_. Essentially, the file -consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by -creating the above configuration file programmatically. - -.. doctest:: - - >>> import configparser - >>> config = configparser.ConfigParser() - >>> config['DEFAULT'] = {'ServerAliveInterval': '45', - ... 'Compression': 'yes', - ... 'CompressionLevel': '9'} - >>> config['bitbucket.org'] = {} - >>> config['bitbucket.org']['User'] = 'hg' - >>> config['topsecret.server.com'] = {} - >>> topsecret = config['topsecret.server.com'] - >>> topsecret['Port'] = '50022' # mutates the parser - >>> topsecret['ForwardX11'] = 'no' # same here - >>> config['DEFAULT']['ForwardX11'] = 'yes' - >>> with open('example.ini', 'w') as configfile: - ... config.write(configfile) - ... - -As you can see, we can treat a config parser much like a dictionary. -There are differences, `outlined later <#mapping-protocol-access>`_, but -the behavior is very close to what you would expect from a dictionary. - -Now that we have created and saved a configuration file, let's read it -back and explore the data it holds. - -.. doctest:: - - >>> import configparser - >>> config = configparser.ConfigParser() - >>> config.sections() - [] - >>> config.read('example.ini') - ['example.ini'] - >>> config.sections() - ['bitbucket.org', 'topsecret.server.com'] - >>> 'bitbucket.org' in config - True - >>> 'bytebong.com' in config - False - >>> config['bitbucket.org']['User'] - 'hg' - >>> config['DEFAULT']['Compression'] - 'yes' - >>> topsecret = config['topsecret.server.com'] - >>> topsecret['ForwardX11'] - 'no' - >>> topsecret['Port'] - '50022' - >>> for key in config['bitbucket.org']: print(key) - ... - user - compressionlevel - serveraliveinterval - compression - forwardx11 - >>> config['bitbucket.org']['ForwardX11'] - 'yes' - -As we can see above, the API is pretty straightforward. The only bit of magic -involves the ``DEFAULT`` section which provides default values for all other -sections [1]_. Note also that keys in sections are -case-insensitive and stored in lowercase [1]_. - - -Supported Datatypes -------------------- - -Config parsers do not guess datatypes of values in configuration files, always -storing them internally as strings. This means that if you need other -datatypes, you should convert on your own: - -.. doctest:: - - >>> int(topsecret['Port']) - 50022 - >>> float(topsecret['CompressionLevel']) - 9.0 - -Since this task is so common, config parsers provide a range of handy getter -methods to handle integers, floats and booleans. The last one is the most -interesting because simply passing the value to ``bool()`` would do no good -since ``bool('False')`` is still ``True``. This is why config parsers also -provide :meth:`~ConfigParser.getboolean`. This method is case-insensitive and -recognizes Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, -``'true'``/``'false'`` and ``'1'``/``'0'`` [1]_. For example: - -.. doctest:: - - >>> topsecret.getboolean('ForwardX11') - False - >>> config['bitbucket.org'].getboolean('ForwardX11') - True - >>> config.getboolean('bitbucket.org', 'Compression') - True - -Apart from :meth:`~ConfigParser.getboolean`, config parsers also -provide equivalent :meth:`~ConfigParser.getint` and -:meth:`~ConfigParser.getfloat` methods. You can register your own -converters and customize the provided ones. [1]_ - -Fallback Values ---------------- - -As with a dictionary, you can use a section's :meth:`get` method to -provide fallback values: - -.. doctest:: - - >>> topsecret.get('Port') - '50022' - >>> topsecret.get('CompressionLevel') - '9' - >>> topsecret.get('Cipher') - >>> topsecret.get('Cipher', '3des-cbc') - '3des-cbc' - -Please note that default values have precedence over fallback values. -For instance, in our example the ``'CompressionLevel'`` key was -specified only in the ``'DEFAULT'`` section. If we try to get it from -the section ``'topsecret.server.com'``, we will always get the default, -even if we specify a fallback: - -.. doctest:: - - >>> topsecret.get('CompressionLevel', '3') - '9' - -One more thing to be aware of is that the parser-level :meth:`get` method -provides a custom, more complex interface, maintained for backwards -compatibility. When using this method, a fallback value can be provided via -the ``fallback`` keyword-only argument: - -.. doctest:: - - >>> config.get('bitbucket.org', 'monster', - ... fallback='No such things as monsters') - 'No such things as monsters' - -The same ``fallback`` argument can be used with the -:meth:`~ConfigParser.getint`, :meth:`~ConfigParser.getfloat` and -:meth:`~ConfigParser.getboolean` methods, for example: - -.. doctest:: - - >>> 'BatchMode' in topsecret - False - >>> topsecret.getboolean('BatchMode', fallback=True) - True - >>> config['DEFAULT']['BatchMode'] = 'no' - >>> topsecret.getboolean('BatchMode', fallback=True) - False - - -Supported INI File Structure ----------------------------- - -A configuration file consists of sections, each led by a ``[section]`` header, -followed by key/value entries separated by a specific string (``=`` or ``:`` by -default [1]_). By default, section names are case sensitive but keys are not -[1]_. Leading and trailing whitespace is removed from keys and values. -Values can be omitted, in which case the key/value delimiter may also be left -out. Values can also span multiple lines, as long as they are indented deeper -than the first line of the value. Depending on the parser's mode, blank lines -may be treated as parts of multiline values or ignored. - -Configuration files may include comments, prefixed by specific -characters (``#`` and ``;`` by default [1]_). Comments may appear on -their own on an otherwise empty line, possibly indented. [1]_ - -For example: - -.. code-block:: ini - - [Simple Values] - key=value - spaces in keys=allowed - spaces in values=allowed as well - spaces around the delimiter = obviously - you can also use : to delimit keys from values - - [All Values Are Strings] - values like this: 1000000 - or this: 3.14159265359 - are they treated as numbers? : no - integers, floats and booleans are held as: strings - can use the API to get converted values directly: true - - [Multiline Values] - chorus: I'm a lumberjack, and I'm okay - I sleep all night and I work all day - - [No Values] - key_without_value - empty string value here = - - [You can use comments] - # like this - ; or this - - # By default only in an empty line. - # Inline comments can be harmful because they prevent users - # from using the delimiting characters as parts of values. - # That being said, this can be customized. - - [Sections Can Be Indented] - can_values_be_as_well = True - does_that_mean_anything_special = False - purpose = formatting for readability - multiline_values = are - handled just fine as - long as they are indented - deeper than the first line - of a value - # Did I mention we can indent comments, too? - - -Interpolation of values ------------------------ - -On top of the core functionality, :class:`ConfigParser` supports -interpolation. This means values can be preprocessed before returning them -from ``get()`` calls. - -.. index:: single: % (percent); interpolation in configuration files - -.. class:: BasicInterpolation() - - The default implementation used by :class:`ConfigParser`. It enables - values to contain format strings which refer to other values in the same - section, or values in the special default section [1]_. Additional default - values can be provided on initialization. - - For example: - - .. code-block:: ini - - [Paths] - home_dir: /Users - my_dir: %(home_dir)s/lumberjack - my_pictures: %(my_dir)s/Pictures - - - In the example above, :class:`ConfigParser` with *interpolation* set to - ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of - ``home_dir`` (``/Users`` in this case). ``%(my_dir)s`` in effect would - resolve to ``/Users/lumberjack``. All interpolations are done on demand so - keys used in the chain of references do not have to be specified in any - specific order in the configuration file. - - With ``interpolation`` set to ``None``, the parser would simply return - ``%(my_dir)s/Pictures`` as the value of ``my_pictures`` and - ``%(home_dir)s/lumberjack`` as the value of ``my_dir``. - -.. index:: single: $ (dollar); interpolation in configuration files - -.. class:: ExtendedInterpolation() - - An alternative handler for interpolation which implements a more advanced - syntax, used for instance in ``zc.buildout``. Extended interpolation is - using ``${section:option}`` to denote a value from a foreign section. - Interpolation can span multiple levels. For convenience, if the - ``section:`` part is omitted, interpolation defaults to the current section - (and possibly the default values from the special section). - - For example, the configuration specified above with basic interpolation, - would look like this with extended interpolation: - - .. code-block:: ini - - [Paths] - home_dir: /Users - my_dir: ${home_dir}/lumberjack - my_pictures: ${my_dir}/Pictures - - Values from other sections can be fetched as well: - - .. code-block:: ini - - [Common] - home_dir: /Users - library_dir: /Library - system_dir: /System - macports_dir: /opt/local - - [Frameworks] - Python: 3.2 - path: ${Common:system_dir}/Library/Frameworks/ - - [Arthur] - nickname: Two Sheds - last_name: Jackson - my_dir: ${Common:home_dir}/twosheds - my_pictures: ${my_dir}/Pictures - python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} - -Mapping Protocol Access ------------------------ - -.. versionadded:: 3.2 - -Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, -the mapping interface implementation is using the -``parser['section']['option']`` notation. - -``parser['section']`` in particular returns a proxy for the section's data in -the parser. This means that the values are not copied but they are taken from -the original parser on demand. What's even more important is that when values -are changed on a section proxy, they are actually mutated in the original -parser. - -:mod:`configparser` objects behave as close to actual dictionaries as possible. -The mapping interface is complete and adheres to the -:class:`~collections.abc.MutableMapping` ABC. -However, there are a few differences that should be taken into account: - -* By default, all keys in sections are accessible in a case-insensitive manner - [1]_. E.g. ``for option in parser["section"]`` yields only ``optionxform``'ed - option key names. This means lowercased keys by default. At the same time, - for a section that holds the key ``'a'``, both expressions return ``True``:: - - "a" in parser["section"] - "A" in parser["section"] - -* All sections include ``DEFAULTSECT`` values as well which means that - ``.clear()`` on a section may not leave the section visibly empty. This is - because default values cannot be deleted from the section (because technically - they are not there). If they are overridden in the section, deleting causes - the default value to be visible again. Trying to delete a default value - causes a ``KeyError``. - -* ``DEFAULTSECT`` cannot be removed from the parser: - - * trying to delete it raises ``ValueError``, - - * ``parser.clear()`` leaves it intact, - - * ``parser.popitem()`` never returns it. - -* ``parser.get(section, option, **kwargs)`` - the second argument is **not** - a fallback value. Note however that the section-level ``get()`` methods are - compatible both with the mapping protocol and the classic configparser API. - -* ``parser.items()`` is compatible with the mapping protocol (returns a list of - *section_name*, *section_proxy* pairs including the DEFAULTSECT). However, - this method can also be invoked with arguments: ``parser.items(section, raw, - vars)``. The latter call returns a list of *option*, *value* pairs for - a specified ``section``, with all interpolations expanded (unless - ``raw=True`` is provided). - -The mapping protocol is implemented on top of the existing legacy API so that -subclasses overriding the original interface still should have mappings working -as expected. - - -Customizing Parser Behaviour ----------------------------- - -There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible -set of INI styles available. The default functionality is mainly dictated by -historical background and it's very likely that you will want to customize some -of the features. - -The most common way to change the way a specific config parser works is to use -the :meth:`__init__` options: - -* *defaults*, default value: ``None`` - - This option accepts a dictionary of key-value pairs which will be initially - put in the ``DEFAULT`` section. This makes for an elegant way to support - concise configuration files that don't specify values which are the same as - the documented default. - - Hint: if you want to specify default values for a specific section, use - :meth:`read_dict` before you read the actual file. - -* *dict_type*, default value: :class:`collections.OrderedDict` - - This option has a major impact on how the mapping protocol will behave and how - the written configuration files look. With the default ordered - dictionary, every section is stored in the order they were added to the - parser. Same goes for options within sections. - - An alternative dictionary type can be used for example to sort sections and - options on write-back. You can also use a regular dictionary for performance - reasons. - - Please note: there are ways to add a set of key-value pairs in a single - operation. When you use a regular dictionary in those operations, the order - of the keys may be random. For example: - - .. doctest:: - - >>> parser = configparser.ConfigParser() - >>> parser.read_dict({'section1': {'key1': 'value1', - ... 'key2': 'value2', - ... 'key3': 'value3'}, - ... 'section2': {'keyA': 'valueA', - ... 'keyB': 'valueB', - ... 'keyC': 'valueC'}, - ... 'section3': {'foo': 'x', - ... 'bar': 'y', - ... 'baz': 'z'} - ... }) - >>> parser.sections() - ['section3', 'section2', 'section1'] - >>> [option for option in parser['section3']] - ['baz', 'foo', 'bar'] - - In these operations you need to use an ordered dictionary as well: - - .. doctest:: - - >>> from collections import OrderedDict - >>> parser = configparser.ConfigParser() - >>> parser.read_dict( - ... OrderedDict(( - ... ('s1', - ... OrderedDict(( - ... ('1', '2'), - ... ('3', '4'), - ... ('5', '6'), - ... )) - ... ), - ... ('s2', - ... OrderedDict(( - ... ('a', 'b'), - ... ('c', 'd'), - ... ('e', 'f'), - ... )) - ... ), - ... )) - ... ) - >>> parser.sections() - ['s1', 's2'] - >>> [option for option in parser['s1']] - ['1', '3', '5'] - >>> [option for option in parser['s2'].values()] - ['b', 'd', 'f'] - -* *allow_no_value*, default value: ``False`` - - Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The - *allow_no_value* parameter to the constructor can be used to - indicate that such values should be accepted: - - .. doctest:: - - >>> import configparser - - >>> sample_config = """ - ... [mysqld] - ... user = mysql - ... pid-file = /var/run/mysqld/mysqld.pid - ... skip-external-locking - ... old_passwords = 1 - ... skip-bdb - ... # we don't need ACID today - ... skip-innodb - ... """ - >>> config = configparser.ConfigParser(allow_no_value=True) - >>> config.read_string(sample_config) - - >>> # Settings with values are treated as before: - >>> config["mysqld"]["user"] - 'mysql' - - >>> # Settings without values provide None: - >>> config["mysqld"]["skip-bdb"] - - >>> # Settings which aren't specified still raise an error: - >>> config["mysqld"]["does-not-exist"] - Traceback (most recent call last): - ... - KeyError: 'does-not-exist' - -* *delimiters*, default value: ``('=', ':')`` - - Delimiters are substrings that delimit keys from values within a section. - The first occurrence of a delimiting substring on a line is considered - a delimiter. This means values (but not keys) can contain the delimiters. - - See also the *space_around_delimiters* argument to - :meth:`ConfigParser.write`. - -* *comment_prefixes*, default value: ``('#', ';')`` - -* *inline_comment_prefixes*, default value: ``None`` - - Comment prefixes are strings that indicate the start of a valid comment within - a config file. *comment_prefixes* are used only on otherwise empty lines - (optionally indented) whereas *inline_comment_prefixes* can be used after - every valid value (e.g. section names, options and empty lines as well). By - default inline comments are disabled and ``'#'`` and ``';'`` are used as - prefixes for whole line comments. - - .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched - ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. - - Please note that config parsers don't support escaping of comment prefixes so - using *inline_comment_prefixes* may prevent users from specifying option - values with characters used as comment prefixes. When in doubt, avoid - setting *inline_comment_prefixes*. In any circumstances, the only way of - storing comment prefix characters at the beginning of a line in multiline - values is to interpolate the prefix, for example:: - - >>> from configparser import ConfigParser, ExtendedInterpolation - >>> parser = ConfigParser(interpolation=ExtendedInterpolation()) - >>> # the default BasicInterpolation could be used as well - >>> parser.read_string(""" - ... [DEFAULT] - ... hash = # - ... - ... [hashes] - ... shebang = - ... ${hash}!/usr/bin/env python - ... ${hash} -*- coding: utf-8 -*- - ... - ... extensions = - ... enabled_extension - ... another_extension - ... #disabled_by_comment - ... yet_another_extension - ... - ... interpolation not necessary = if # is not at line start - ... even in multiline values = line #1 - ... line #2 - ... line #3 - ... """) - >>> print(parser['hashes']['shebang']) - - #!/usr/bin/env python - # -*- coding: utf-8 -*- - >>> print(parser['hashes']['extensions']) - - enabled_extension - another_extension - yet_another_extension - >>> print(parser['hashes']['interpolation not necessary']) - if # is not at line start - >>> print(parser['hashes']['even in multiline values']) - line #1 - line #2 - line #3 - -* *strict*, default value: ``True`` - - When set to ``True``, the parser will not allow for any section or option - duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict - parsers in new applications. - - .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched - ``strict=False``. - -* *empty_lines_in_values*, default value: ``True`` - - In config parsers, values can span multiple lines as long as they are - indented more than the key that holds them. By default parsers also let - empty lines to be parts of values. At the same time, keys can be arbitrarily - indented themselves to improve readability. In consequence, when - configuration files get big and complex, it is easy for the user to lose - track of the file structure. Take for instance: - - .. code-block:: ini - - [Section] - key = multiline - value with a gotcha - - this = is still a part of the multiline value of 'key' - - This can be especially problematic for the user to see if she's using a - proportional font to edit the file. That is why when your application does - not need values with empty lines, you should consider disallowing them. This - will make empty lines split keys every time. In the example above, it would - produce two keys, ``key`` and ``this``. - -* *default_section*, default value: ``configparser.DEFAULTSECT`` (that is: - ``"DEFAULT"``) - - The convention of allowing a special section of default values for other - sections or interpolation purposes is a powerful concept of this library, - letting users create complex declarative configurations. This section is - normally called ``"DEFAULT"`` but this can be customized to point to any - other valid section name. Some typical values include: ``"general"`` or - ``"common"``. The name provided is used for recognizing default sections - when reading from any source and is used when writing configuration back to - a file. Its current value can be retrieved using the - ``parser_instance.default_section`` attribute and may be modified at runtime - (i.e. to convert files from one format to another). - -* *interpolation*, default value: ``configparser.BasicInterpolation`` - - Interpolation behaviour may be customized by providing a custom handler - through the *interpolation* argument. ``None`` can be used to turn off - interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the - `dedicated documentation section <#interpolation-of-values>`_. - :class:`RawConfigParser` has a default value of ``None``. - -* *converters*, default value: not set - - Config parsers provide option value getters that perform type conversion. By - default :meth:`~ConfigParser.getint`, :meth:`~ConfigParser.getfloat`, and - :meth:`~ConfigParser.getboolean` are implemented. Should other getters be - desirable, users may define them in a subclass or pass a dictionary where each - key is a name of the converter and each value is a callable implementing said - conversion. For instance, passing ``{'decimal': decimal.Decimal}`` would add - :meth:`getdecimal` on both the parser object and all section proxies. In - other words, it will be possible to write both - ``parser_instance.getdecimal('section', 'key', fallback=0)`` and - ``parser_instance['section'].getdecimal('key', 0)``. - - If the converter needs to access the state of the parser, it can be - implemented as a method on a config parser subclass. If the name of this - method starts with ``get``, it will be available on all section proxies, in - the dict-compatible form (see the ``getdecimal()`` example above). - -More advanced customization may be achieved by overriding default values of -these parser attributes. The defaults are defined on the classes, so they may -be overridden by subclasses or by attribute assignment. - -.. attribute:: ConfigParser.BOOLEAN_STATES - - By default when using :meth:`~ConfigParser.getboolean`, config parsers - consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, - ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, - ``'off'``. You can override this by specifying a custom dictionary of strings - and their Boolean outcomes. For example: - - .. doctest:: - - >>> custom = configparser.ConfigParser() - >>> custom['section1'] = {'funky': 'nope'} - >>> custom['section1'].getboolean('funky') - Traceback (most recent call last): - ... - ValueError: Not a boolean: nope - >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} - >>> custom['section1'].getboolean('funky') - False - - Other typical Boolean pairs include ``accept``/``reject`` or - ``enabled``/``disabled``. - -.. method:: ConfigParser.optionxform(option) - - This method transforms option names on every read, get, or set - operation. The default converts the name to lowercase. This also - means that when a configuration file gets written, all keys will be - lowercase. Override this method if that's unsuitable. - For example: - - .. doctest:: - - >>> config = """ - ... [Section1] - ... Key = Value - ... - ... [Section2] - ... AnotherKey = Value - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> list(typical['Section1'].keys()) - ['key'] - >>> list(typical['Section2'].keys()) - ['anotherkey'] - >>> custom = configparser.RawConfigParser() - >>> custom.optionxform = lambda option: option - >>> custom.read_string(config) - >>> list(custom['Section1'].keys()) - ['Key'] - >>> list(custom['Section2'].keys()) - ['AnotherKey'] - -.. attribute:: ConfigParser.SECTCRE - - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: - - .. doctest:: - - >>> config = """ - ... [Section 1] - ... option = value - ... - ... [ Section 2 ] - ... another = val - ... """ - >>> typical = ConfigParser() - >>> typical.read_string(config) - >>> typical.sections() - ['Section 1', ' Section 2 '] - >>> custom = ConfigParser() - >>> custom.SECTCRE = re.compile(r"\[ *(?P

    [^]]+?) *\]") - >>> custom.read_string(config) - >>> custom.sections() - ['Section 1', 'Section 2'] - - .. note:: - - While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing - option lines, it's not recommended to override it because that would - interfere with constructor options *allow_no_value* and *delimiters*. - - -Legacy API Examples -------------------- - -Mainly because of backwards compatibility concerns, :mod:`configparser` -provides also a legacy API with explicit ``get``/``set`` methods. While there -are valid use cases for the methods outlined below, mapping protocol access is -preferred for new projects. The legacy API is at times more advanced, -low-level and downright counterintuitive. - -An example of writing to a configuration file:: - - import configparser - - config = configparser.RawConfigParser() - - # Please note that using RawConfigParser's set functions, you can assign - # non-string values to keys internally, but will receive an error when - # attempting to write to a file or when you get it in non-raw mode. Setting - # values using the mapping protocol or ConfigParser's set() does not allow - # such assignments to take place. - config.add_section('Section1') - config.set('Section1', 'an_int', '15') - config.set('Section1', 'a_bool', 'true') - config.set('Section1', 'a_float', '3.1415') - config.set('Section1', 'baz', 'fun') - config.set('Section1', 'bar', 'Python') - config.set('Section1', 'foo', '%(bar)s is %(baz)s!') - - # Writing our configuration file to 'example.cfg' - with open('example.cfg', 'w') as configfile: - config.write(configfile) - -An example of reading the configuration file again:: - - import configparser - - config = configparser.RawConfigParser() - config.read('example.cfg') - - # getfloat() raises an exception if the value is not a float - # getint() and getboolean() also do this for their respective types - a_float = config.getfloat('Section1', 'a_float') - an_int = config.getint('Section1', 'an_int') - print(a_float + an_int) - - # Notice that the next output does not interpolate '%(bar)s' or '%(baz)s'. - # This is because we are using a RawConfigParser(). - if config.getboolean('Section1', 'a_bool'): - print(config.get('Section1', 'foo')) - -To get interpolation, use :class:`ConfigParser`:: - - import configparser - - cfg = configparser.ConfigParser() - cfg.read('example.cfg') - - # Set the optional *raw* argument of get() to True if you wish to disable - # interpolation in a single get operation. - print(cfg.get('Section1', 'foo', raw=False)) # -> "Python is fun!" - print(cfg.get('Section1', 'foo', raw=True)) # -> "%(bar)s is %(baz)s!" - - # The optional *vars* argument is a dict with members that will take - # precedence in interpolation. - print(cfg.get('Section1', 'foo', vars={'bar': 'Documentation', - 'baz': 'evil'})) - - # The optional *fallback* argument can be used to provide a fallback value - print(cfg.get('Section1', 'foo')) - # -> "Python is fun!" - - print(cfg.get('Section1', 'foo', fallback='Monty is not.')) - # -> "Python is fun!" - - print(cfg.get('Section1', 'monster', fallback='No such things as monsters.')) - # -> "No such things as monsters." - - # A bare print(cfg.get('Section1', 'monster')) would raise NoOptionError - # but we can also use: - - print(cfg.get('Section1', 'monster', fallback=None)) - # -> None - -Default values are available in both types of ConfigParsers. They are used in -interpolation if an option used is not defined elsewhere. :: - - import configparser - - # New instance with 'bar' and 'baz' defaulting to 'Life' and 'hard' each - config = configparser.ConfigParser({'bar': 'Life', 'baz': 'hard'}) - config.read('example.cfg') - - print(config.get('Section1', 'foo')) # -> "Python is fun!" - config.remove_option('Section1', 'bar') - config.remove_option('Section1', 'baz') - print(config.get('Section1', 'foo')) # -> "Life is hard!" - - -.. _configparser-objects: - -ConfigParser Objects --------------------- - -.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) - - The main configuration parser. When *defaults* is given, it is initialized - into the dictionary of intrinsic defaults. When *dict_type* is given, it - will be used to create the dictionary objects for the list of sections, for - the options within a section, and for the default values. - - When *delimiters* is given, it is used as the set of substrings that - divide keys from values. When *comment_prefixes* is given, it will be used - as the set of substrings that prefix comments in otherwise empty lines. - Comments can be indented. When *inline_comment_prefixes* is given, it will - be used as the set of substrings that prefix comments in non-empty lines. - - When *strict* is ``True`` (the default), the parser won't allow for - any section or option duplicates while reading from a single source (file, - string or dictionary), raising :exc:`DuplicateSectionError` or - :exc:`DuplicateOptionError`. When *empty_lines_in_values* is ``False`` - (default: ``True``), each empty line marks the end of an option. Otherwise, - internal empty lines of a multiline option are kept as part of the value. - When *allow_no_value* is ``True`` (default: ``False``), options without - values are accepted; the value held for these is ``None`` and they are - serialized without the trailing delimiter. - - When *default_section* is given, it specifies the name for the special - section holding default values for other sections and interpolation purposes - (normally named ``"DEFAULT"``). This value can be retrieved and changed on - runtime using the ``default_section`` instance attribute. - - Interpolation behaviour may be customized by providing a custom handler - through the *interpolation* argument. ``None`` can be used to turn off - interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the - `dedicated documentation section <#interpolation-of-values>`_. - - All option names used in interpolation will be passed through the - :meth:`optionxform` method just like any other option name reference. For - example, using the default implementation of :meth:`optionxform` (which - converts option names to lower case), the values ``foo %(bar)s`` and ``foo - %(BAR)s`` are equivalent. - - When *converters* is given, it should be a dictionary where each key - represents the name of a type converter and each value is a callable - implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser - object and section proxies. - - .. versionchanged:: 3.1 - The default *dict_type* is :class:`collections.OrderedDict`. - - .. versionchanged:: 3.2 - *allow_no_value*, *delimiters*, *comment_prefixes*, *strict*, - *empty_lines_in_values*, *default_section* and *interpolation* were - added. - - .. versionchanged:: 3.5 - The *converters* argument was added. - - - .. method:: defaults() - - Return a dictionary containing the instance-wide defaults. - - - .. method:: sections() - - Return a list of the sections available; the *default section* is not - included in the list. - - - .. method:: add_section(section) - - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. The name - of the section must be a string; if not, :exc:`TypeError` is raised. - - .. versionchanged:: 3.2 - Non-string section names raise :exc:`TypeError`. - - - .. method:: has_section(section) - - Indicates whether the named *section* is present in the configuration. - The *default section* is not acknowledged. - - - .. method:: options(section) - - Return a list of options available in the specified *section*. - - - .. method:: has_option(section, option) - - If the given *section* exists, and contains the given *option*, return - :const:`True`; otherwise return :const:`False`. If the specified - *section* is :const:`None` or an empty string, DEFAULT is assumed. - - - .. method:: read(filenames, encoding=None) - - Attempt to read and parse an iterable of filenames, returning a list of - filenames which were successfully parsed. - - If *filenames* is a string or :term:`path-like object`, it is treated as - a single filename. If a file named in *filenames* cannot be opened, that - file will be ignored. This is designed so that you can specify an - iterable of potential configuration file locations (for example, the - current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the iterable will be - read. - - If none of the named files exist, the :class:`ConfigParser` - instance will contain an empty dataset. An application which requires - initial values to be loaded from a file should load the required file or - files using :meth:`read_file` before calling :meth:`read` for any - optional files:: - - import configparser, os - - config = configparser.ConfigParser() - config.read_file(open('defaults.cfg')) - config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')], - encoding='cp1250') - - .. versionadded:: 3.2 - The *encoding* parameter. Previously, all files were read using the - default encoding for :func:`open`. - - .. versionadded:: 3.6.1 - The *filenames* parameter accepts a :term:`path-like object`. - - - .. method:: read_file(f, source=None) - - Read and parse configuration data from *f* which must be an iterable - yielding Unicode strings (for example files opened in text mode). - - Optional argument *source* specifies the name of the file being read. If - not given and *f* has a :attr:`name` attribute, that is used for - *source*; the default is ``''``. - - .. versionadded:: 3.2 - Replaces :meth:`readfp`. - - .. method:: read_string(string, source='') - - Parse configuration data from a string. - - Optional argument *source* specifies a context-specific name of the - string passed. If not given, ``''`` is used. This should - commonly be a filesystem path or a URL. - - .. versionadded:: 3.2 - - - .. method:: read_dict(dictionary, source='') - - Load configuration from any object that provides a dict-like ``items()`` - method. Keys are section names, values are dictionaries with keys and - values that should be present in the section. If the used dictionary - type preserves order, sections and their keys will be added in order. - Values are automatically converted to strings. - - Optional argument *source* specifies a context-specific name of the - dictionary passed. If not given, ```` is used. - - This method can be used to copy state between parsers. - - .. versionadded:: 3.2 - - - .. method:: get(section, option, *, raw=False, vars=None[, fallback]) - - Get an *option* value for the named *section*. If *vars* is provided, it - must be a dictionary. The *option* is looked up in *vars* (if provided), - *section*, and in *DEFAULTSECT* in that order. If the key is not found - and *fallback* is provided, it is used as a fallback value. ``None`` can - be provided as a *fallback* value. - - All the ``'%'`` interpolations are expanded in the return values, unless - the *raw* argument is true. Values for interpolation keys are looked up - in the same manner as the option. - - .. versionchanged:: 3.2 - Arguments *raw*, *vars* and *fallback* are keyword only to protect - users from trying to use the third argument as the *fallback* fallback - (especially when using the mapping protocol). - - - .. method:: getint(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to an integer. See :meth:`get` for explanation of *raw*, *vars* and - *fallback*. - - - .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to a floating point number. See :meth:`get` for explanation of *raw*, - *vars* and *fallback*. - - - .. method:: getboolean(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to a Boolean value. Note that the accepted values for the option are - ``'1'``, ``'yes'``, ``'true'``, and ``'on'``, which cause this method to - return ``True``, and ``'0'``, ``'no'``, ``'false'``, and ``'off'``, which - cause it to return ``False``. These string values are checked in a - case-insensitive manner. Any other value will cause it to raise - :exc:`ValueError`. See :meth:`get` for explanation of *raw*, *vars* and - *fallback*. - - - .. method:: items(raw=False, vars=None) - items(section, raw=False, vars=None) - - When *section* is not given, return a list of *section_name*, - *section_proxy* pairs, including DEFAULTSECT. - - Otherwise, return a list of *name*, *value* pairs for the options in the - given *section*. Optional arguments have the same meaning as for the - :meth:`get` method. - - - .. method:: set(section, option, value) - - If the given section exists, set the given option to the specified value; - otherwise raise :exc:`NoSectionError`. *option* and *value* must be - strings; if not, :exc:`TypeError` is raised. - - - .. method:: write(fileobject, space_around_delimiters=True) - - Write a representation of the configuration to the specified :term:`file - object`, which must be opened in text mode (accepting strings). This - representation can be parsed by a future :meth:`read` call. If - *space_around_delimiters* is true, delimiters between - keys and values are surrounded by spaces. - - - .. method:: remove_option(section, option) - - Remove the specified *option* from the specified *section*. If the - section does not exist, raise :exc:`NoSectionError`. If the option - existed to be removed, return :const:`True`; otherwise return - :const:`False`. - - - .. method:: remove_section(section) - - Remove the specified *section* from the configuration. If the section in - fact existed, return ``True``. Otherwise return ``False``. - - - .. method:: optionxform(option) - - Transforms the option name *option* as found in an input file or as passed - in by client code to the form that should be used in the internal - structures. The default implementation returns a lower-case version of - *option*; subclasses may override this or client code can set an attribute - of this name on instances to affect this behavior. - - You don't need to subclass the parser to use this method, you can also - set it on an instance, to a function that takes a string argument and - returns a string. Setting it to ``str``, for example, would make option - names case sensitive:: - - cfgparser = ConfigParser() - cfgparser.optionxform = str - - Note that when reading configuration files, whitespace around the option - names is stripped before :meth:`optionxform` is called. - - - .. method:: readfp(fp, filename=None) - - .. deprecated:: 3.2 - Use :meth:`read_file` instead. - - .. versionchanged:: 3.2 - :meth:`readfp` now iterates on *fp* instead of calling ``fp.readline()``. - - For existing code calling :meth:`readfp` with arguments which don't - support iteration, the following generator may be used as a wrapper - around the file-like object:: - - def readline_generator(fp): - line = fp.readline() - while line: - yield line - line = fp.readline() - - Instead of ``parser.readfp(fp)`` use - ``parser.read_file(readline_generator(fp))``. - - -.. data:: MAX_INTERPOLATION_DEPTH - - The maximum depth for recursive interpolation for :meth:`get` when the *raw* - parameter is false. This is relevant only when the default *interpolation* - is used. - - -.. _rawconfigparser-objects: - -RawConfigParser Objects ------------------------ - -.. class:: RawConfigParser(defaults=None, dict_type=collections.OrderedDict, \ - allow_no_value=False, *, delimiters=('=', ':'), \ - comment_prefixes=('#', ';'), \ - inline_comment_prefixes=None, strict=True, \ - empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) - - Legacy variant of the :class:`ConfigParser` with interpolation disabled - by default and unsafe ``add_section`` and ``set`` methods. - - .. note:: - Consider using :class:`ConfigParser` instead which checks types of - the values to be stored internally. If you don't want interpolation, you - can use ``ConfigParser(interpolation=None)``. - - - .. method:: add_section(section) - - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. - - Type of *section* is not checked which lets users create non-string named - sections. This behaviour is unsupported and may cause internal errors. - - - .. method:: set(section, option, value) - - If the given section exists, set the given option to the specified value; - otherwise raise :exc:`NoSectionError`. While it is possible to use - :class:`RawConfigParser` (or :class:`ConfigParser` with *raw* parameters - set to true) for *internal* storage of non-string values, full - functionality (including interpolation and output to files) can only be - achieved using string values. - - This method lets users assign non-string values to keys internally. This - behaviour is unsupported and will cause errors when attempting to write - to a file or get it in non-raw mode. **Use the mapping protocol API** - which does not allow such assignments to take place. - - -Exceptions ----------- - -.. exception:: Error - - Base class for all other :mod:`configparser` exceptions. - - -.. exception:: NoSectionError - - Exception raised when a specified section is not found. - - -.. exception:: DuplicateSectionError - - Exception raised if :meth:`add_section` is called with the name of a section - that is already present or in strict parsers when a section if found more - than once in a single input file, string or dictionary. - - .. versionadded:: 3.2 - Optional ``source`` and ``lineno`` attributes and arguments to - :meth:`__init__` were added. - - -.. exception:: DuplicateOptionError - - Exception raised by strict parsers if a single option appears twice during - reading from a single file, string or dictionary. This catches misspellings - and case sensitivity-related errors, e.g. a dictionary may have two keys - representing the same case-insensitive configuration key. - - -.. exception:: NoOptionError - - Exception raised when a specified option is not found in the specified - section. - - -.. exception:: InterpolationError - - Base class for exceptions raised when problems occur performing string - interpolation. - - -.. exception:: InterpolationDepthError - - Exception raised when string interpolation cannot be completed because the - number of iterations exceeds :const:`MAX_INTERPOLATION_DEPTH`. Subclass of - :exc:`InterpolationError`. - - -.. exception:: InterpolationMissingOptionError - - Exception raised when an option referenced from a value does not exist. - Subclass of :exc:`InterpolationError`. - - -.. exception:: InterpolationSyntaxError - - Exception raised when the source text into which substitutions are made does - not conform to the required syntax. Subclass of :exc:`InterpolationError`. - - -.. exception:: MissingSectionHeaderError - - Exception raised when attempting to parse a file which has no section - headers. - - -.. exception:: ParsingError - - Exception raised when errors occur attempting to parse a file. - - .. versionchanged:: 3.2 - The ``filename`` attribute and :meth:`__init__` argument were renamed to - ``source`` for consistency. - - -.. rubric:: Footnotes - -.. [1] Config parsers allow for heavy customization. If you are interested in - changing the behaviour outlined by the footnote reference, consult the - `Customizing Parser Behaviour`_ section. - diff --git a/third_party/python/Doc/library/constants.rst b/third_party/python/Doc/library/constants.rst deleted file mode 100644 index 634ff0003..000000000 --- a/third_party/python/Doc/library/constants.rst +++ /dev/null @@ -1,100 +0,0 @@ -.. _built-in-consts: - -Built-in Constants -================== - -A small number of constants live in the built-in namespace. They are: - -.. data:: False - - The false value of the :class:`bool` type. Assignments to ``False`` - are illegal and raise a :exc:`SyntaxError`. - - -.. data:: True - - The true value of the :class:`bool` type. Assignments to ``True`` - are illegal and raise a :exc:`SyntaxError`. - - -.. data:: None - - The sole value of the type ``NoneType``. ``None`` is frequently used to - represent the absence of a value, as when default arguments are not passed to a - function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - - -.. data:: NotImplemented - - Special value which should be returned by the binary special methods - (e.g. :meth:`__eq__`, :meth:`__lt__`, :meth:`__add__`, :meth:`__rsub__`, - etc.) to indicate that the operation is not implemented with respect to - the other type; may be returned by the in-place binary special methods - (e.g. :meth:`__imul__`, :meth:`__iand__`, etc.) for the same purpose. - Its truth value is true. - - .. note:: - - When a binary (or in-place) method returns ``NotImplemented`` the - interpreter will try the reflected operation on the other type (or some - other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. - - See :ref:`implementing-the-arithmetic-operations` for examples. - - .. note:: - - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, - even though they have similar names and purposes. - See :exc:`NotImplementedError` for details on when to use it. - - -.. index:: single: ...; ellipsis literal -.. data:: Ellipsis - - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. - - -.. data:: __debug__ - - This constant is true if Python was not started with an :option:`-O` option. - See also the :keyword:`assert` statement. - - -.. note:: - - The names :data:`None`, :data:`False`, :data:`True` and :data:`__debug__` - cannot be reassigned (assignments to them, even as an attribute name, raise - :exc:`SyntaxError`), so they can be considered "true" constants. - - -Constants added by the :mod:`site` module ------------------------------------------ - -The :mod:`site` module (which is imported automatically during startup, except -if the :option:`-S` command-line option is given) adds several constants to the -built-in namespace. They are useful for the interactive interpreter shell and -should not be used in programs. - -.. data:: quit(code=None) - exit(code=None) - - Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the - specified exit code. - -.. data:: copyright - credits - - Objects that when printed or called, print the text of copyright or - credits, respectively. - -.. data:: license - - Object that when printed, prints the message "Type license() to see the - full license text", and when called, displays the full license text in a - pager-like fashion (one screen at a time). - diff --git a/third_party/python/Doc/library/contextlib.rst b/third_party/python/Doc/library/contextlib.rst deleted file mode 100644 index 49eaba3d0..000000000 --- a/third_party/python/Doc/library/contextlib.rst +++ /dev/null @@ -1,777 +0,0 @@ -:mod:`contextlib` --- Utilities for :keyword:`with`\ -statement contexts -======================================================================== - -.. module:: contextlib - :synopsis: Utilities for with-statement contexts. - -**Source code:** :source:`Lib/contextlib.py` - --------------- - -This module provides utilities for common tasks involving the :keyword:`with` -statement. For more information see also :ref:`typecontextmanager` and -:ref:`context-managers`. - - -Utilities ---------- - -Functions and classes provided: - -.. class:: AbstractContextManager - - An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default - returns ``None``. See also the definition of :ref:`typecontextmanager`. - - .. versionadded:: 3.6 - - - -.. decorator:: contextmanager - - This function is a :term:`decorator` that can be used to define a factory - function for :keyword:`with` statement context managers, without needing to - create a class or separate :meth:`__enter__` and :meth:`__exit__` methods. - - While many objects natively support use in with statements, sometimes a - resource needs to be managed that isn't a context manager in its own right, - and doesn't implement a ``close()`` method for use with ``contextlib.closing`` - - An abstract example would be the following to ensure correct resource - management:: - - from contextlib import contextmanager - - @contextmanager - def managed_resource(*args, **kwds): - # Code to acquire resource, e.g.: - resource = acquire_resource(*args, **kwds) - try: - yield resource - finally: - # Code to release resource, e.g.: - release_resource(resource) - - >>> with managed_resource(timeout=3600) as resource: - ... # Resource is released at the end of this block, - ... # even if code in the block raises an exception - - The function being decorated must return a :term:`generator`-iterator when - called. This iterator must yield exactly one value, which will be bound to - the targets in the :keyword:`with` statement's :keyword:`as` clause, if any. - - At the point where the generator yields, the block nested in the :keyword:`with` - statement is executed. The generator is then resumed after the block is exited. - If an unhandled exception occurs in the block, it is reraised inside the - generator at the point where the yield occurred. Thus, you can use a - :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` statement to trap - the error (if any), or ensure that some cleanup takes place. If an exception is - trapped merely in order to log it or to perform some action (rather than to - suppress it entirely), the generator must reraise that exception. Otherwise the - generator context manager will indicate to the :keyword:`with` statement that - the exception has been handled, and execution will resume with the statement - immediately following the :keyword:`with` statement. - - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers - it creates can be used as decorators as well as in :keyword:`with` statements. - When used as a decorator, a new generator instance is implicitly created on - each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context - managers support multiple invocations in order to be used as decorators). - - .. versionchanged:: 3.2 - Use of :class:`ContextDecorator`. - - -.. function:: closing(thing) - - Return a context manager that closes *thing* upon completion of the block. This - is basically equivalent to:: - - from contextlib import contextmanager - - @contextmanager - def closing(thing): - try: - yield thing - finally: - thing.close() - - And lets you write code like this:: - - from contextlib import closing - from urllib.request import urlopen - - with closing(urlopen('http://www.python.org')) as page: - for line in page: - print(line) - - without needing to explicitly close ``page``. Even if an error occurs, - ``page.close()`` will be called when the :keyword:`with` block is exited. - - -.. function:: suppress(*exceptions) - - Return a context manager that suppresses any of the specified exceptions - if they occur in the body of a with statement and then resumes execution - with the first statement following the end of the with statement. - - As with any other mechanism that completely suppresses exceptions, this - context manager should be used only to cover very specific errors where - silently continuing with program execution is known to be the right - thing to do. - - For example:: - - from contextlib import suppress - - with suppress(FileNotFoundError): - os.remove('somefile.tmp') - - with suppress(FileNotFoundError): - os.remove('someotherfile.tmp') - - This code is equivalent to:: - - try: - os.remove('somefile.tmp') - except FileNotFoundError: - pass - - try: - os.remove('someotherfile.tmp') - except FileNotFoundError: - pass - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stdout(new_target) - - Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. - - This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. - - For example, the output of :func:`help` normally is sent to *sys.stdout*. - You can capture that output in a string by redirecting the output to an - :class:`io.StringIO` object:: - - f = io.StringIO() - with redirect_stdout(f): - help(pow) - s = f.getvalue() - - To send the output of :func:`help` to a file on disk, redirect the output - to a regular file:: - - with open('help.txt', 'w') as f: - with redirect_stdout(f): - help(pow) - - To send the output of :func:`help` to *sys.stderr*:: - - with redirect_stdout(sys.stderr): - help(pow) - - Note that the global side effect on :data:`sys.stdout` means that this - context manager is not suitable for use in library code and most threaded - applications. It also has no effect on the output of subprocesses. - However, it is still a useful approach for many utility scripts. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stderr(new_target) - - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.5 - - -.. class:: ContextDecorator() - - A base class that enables a context manager to also be used as a decorator. - - Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional - exception handling even when used as a decorator. - - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this - functionality automatically. - - Example of ``ContextDecorator``:: - - from contextlib import ContextDecorator - - class mycontext(ContextDecorator): - def __enter__(self): - print('Starting') - return self - - def __exit__(self, *exc): - print('Finishing') - return False - - >>> @mycontext() - ... def function(): - ... print('The bit in the middle') - ... - >>> function() - Starting - The bit in the middle - Finishing - - >>> with mycontext(): - ... print('The bit in the middle') - ... - Starting - The bit in the middle - Finishing - - This change is just syntactic sugar for any construct of the following form:: - - def f(): - with cm(): - # Do stuff - - ``ContextDecorator`` lets you instead write:: - - @cm() - def f(): - # Do stuff - - It makes it clear that the ``cm`` applies to the whole function, rather than - just a piece of it (and saving an indentation level is nice, too). - - Existing context managers that already have a base class can be extended by - using ``ContextDecorator`` as a mixin class:: - - from contextlib import ContextDecorator - - class mycontext(ContextBaseClass, ContextDecorator): - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - .. note:: - As the decorated function must be able to be called multiple times, the - underlying context manager must support use in multiple :keyword:`with` - statements. If this is not the case, then the original construct with the - explicit :keyword:`with` statement inside the function should be used. - - .. versionadded:: 3.2 - - -.. class:: ExitStack() - - A context manager that is designed to make it easy to programmatically - combine other context managers and cleanup functions, especially those - that are optional or otherwise driven by input data. - - For example, a set of files may easily be handled in a single with - statement as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception - - Each instance maintains a stack of registered callbacks that are called in - reverse order when the instance is closed (either explicitly or implicitly - at the end of a :keyword:`with` statement). Note that callbacks are *not* - invoked implicitly when the context stack instance is garbage collected. - - This stack model is used so that context managers that acquire their - resources in their ``__init__`` method (such as file objects) can be - handled correctly. - - Since registered callbacks are invoked in the reverse order of - registration, this ends up behaving as if multiple nested :keyword:`with` - statements had been used with the registered set of callbacks. This even - extends to exception handling - if an inner callback suppresses or replaces - an exception, then outer callbacks will be passed arguments based on that - updated state. - - This is a relatively low level API that takes care of the details of - correctly unwinding the stack of exit callbacks. It provides a suitable - foundation for higher level context managers that manipulate the exit - stack in application specific ways. - - .. versionadded:: 3.3 - - .. method:: enter_context(cm) - - Enters a new context manager and adds its :meth:`__exit__` method to - the callback stack. The return value is the result of the context - manager's own :meth:`__enter__` method. - - These context managers may suppress exceptions just as they normally - would if used directly as part of a :keyword:`with` statement. - - .. method:: push(exit) - - Adds a context manager's :meth:`__exit__` method to the callback stack. - - As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`__enter__` implementation with a context manager's own - :meth:`__exit__` method. - - If passed an object that is not a context manager, this method assumes - it is a callback with the same signature as a context manager's - :meth:`__exit__` method and adds it directly to the callback stack. - - By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`__exit__` methods can. - - The passed in object is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: callback(callback, *args, **kwds) - - Accepts an arbitrary callback function and arguments and adds it to - the callback stack. - - Unlike the other methods, callbacks added this way cannot suppress - exceptions (as they are never passed the exception details). - - The passed in callback is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: pop_all() - - Transfers the callback stack to a fresh :class:`ExitStack` instance - and returns it. No callbacks are invoked by this operation - instead, - they will now be invoked when the new stack is closed (either - explicitly or implicitly at the end of a :keyword:`with` statement). - - For example, a group of files can be opened as an "all or nothing" - operation as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # Hold onto the close method, but don't call it yet. - close_files = stack.pop_all().close - # If opening any file fails, all previously opened files will be - # closed automatically. If all files are opened successfully, - # they will remain open even after the with statement ends. - # close_files() can then be invoked explicitly to close them all. - - .. method:: close() - - Immediately unwinds the callback stack, invoking callbacks in the - reverse order of registration. For any context managers and exit - callbacks registered, the arguments passed in will indicate that no - exception occurred. - - -Examples and Recipes --------------------- - -This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. - - -Supporting a variable number of context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The primary use case for :class:`ExitStack` is the one given in the class -documentation: supporting a variable number of context managers and other -cleanup operations in a single :keyword:`with` statement. The variability -may come from the number of context managers needed being driven by user -input (such as opening a user specified collection of files), or from -some of the context managers being optional:: - - with ExitStack() as stack: - for resource in resources: - stack.enter_context(resource) - if need_special_resource(): - special = acquire_special_resource() - stack.callback(release_special_resource, special) - # Perform operations that use the acquired resources - -As shown, :class:`ExitStack` also makes it quite easy to use :keyword:`with` -statements to manage arbitrary resources that don't natively support the -context management protocol. - - -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - -Catching exceptions from ``__enter__`` methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is occasionally desirable to catch exceptions from an ``__enter__`` -method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` -method. By using :class:`ExitStack` the steps in the context management -protocol can be separated slightly in order to allow this:: - - stack = ExitStack() - try: - x = stack.enter_context(cm) - except Exception: - # handle __enter__ exception - else: - with stack: - # Handle normal case - -Actually needing to do this is likely to indicate that the underlying API -should be providing a direct resource management interface for use with -:keyword:`try`/:keyword:`except`/:keyword:`finally` statements, but not -all APIs are well designed in that regard. When a context manager is the -only resource management API provided, then :class:`ExitStack` can make it -easier to handle various situations that can't be handled directly in a -:keyword:`with` statement. - - -Cleaning up in an ``__enter__`` implementation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As noted in the documentation of :meth:`ExitStack.push`, this -method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`__enter__` implementation fail. - -Here's an example of doing this for a context manager that accepts resource -acquisition and release functions, along with an optional validation function, -and maps them to the context management protocol:: - - from contextlib import contextmanager, AbstractContextManager, ExitStack - - class ResourceManager(AbstractContextManager): - - def __init__(self, acquire_resource, release_resource, check_resource_ok=None): - self.acquire_resource = acquire_resource - self.release_resource = release_resource - if check_resource_ok is None: - def check_resource_ok(resource): - return True - self.check_resource_ok = check_resource_ok - - @contextmanager - def _cleanup_on_error(self): - with ExitStack() as stack: - stack.push(self) - yield - # The validation check passed and didn't raise an exception - # Accordingly, we want to keep the resource, and pass it - # back to our caller - stack.pop_all() - - def __enter__(self): - resource = self.acquire_resource() - with self._cleanup_on_error(): - if not self.check_resource_ok(resource): - msg = "Failed validation for {!r}" - raise RuntimeError(msg.format(resource)) - return resource - - def __exit__(self, *exc_details): - # We don't need to duplicate any of our resource release logic - self.release_resource() - - -Replacing any use of ``try-finally`` and flag variables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A pattern you will sometimes see is a ``try-finally`` statement with a flag -variable to indicate whether or not the body of the ``finally`` clause should -be executed. In its simplest form (that can't already be handled just by -using an ``except`` clause instead), it looks something like this:: - - cleanup_needed = True - try: - result = perform_operation() - if result: - cleanup_needed = False - finally: - if cleanup_needed: - cleanup_resources() - -As with any ``try`` statement based code, this can cause problems for -development and review, because the setup code and the cleanup code can end -up being separated by arbitrarily long sections of code. - -:class:`ExitStack` makes it possible to instead register a callback for -execution at the end of a ``with`` statement, and then later decide to skip -executing that callback:: - - from contextlib import ExitStack - - with ExitStack() as stack: - stack.callback(cleanup_resources) - result = perform_operation() - if result: - stack.pop_all() - -This allows the intended cleanup up behaviour to be made explicit up front, -rather than requiring a separate flag variable. - -If a particular application uses this pattern a lot, it can be simplified -even further by means of a small helper class:: - - from contextlib import ExitStack - - class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): - super(Callback, self).__init__() - self.callback(callback, *args, **kwds) - - def cancel(self): - self.pop_all() - - with Callback(cleanup_resources) as cb: - result = perform_operation() - if result: - cb.cancel() - -If the resource cleanup isn't already neatly bundled into a standalone -function, then it is still possible to use the decorator form of -:meth:`ExitStack.callback` to declare the resource cleanup in -advance:: - - from contextlib import ExitStack - - with ExitStack() as stack: - @stack.callback - def cleanup_resources(): - ... - result = perform_operation() - if result: - stack.pop_all() - -Due to the way the decorator protocol works, a callback function -declared this way cannot take any parameters. Instead, any resources to -be released must be accessed as closure variables. - - -Using a context manager as a function decorator -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`ContextDecorator` makes it possible to use a context manager in -both an ordinary ``with`` statement and also as a function decorator. - -For example, it is sometimes useful to wrap functions or groups of statements -with a logger that can track the time of entry and time of exit. Rather than -writing both a function decorator and a context manager for the task, -inheriting from :class:`ContextDecorator` provides both capabilities in a -single definition:: - - from contextlib import ContextDecorator - import logging - - logging.basicConfig(level=logging.INFO) - - class track_entry_and_exit(ContextDecorator): - def __init__(self, name): - self.name = name - - def __enter__(self): - logging.info('Entering: %s', self.name) - - def __exit__(self, exc_type, exc, exc_tb): - logging.info('Exiting: %s', self.name) - -Instances of this class can be used as both a context manager:: - - with track_entry_and_exit('widget loader'): - print('Some time consuming activity goes here') - load_widget() - -And also as a function decorator:: - - @track_entry_and_exit('widget loader') - def activity(): - print('Some time consuming activity goes here') - load_widget() - -Note that there is one additional limitation when using context managers -as function decorators: there's no way to access the return value of -:meth:`__enter__`. If that value is needed, then it is still necessary to use -an explicit ``with`` statement. - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - -.. _single-use-reusable-and-reentrant-cms: - -Single use, reusable and reentrant context managers ---------------------------------------------------- - -Most context managers are written in a way that means they can only be -used effectively in a :keyword:`with` statement once. These single use -context managers must be created afresh each time they're used - -attempting to use them a second time will trigger an exception or -otherwise not work correctly. - -This common limitation means that it is generally advisable to create -context managers directly in the header of the :keyword:`with` statement -where they are used (as shown in all of the usage examples above). - -Files are an example of effectively single use context managers, since -the first :keyword:`with` statement will close the file, preventing any -further IO operations using that file object. - -Context managers created using :func:`contextmanager` are also single use -context managers, and will complain about the underlying generator failing -to yield if an attempt is made to use them a second time:: - - >>> from contextlib import contextmanager - >>> @contextmanager - ... def singleuse(): - ... print("Before") - ... yield - ... print("After") - ... - >>> cm = singleuse() - >>> with cm: - ... pass - ... - Before - After - >>> with cm: - ... pass - ... - Traceback (most recent call last): - ... - RuntimeError: generator didn't yield - - -.. _reentrant-cms: - -Reentrant context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -More sophisticated context managers may be "reentrant". These context -managers can not only be used in multiple :keyword:`with` statements, -but may also be used *inside* a :keyword:`with` statement that is already -using the same context manager. - -:class:`threading.RLock` is an example of a reentrant context manager, as are -:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of -reentrant use:: - - >>> from contextlib import redirect_stdout - >>> from io import StringIO - >>> stream = StringIO() - >>> write_to_stream = redirect_stdout(stream) - >>> with write_to_stream: - ... print("This is written to the stream rather than stdout") - ... with write_to_stream: - ... print("This is also written to the stream") - ... - >>> print("This is written directly to stdout") - This is written directly to stdout - >>> print(stream.getvalue()) - This is written to the stream rather than stdout - This is also written to the stream - -Real world examples of reentrancy are more likely to involve multiple -functions calling each other and hence be far more complicated than this -example. - -Note also that being reentrant is *not* the same thing as being thread safe. -:func:`redirect_stdout`, for example, is definitely not thread safe, as it -makes a global modification to the system state by binding :data:`sys.stdout` -to a different stream. - - -.. _reusable-cms: - -Reusable context managers -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Distinct from both single use and reentrant context managers are "reusable" -context managers (or, to be completely explicit, "reusable, but not -reentrant" context managers, since reentrant context managers are also -reusable). These context managers support being used multiple times, but -will fail (or otherwise not work correctly) if the specific context manager -instance has already been used in a containing with statement. - -:class:`threading.Lock` is an example of a reusable, but not reentrant, -context manager (for a reentrant lock, it is necessary to use -:class:`threading.RLock` instead). - -Another example of a reusable, but not reentrant, context manager is -:class:`ExitStack`, as it invokes *all* currently registered callbacks -when leaving any with statement, regardless of where those callbacks -were added:: - - >>> from contextlib import ExitStack - >>> stack = ExitStack() - >>> with stack: - ... stack.callback(print, "Callback: from first context") - ... print("Leaving first context") - ... - Leaving first context - Callback: from first context - >>> with stack: - ... stack.callback(print, "Callback: from second context") - ... print("Leaving second context") - ... - Leaving second context - Callback: from second context - >>> with stack: - ... stack.callback(print, "Callback: from outer context") - ... with stack: - ... stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Callback: from outer context - Leaving outer context - -As the output from the example shows, reusing a single stack object across -multiple with statements works correctly, but attempting to nest them -will cause the stack to be cleared at the end of the innermost with -statement, which is unlikely to be desirable behaviour. - -Using separate :class:`ExitStack` instances instead of reusing a single -instance avoids that problem:: - - >>> from contextlib import ExitStack - >>> with ExitStack() as outer_stack: - ... outer_stack.callback(print, "Callback: from outer context") - ... with ExitStack() as inner_stack: - ... inner_stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Leaving outer context - Callback: from outer context diff --git a/third_party/python/Doc/library/copy.rst b/third_party/python/Doc/library/copy.rst deleted file mode 100644 index c7bd89f96..000000000 --- a/third_party/python/Doc/library/copy.rst +++ /dev/null @@ -1,95 +0,0 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ - -.. module:: copy - :synopsis: Shallow and deep copy operations. - -**Source code:** :source:`Lib/copy.py` - --------------- - -Assignment statements in Python do not copy objects, they create bindings -between a target and an object. For collections that are mutable or contain -mutable items, a copy is sometimes needed so one can change one copy without -changing the other. This module provides generic shallow and deep copy -operations (explained below). - - -Interface summary: - -.. function:: copy(x) - - Return a shallow copy of *x*. - - -.. function:: deepcopy(x[, memo]) - - Return a deep copy of *x*. - - -.. exception:: error - - Raised for module specific errors. - - -The difference between shallow and deep copying is only relevant for compound -objects (objects that contain other objects, like lists or class instances): - -* A *shallow copy* constructs a new compound object and then (to the extent - possible) inserts *references* into it to the objects found in the original. - -* A *deep copy* constructs a new compound object and then, recursively, inserts - *copies* into it of the objects found in the original. - -Two problems often exist with deep copy operations that don't exist with shallow -copy operations: - -* Recursive objects (compound objects that, directly or indirectly, contain a - reference to themselves) may cause a recursive loop. - -* Because deep copy copies everything it may copy too much, such as data - which is intended to be shared between copies. - -The :func:`deepcopy` function avoids these problems by: - -* keeping a ``memo`` dictionary of objects already copied during the current - copying pass; and - -* letting user-defined classes override the copying operation or the set of - components copied. - -This module does not copy types like module, method, stack trace, stack frame, -file, socket, window, array, or any similar types. It does "copy" functions and -classes (shallow and deeply), by returning the original object unchanged; this -is compatible with the way these are treated by the :mod:`pickle` module. - -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. - -.. index:: module: pickle - -Classes can use the same interfaces to control copying that they use to control -pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered -pickle functions from the :mod:`copyreg` module. - -.. index:: - single: __copy__() (copy protocol) - single: __deepcopy__() (copy protocol) - -In order for a class to define its own copy implementation, it can define -special methods :meth:`__copy__` and :meth:`__deepcopy__`. The former is called -to implement the shallow copy operation; no additional arguments are passed. -The latter is called to implement the deep copy operation; it is passed one -argument, the ``memo`` dictionary. If the :meth:`__deepcopy__` implementation needs -to make a deep copy of a component, it should call the :func:`deepcopy` function -with the component as first argument and the memo dictionary as second argument. - - -.. seealso:: - - Module :mod:`pickle` - Discussion of the special methods used to support object state retrieval and - restoration. - diff --git a/third_party/python/Doc/library/copyreg.rst b/third_party/python/Doc/library/copyreg.rst deleted file mode 100644 index eeabb4c65..000000000 --- a/third_party/python/Doc/library/copyreg.rst +++ /dev/null @@ -1,65 +0,0 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== - -.. module:: copyreg - :synopsis: Register pickle support functions. - -**Source code:** :source:`Lib/copyreg.py` - -.. index:: - module: pickle - module: copy - --------------- - -The :mod:`copyreg` module offers a way to define functions used while pickling -specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions -when pickling/copying those objects. The module provides configuration -information about object constructors which are not classes. -Such constructors may be factory functions or class instances. - - -.. function:: constructor(object) - - Declares *object* to be a valid constructor. If *object* is not callable (and - hence not valid as a constructor), raises :exc:`TypeError`. - - -.. function:: pickle(type, function, constructor=None) - - Declares that *function* should be used as a "reduction" function for objects - of type *type*. *function* should return either a string or a tuple - containing two or three elements. - - The optional *constructor* parameter, if provided, is a callable object which - can be used to reconstruct the object when called with the tuple of arguments - returned by *function* at pickling time. :exc:`TypeError` will be raised if - *object* is a class or *constructor* is not callable. - - See the :mod:`pickle` module for more details on the interface - expected of *function* and *constructor*. Note that the - :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler - object or subclass of :class:`pickle.Pickler` can also be used for - declaring reduction functions. - -Example -------- - -The example below would like to show how to register a pickle function and how -it will be used: - - >>> import copyreg, copy, pickle - >>> class C(object): - ... def __init__(self, a): - ... self.a = a - ... - >>> def pickle_c(c): - ... print("pickling a C instance...") - ... return C, (c.a,) - ... - >>> copyreg.pickle(C, pickle_c) - >>> c = C(1) - >>> d = copy.copy(c) - pickling a C instance... - >>> p = pickle.dumps(c) - pickling a C instance... diff --git a/third_party/python/Doc/library/crypt.rst b/third_party/python/Doc/library/crypt.rst deleted file mode 100644 index bccc76b82..000000000 --- a/third_party/python/Doc/library/crypt.rst +++ /dev/null @@ -1,156 +0,0 @@ -:mod:`crypt` --- Function to check Unix passwords -================================================= - -.. module:: crypt - :platform: Unix - :synopsis: The crypt() function used to check Unix passwords. - -.. moduleauthor:: Steven D. Majewski -.. sectionauthor:: Steven D. Majewski -.. sectionauthor:: Peter Funk - -**Source code:** :source:`Lib/crypt.py` - -.. index:: - single: crypt(3) - pair: cipher; DES - --------------- - -This module implements an interface to the :manpage:`crypt(3)` routine, which is -a one-way hash function based upon a modified DES algorithm; see the Unix man -page for further details. Possible uses include storing hashed passwords -so you can check passwords without storing the actual password, or attempting -to crack Unix passwords with a dictionary. - -.. index:: single: crypt(3) - -Notice that the behavior of this module depends on the actual implementation of -the :manpage:`crypt(3)` routine in the running system. Therefore, any -extensions available on the current implementation will also be available on -this module. - -Hashing Methods ---------------- - -.. versionadded:: 3.3 - -The :mod:`crypt` module defines the list of hashing methods (not all methods -are available on all platforms): - -.. data:: METHOD_SHA512 - - A Modular Crypt Format method with 16 character salt and 86 character - hash. This is the strongest method. - -.. data:: METHOD_SHA256 - - Another Modular Crypt Format method with 16 character salt and 43 - character hash. - -.. data:: METHOD_MD5 - - Another Modular Crypt Format method with 8 character salt and 22 - character hash. - -.. data:: METHOD_CRYPT - - The traditional method with a 2 character salt and 13 characters of - hash. This is the weakest method. - - -Module Attributes ------------------ - -.. versionadded:: 3.3 - -.. attribute:: methods - - A list of available password hashing algorithms, as - ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest. - - -Module Functions ----------------- - -The :mod:`crypt` module defines the following functions: - -.. function:: crypt(word, salt=None) - - *word* will usually be a user's password as typed at a prompt or in a graphical - interface. The optional *salt* is either a string as returned from - :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all - may be available on all platforms), or a full encrypted password - including salt, as returned by this function. If *salt* is not - provided, the strongest method will be used (as returned by - :func:`methods`). - - Checking a password is usually done by passing the plain-text password - as *word* and the full results of a previous :func:`crypt` call, - which should be the same as the results of this call. - - *salt* (either a random 2 or 16 character string, possibly prefixed with - ``$digit$`` to indicate the method) which will be used to perturb the - encryption algorithm. The characters in *salt* must be in the set - ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which - prefixes a ``$digit$``. - - Returns the hashed password as a string, which will be composed of - characters from the same alphabet as the salt. - - .. index:: single: crypt(3) - - Since a few :manpage:`crypt(3)` extensions allow different values, with - different sizes in the *salt*, it is recommended to use the full crypted - password as salt when checking for a password. - - .. versionchanged:: 3.3 - Accept ``crypt.METHOD_*`` values in addition to strings for *salt*. - - -.. function:: mksalt(method=None) - - Return a randomly generated salt of the specified method. If no - *method* is given, the strongest method available as returned by - :func:`methods` is used. - - The return value is a string either of 2 characters in length for - ``crypt.METHOD_CRYPT``, or 19 characters starting with ``$digit$`` and - 16 random characters from the set ``[./a-zA-Z0-9]``, suitable for - passing as the *salt* argument to :func:`crypt`. - - .. versionadded:: 3.3 - -Examples --------- - -A simple example illustrating typical use (a constant-time comparison -operation is needed to limit exposure to timing attacks. -:func:`hmac.compare_digest` is suitable for this purpose):: - - import pwd - import crypt - import getpass - from hmac import compare_digest as compare_hash - - def login(): - username = input('Python login: ') - cryptedpasswd = pwd.getpwnam(username)[1] - if cryptedpasswd: - if cryptedpasswd == 'x' or cryptedpasswd == '*': - raise ValueError('no support for shadow passwords') - cleartext = getpass.getpass() - return compare_hash(crypt.crypt(cleartext, cryptedpasswd), cryptedpasswd) - else: - return True - -To generate a hash of a password using the strongest available method and -check it against the original:: - - import crypt - from hmac import compare_digest as compare_hash - - hashed = crypt.crypt(plaintext) - if not compare_hash(hashed, crypt.crypt(plaintext, hashed)): - raise ValueError("hashed version doesn't validate against original") diff --git a/third_party/python/Doc/library/crypto.rst b/third_party/python/Doc/library/crypto.rst deleted file mode 100644 index ae45549a6..000000000 --- a/third_party/python/Doc/library/crypto.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _crypto: - -********************** -Cryptographic Services -********************** - -.. index:: single: cryptography - -The modules described in this chapter implement various algorithms of a -cryptographic nature. They are available at the discretion of the installation. -On Unix systems, the :mod:`crypt` module may also be available. -Here's an overview: - - -.. toctree:: - - hashlib.rst - hmac.rst - secrets.rst diff --git a/third_party/python/Doc/library/csv.rst b/third_party/python/Doc/library/csv.rst deleted file mode 100644 index 08b8edc52..000000000 --- a/third_party/python/Doc/library/csv.rst +++ /dev/null @@ -1,552 +0,0 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== - -.. module:: csv - :synopsis: Write and read tabular data to and from delimited files. - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/csv.py` - -.. index:: - single: csv - pair: data; tabular - --------------- - -The so-called CSV (Comma Separated Values) format is the most common import and -export format for spreadsheets and databases. CSV format was used for many -years prior to attempts to describe the format in a standardized way in -:rfc:`4180`. The lack of a well-defined standard means that subtle differences -often exist in the data produced and consumed by different applications. These -differences can make it annoying to process CSV files from multiple sources. -Still, while the delimiters and quoting characters vary, the overall format is -similar enough that it is possible to write a single module which can -efficiently manipulate such data, hiding the details of reading and writing the -data from the programmer. - -The :mod:`csv` module implements classes to read and write tabular data in CSV -format. It allows programmers to say, "write this data in the format preferred -by Excel," or "read data from this file which was generated by Excel," without -knowing the precise details of the CSV format used by Excel. Programmers can -also describe the CSV formats understood by other applications or define their -own special-purpose CSV formats. - -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and -write sequences. Programmers can also read and write data in dictionary form -using the :class:`DictReader` and :class:`DictWriter` classes. - -.. seealso:: - - :pep:`305` - CSV File API - The Python Enhancement Proposal which proposed this addition to Python. - - -.. _csv-contents: - -Module Contents ---------------- - -The :mod:`csv` module defines the following functions: - - -.. index:: - single: universal newlines; csv.reader function - -.. function:: reader(csvfile, dialect='excel', **fmtparams) - - Return a reader object which will iterate over lines in the given *csvfile*. - *csvfile* can be any object which supports the :term:`iterator` protocol and returns a - string each time its :meth:`!__next__` method is called --- :term:`file objects - ` and list objects are both suitable. If *csvfile* is a file object, - it should be opened with ``newline=''``. [1]_ An optional - *dialect* parameter can be given which is used to define a set of parameters - specific to a particular CSV dialect. It may be an instance of a subclass of - the :class:`Dialect` class or one of the strings returned by the - :func:`list_dialects` function. The other optional *fmtparams* keyword arguments - can be given to override individual formatting parameters in the current - dialect. For full details about the dialect and formatting parameters, see - section :ref:`csv-fmt-params`. - - Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format - option is specified (in which case unquoted fields are transformed into floats). - - A short usage example:: - - >>> import csv - >>> with open('eggs.csv', newline='') as csvfile: - ... spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|') - ... for row in spamreader: - ... print(', '.join(row)) - Spam, Spam, Spam, Spam, Spam, Baked Beans - Spam, Lovely Spam, Wonderful Spam - - -.. function:: writer(csvfile, dialect='excel', **fmtparams) - - Return a writer object responsible for converting the user's data into delimited - strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with - ``newline=''`` [1]_. An optional *dialect* - parameter can be given which is used to define a set of parameters specific to a - particular CSV dialect. It may be an instance of a subclass of the - :class:`Dialect` class or one of the strings returned by the - :func:`list_dialects` function. The other optional *fmtparams* keyword arguments - can be given to override individual formatting parameters in the current - dialect. For full details about the dialect and formatting parameters, see - section :ref:`csv-fmt-params`. To make it - as easy as possible to interface with modules which implement the DB API, the - value :const:`None` is written as the empty string. While this isn't a - reversible transformation, it makes it easier to dump SQL NULL data values to - CSV files without preprocessing the data returned from a ``cursor.fetch*`` call. - All other non-string data are stringified with :func:`str` before being written. - - A short usage example:: - - import csv - with open('eggs.csv', 'w', newline='') as csvfile: - spamwriter = csv.writer(csvfile, delimiter=' ', - quotechar='|', quoting=csv.QUOTE_MINIMAL) - spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) - spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) - - -.. function:: register_dialect(name[, dialect[, **fmtparams]]) - - Associate *dialect* with *name*. *name* must be a string. The - dialect can be specified either by passing a sub-class of :class:`Dialect`, or - by *fmtparams* keyword arguments, or both, with keyword arguments overriding - parameters of the dialect. For full details about the dialect and formatting - parameters, see section :ref:`csv-fmt-params`. - - -.. function:: unregister_dialect(name) - - Delete the dialect associated with *name* from the dialect registry. An - :exc:`Error` is raised if *name* is not a registered dialect name. - - -.. function:: get_dialect(name) - - Return the dialect associated with *name*. An :exc:`Error` is raised if - *name* is not a registered dialect name. This function returns an immutable - :class:`Dialect`. - -.. function:: list_dialects() - - Return the names of all registered dialects. - - -.. function:: field_size_limit([new_limit]) - - Returns the current maximum field size allowed by the parser. If *new_limit* is - given, this becomes the new limit. - - -The :mod:`csv` module defines the following classes: - -.. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ - dialect='excel', *args, **kwds) - - Create an object that operates like a regular reader but maps the - information in each row to an :mod:`OrderedDict ` - whose keys are given by the optional *fieldnames* parameter. - - The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is - omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the ordered - dictionary preserves their original ordering. - - If a row has more fields than fieldnames, the remaining data is put in a - list and stored with the fieldname specified by *restkey* (which defaults - to ``None``). If a non-blank row has fewer fields than fieldnames, the - missing values are filled-in with ``None``. - - All other optional or keyword arguments are passed to the underlying - :class:`reader` instance. - - .. versionchanged:: 3.6 - Returned rows are now of type :class:`OrderedDict`. - - A short usage example:: - - >>> import csv - >>> with open('names.csv', newline='') as csvfile: - ... reader = csv.DictReader(csvfile) - ... for row in reader: - ... print(row['first_name'], row['last_name']) - ... - Eric Idle - John Cleese - - >>> print(row) - OrderedDict([('first_name', 'John'), ('last_name', 'Cleese')]) - - -.. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ - dialect='excel', *args, **kwds) - - Create an object which operates like a regular writer but maps dictionaries - onto output rows. The *fieldnames* parameter is a :mod:`sequence - ` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file - *f*. The optional *restval* parameter specifies the value to be - written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in - *fieldnames*, the optional *extrasaction* parameter indicates what action to - take. - If it is set to ``'raise'``, the default value, a :exc:`ValueError` - is raised. - If it is set to ``'ignore'``, extra values in the dictionary are ignored. - Any other optional or keyword arguments are passed to the underlying - :class:`writer` instance. - - Note that unlike the :class:`DictReader` class, the *fieldnames* parameter - of the :class:`DictWriter` is not optional. Since Python's :class:`dict` - objects are not ordered, there is not enough information available to deduce - the order in which the row should be written to file *f*. - - A short usage example:: - - import csv - - with open('names.csv', 'w', newline='') as csvfile: - fieldnames = ['first_name', 'last_name'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - - writer.writeheader() - writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'}) - writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) - writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) - - -.. class:: Dialect - - The :class:`Dialect` class is a container class relied on primarily for its - attributes, which are used to define the parameters for a specific - :class:`reader` or :class:`writer` instance. - - -.. class:: excel() - - The :class:`excel` class defines the usual properties of an Excel-generated CSV - file. It is registered with the dialect name ``'excel'``. - - -.. class:: excel_tab() - - The :class:`excel_tab` class defines the usual properties of an Excel-generated - TAB-delimited file. It is registered with the dialect name ``'excel-tab'``. - - -.. class:: unix_dialect() - - The :class:`unix_dialect` class defines the usual properties of a CSV file - generated on UNIX systems, i.e. using ``'\n'`` as line terminator and quoting - all fields. It is registered with the dialect name ``'unix'``. - - .. versionadded:: 3.2 - - -.. class:: Sniffer() - - The :class:`Sniffer` class is used to deduce the format of a CSV file. - - The :class:`Sniffer` class provides two methods: - - .. method:: sniff(sample, delimiters=None) - - Analyze the given *sample* and return a :class:`Dialect` subclass - reflecting the parameters found. If the optional *delimiters* parameter - is given, it is interpreted as a string containing possible valid - delimiter characters. - - - .. method:: has_header(sample) - - Analyze the sample text (presumed to be in CSV format) and return - :const:`True` if the first row appears to be a series of column headers. - -An example for :class:`Sniffer` use:: - - with open('example.csv', newline='') as csvfile: - dialect = csv.Sniffer().sniff(csvfile.read(1024)) - csvfile.seek(0) - reader = csv.reader(csvfile, dialect) - # ... process CSV file contents here ... - - -The :mod:`csv` module defines the following constants: - -.. data:: QUOTE_ALL - - Instructs :class:`writer` objects to quote all fields. - - -.. data:: QUOTE_MINIMAL - - Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. - - -.. data:: QUOTE_NONNUMERIC - - Instructs :class:`writer` objects to quote all non-numeric fields. - - Instructs the reader to convert all non-quoted fields to type *float*. - - -.. data:: QUOTE_NONE - - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if - any characters that require escaping are encountered. - - Instructs :class:`reader` to perform no special processing of quote characters. - -The :mod:`csv` module defines the following exception: - - -.. exception:: Error - - Raised by any of the functions when an error is detected. - -.. _csv-fmt-params: - -Dialects and Formatting Parameters ----------------------------------- - -To make it easier to specify the format of input and output records, specific -formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or -:class:`writer` objects, the programmer can specify a string or a subclass of -the :class:`Dialect` class as the dialect parameter. In addition to, or instead -of, the *dialect* parameter, the programmer can also specify individual -formatting parameters, which have the same names as the attributes defined below -for the :class:`Dialect` class. - -Dialects support the following attributes: - - -.. attribute:: Dialect.delimiter - - A one-character string used to separate fields. It defaults to ``','``. - - -.. attribute:: Dialect.doublequote - - Controls how instances of *quotechar* appearing inside a field should - themselves be quoted. When :const:`True`, the character is doubled. When - :const:`False`, the *escapechar* is used as a prefix to the *quotechar*. It - defaults to :const:`True`. - - On output, if *doublequote* is :const:`False` and no *escapechar* is set, - :exc:`Error` is raised if a *quotechar* is found in a field. - - -.. attribute:: Dialect.escapechar - - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from - the following character. It defaults to :const:`None`, which disables escaping. - - -.. attribute:: Dialect.lineterminator - - The string used to terminate lines produced by the :class:`writer`. It defaults - to ``'\r\n'``. - - .. note:: - - The :class:`reader` is hard-coded to recognise either ``'\r'`` or ``'\n'`` as - end-of-line, and ignores *lineterminator*. This behavior may change in the - future. - - -.. attribute:: Dialect.quotechar - - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. - - -.. attribute:: Dialect.quoting - - Controls when quotes should be generated by the writer and recognised by the - reader. It can take on any of the :const:`QUOTE_\*` constants (see section - :ref:`csv-contents`) and defaults to :const:`QUOTE_MINIMAL`. - - -.. attribute:: Dialect.skipinitialspace - - When :const:`True`, whitespace immediately following the *delimiter* is ignored. - The default is :const:`False`. - - -.. attribute:: Dialect.strict - - When ``True``, raise exception :exc:`Error` on bad CSV input. - The default is ``False``. - -Reader Objects --------------- - -Reader objects (:class:`DictReader` instances and objects returned by the -:func:`reader` function) have the following public methods: - -.. method:: csvreader.__next__() - - Return the next row of the reader's iterable object as a list (if the object - was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` - instance), parsed according to the current dialect. Usually you should call - this as ``next(reader)``. - - -Reader objects have the following public attributes: - -.. attribute:: csvreader.dialect - - A read-only description of the dialect in use by the parser. - - -.. attribute:: csvreader.line_num - - The number of lines read from the source iterator. This is not the same as the - number of records returned, as records can span multiple lines. - - -DictReader objects have the following public attribute: - -.. attribute:: csvreader.fieldnames - - If not passed as a parameter when creating the object, this attribute is - initialized upon first access or when the first record is read from the - file. - - - -Writer Objects --------------- - -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by -the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary -mapping fieldnames to strings or numbers (by passing them through :func:`str` -first) for :class:`DictWriter` objects. Note that complex numbers are written -out surrounded by parens. This may cause some problems for other programs which -read CSV files (assuming they support complex numbers at all). - - -.. method:: csvwriter.writerow(row) - - Write the *row* parameter to the writer's file object, formatted according to - the current dialect. - - .. versionchanged:: 3.5 - Added support of arbitrary iterables. - -.. method:: csvwriter.writerows(rows) - - Write all elements in *rows* (an iterable of *row* objects as described - above) to the writer's file object, formatted according to the current - dialect. - -Writer objects have the following public attribute: - - -.. attribute:: csvwriter.dialect - - A read-only description of the dialect in use by the writer. - - -DictWriter objects have the following public method: - - -.. method:: DictWriter.writeheader() - - Write a row with the field names (as specified in the constructor). - - .. versionadded:: 3.2 - - -.. _csv-examples: - -Examples --------- - -The simplest example of reading a CSV file:: - - import csv - with open('some.csv', newline='') as f: - reader = csv.reader(f) - for row in reader: - print(row) - -Reading a file with an alternate format:: - - import csv - with open('passwd', newline='') as f: - reader = csv.reader(f, delimiter=':', quoting=csv.QUOTE_NONE) - for row in reader: - print(row) - -The corresponding simplest possible writing example is:: - - import csv - with open('some.csv', 'w', newline='') as f: - writer = csv.writer(f) - writer.writerows(someiterable) - -Since :func:`open` is used to open a CSV file for reading, the file -will by default be decoded into unicode using the system default -encoding (see :func:`locale.getpreferredencoding`). To decode a file -using a different encoding, use the ``encoding`` argument of open:: - - import csv - with open('some.csv', newline='', encoding='utf-8') as f: - reader = csv.reader(f) - for row in reader: - print(row) - -The same applies to writing in something other than the system default -encoding: specify the encoding argument when opening the output file. - -Registering a new dialect:: - - import csv - csv.register_dialect('unixpwd', delimiter=':', quoting=csv.QUOTE_NONE) - with open('passwd', newline='') as f: - reader = csv.reader(f, 'unixpwd') - -A slightly more advanced use of the reader --- catching and reporting errors:: - - import csv, sys - filename = 'some.csv' - with open(filename, newline='') as f: - reader = csv.reader(f) - try: - for row in reader: - print(row) - except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) - -And while the module doesn't directly support parsing strings, it can easily be -done:: - - import csv - for row in csv.reader(['one,two,three']): - print(row) - - -.. rubric:: Footnotes - -.. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings - on write an extra ``\r`` will be added. It should always be safe to specify - ``newline=''``, since the csv module does its own - (:term:`universal `) newline handling. diff --git a/third_party/python/Doc/library/ctypes.rst b/third_party/python/Doc/library/ctypes.rst deleted file mode 100644 index a7cc0c84d..000000000 --- a/third_party/python/Doc/library/ctypes.rst +++ /dev/null @@ -1,2484 +0,0 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= - -.. module:: ctypes - :synopsis: A foreign function library for Python. - -.. moduleauthor:: Thomas Heller - --------------- - -:mod:`ctypes` is a foreign function library for Python. It provides C compatible -data types, and allows calling functions in DLLs or shared libraries. It can be -used to wrap these libraries in pure Python. - - -.. _ctypes-ctypes-tutorial: - -ctypes tutorial ---------------- - -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or Mac OS X, they contain doctest directives in comments. - -Note: Some code samples reference the ctypes :class:`c_int` type. On platforms -where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. -So, you should not be confused if :class:`c_long` is printed if you would expect -:class:`c_int` --- they are actually the same type. - -.. _ctypes-loading-dynamic-link-libraries: - -Loading dynamic link libraries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* -objects, for loading dynamic link libraries. - -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error -code is used to automatically raise an :class:`OSError` exception when the -function call fails. - -.. versionchanged:: 3.3 - Windows errors used to raise :exc:`WindowsError`, which is now an alias - of :exc:`OSError`. - - -Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C -library containing most standard C functions, and uses the cdecl calling -convention:: - - >>> from ctypes import * - >>> print(windll.kernel32) # doctest: +WINDOWS - - >>> print(cdll.msvcrt) # doctest: +WINDOWS - - >>> libc = cdll.msvcrt # doctest: +WINDOWS - >>> - -Windows appends the usual ``.dll`` file suffix automatically. - -.. note:: - Accessing the standard C library through ``cdll.msvcrt`` will use an - outdated version of the library that may be incompatible with the one - being used by Python. Where possible, use native Python functionality, - or else import and use the ``msvcrt`` module. - -On Linux, it is required to specify the filename *including* the extension to -load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: - - >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX - - >>> libc = CDLL("libc.so.6") # doctest: +LINUX - >>> libc # doctest: +LINUX - - >>> - -.. XXX Add section for Mac OS X. - - -.. _ctypes-accessing-functions-from-loaded-dlls: - -Accessing functions from loaded dlls -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Functions are accessed as attributes of dll objects:: - - >>> from ctypes import * - >>> libc.printf - <_FuncPtr object at 0x...> - >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "ctypes.py", line 239, in __getattr__ - func = _StdcallFuncPtr(name, self) - AttributeError: function 'MyOwnFunction' not found - >>> - -Note that win32 system dlls like ``kernel32`` and ``user32`` often export ANSI -as well as UNICODE versions of a function. The UNICODE version is exported with -an ``W`` appended to the name, while the ANSI version is exported with an ``A`` -appended to the name. The win32 ``GetModuleHandle`` function, which returns a -*module handle* for a given module name, has the following C prototype, and a -macro is used to expose one of them as ``GetModuleHandle`` depending on whether -UNICODE is defined or not:: - - /* ANSI version */ - HMODULE GetModuleHandleA(LPCSTR lpModuleName); - /* UNICODE version */ - HMODULE GetModuleHandleW(LPCWSTR lpModuleName); - -*windll* does not try to select one of them by magic, you must access the -version you need by specifying ``GetModuleHandleA`` or ``GetModuleHandleW`` -explicitly, and then call it with bytes or string objects respectively. - -Sometimes, dlls export functions with names which aren't valid Python -identifiers, like ``"??2@YAPAXI@Z"``. In this case you have to use -:func:`getattr` to retrieve the function:: - - >>> getattr(cdll.msvcrt, "??2@YAPAXI@Z") # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> - -On Windows, some dlls export functions not by name but by ordinal. These -functions can be accessed by indexing the dll object with the ordinal number:: - - >>> cdll.kernel32[1] # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> cdll.kernel32[0] # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "ctypes.py", line 310, in __getitem__ - func = _StdcallFuncPtr(name, self) - AttributeError: function ordinal 0 not found - >>> - - -.. _ctypes-calling-functions: - -Calling functions -^^^^^^^^^^^^^^^^^ - -You can call these functions like any other Python callable. This example uses -the ``time()`` function, which returns system time in seconds since the Unix -epoch, and the ``GetModuleHandleA()`` function, which returns a win32 module -handle. - -This example calls both functions with a NULL pointer (``None`` should be used -as the NULL pointer):: - - >>> print(libc.time(None)) # doctest: +SKIP - 1150640792 - >>> print(hex(windll.kernel32.GetModuleHandleA(None))) # doctest: +WINDOWS - 0x1d000000 - >>> - -.. note:: - - :mod:`ctypes` may raise a :exc:`ValueError` after calling the function, if - it detects that an invalid number of arguments were passed. This behavior - should not be relied upon. It is deprecated in 3.6.2, and will be removed - in 3.7. - -:exc:`ValueError` is raised when you call an ``stdcall`` function with the -``cdecl`` calling convention, or vice versa:: - - >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> - - >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> - -To find out the correct calling convention you have to look into the C header -file or the documentation for the function you want to call. - -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent -crashes from general protection faults when functions are called with invalid -argument values:: - - >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - OSError: exception: access violation reading 0x00000020 - >>> - -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). - -``None``, integers, bytes objects and (unicode) strings are the only native -Python objects that can directly be used as parameters in these function calls. -``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed -as pointer to the memory block that contains their data (:c:type:`char *` or -:c:type:`wchar_t *`). Python integers are passed as the platforms default C -:c:type:`int` type, their value is masked to fit into the C type. - -Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. - - -.. _ctypes-fundamental-data-types: - -Fundamental data types -^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` defines a number of primitive C compatible data types: - -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:type:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:type:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:type:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:type:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:type:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:type:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:type:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:type:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:type:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:type:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:type:`__int64` or :c:type:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:type:`unsigned __int64` or | int | -| | :c:type:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:type:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:type:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:type:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:type:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:type:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:type:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:type:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. - -All these types can be created by calling them with an optional initializer of -the correct type and value:: - - >>> c_int() - c_long(0) - >>> c_wchar_p("Hello, World") - c_wchar_p(140018365411392) - >>> c_ushort(-3) - c_ushort(65533) - >>> - -Since these types are mutable, their value can also be changed afterwards:: - - >>> i = c_int(42) - >>> print(i) - c_long(42) - >>> print(i.value) - 42 - >>> i.value = -99 - >>> print(i.value) - -99 - >>> - -Assigning a new value to instances of the pointer types :class:`c_char_p`, -:class:`c_wchar_p`, and :class:`c_void_p` changes the *memory location* they -point to, *not the contents* of the memory block (of course not, because Python -bytes objects are immutable):: - - >>> s = "Hello, World" - >>> c_s = c_wchar_p(s) - >>> print(c_s) - c_wchar_p(139966785747344) - >>> print(c_s.value) - Hello World - >>> c_s.value = "Hi, there" - >>> print(c_s) # the memory location has changed - c_wchar_p(139966783348904) - >>> print(c_s.value) - Hi, there - >>> print(s) # first object is unchanged - Hello, World - >>> - -You should be careful, however, not to pass them to functions expecting pointers -to mutable memory. If you need mutable memory blocks, ctypes has a -:func:`create_string_buffer` function which creates these in various ways. The -current memory block contents can be accessed (or changed) with the ``raw`` -property; if you want to access it as NUL terminated string, use the ``value`` -property:: - - >>> from ctypes import * - >>> p = create_string_buffer(3) # create a 3 byte buffer, initialized to NUL bytes - >>> print(sizeof(p), repr(p.raw)) - 3 b'\x00\x00\x00' - >>> p = create_string_buffer(b"Hello") # create a buffer containing a NUL terminated string - >>> print(sizeof(p), repr(p.raw)) - 6 b'Hello\x00' - >>> print(repr(p.value)) - b'Hello' - >>> p = create_string_buffer(b"Hello", 10) # create a 10 byte buffer - >>> print(sizeof(p), repr(p.raw)) - 10 b'Hello\x00\x00\x00\x00\x00' - >>> p.value = b"Hi" - >>> print(sizeof(p), repr(p.raw)) - 10 b'Hi\x00lo\x00\x00\x00\x00\x00' - >>> - -The :func:`create_string_buffer` function replaces the :func:`c_buffer` function -(which is still available as an alias), as well as the :func:`c_string` function -from earlier ctypes releases. To create a mutable memory block containing -unicode characters of the C type :c:type:`wchar_t` use the -:func:`create_unicode_buffer` function. - - -.. _ctypes-calling-functions-continued: - -Calling functions, continued -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that printf prints to the real standard output channel, *not* to -:data:`sys.stdout`, so these examples will only work at the console prompt, not -from within *IDLE* or *PythonWin*:: - - >>> printf = libc.printf - >>> printf(b"Hello, %s\n", b"World!") - Hello, World! - 14 - >>> printf(b"Hello, %S\n", "World!") - Hello, World! - 14 - >>> printf(b"%d bottles of beer\n", 42) - 42 bottles of beer - 19 - >>> printf(b"%f bottles of beer\n", 42.5) - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: Don't know how to convert parameter 2 - >>> - -As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so -that they can be converted to the required C data type:: - - >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) - An int 1234, a double 3.140000 - 31 - >>> - - -.. _ctypes-calling-functions-with-own-custom-data-types: - -Calling functions with your own custom data types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: - - >>> class Bottles: - ... def __init__(self, number): - ... self._as_parameter_ = number - ... - >>> bottles = Bottles(42) - >>> printf(b"%d bottles of beer\n", bottles) - 42 bottles of beer - 19 - >>> - -If you don't want to store the instance's data in the :attr:`_as_parameter_` -instance variable, you could define a :class:`property` which makes the -attribute available on request. - - -.. _ctypes-specifying-required-argument-types: - -Specifying the required argument types (function prototypes) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. - -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is -probably not a good example here, because it takes a variable number and -different types of parameters depending on the format string, on the other hand -this is quite handy to experiment with this feature):: - - >>> printf.argtypes = [c_char_p, c_char_p, c_int, c_double] - >>> printf(b"String '%s', Int %d, Double %f\n", b"Hi", 10, 2.2) - String 'Hi', Int 10, Double 2.200000 - 37 - >>> - -Specifying a format protects against incompatible argument types (just as a -prototype for a C function), and tries to convert the arguments to valid types:: - - >>> printf(b"%d %d %d", 1, 2, 3) - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: wrong type - >>> printf(b"%s %d %f\n", b"X", 2, 3) - X 2 3.000000 - 13 - >>> - -If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives -the Python object passed to the function call, it should do a typecheck or -whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to -pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. - - -.. _ctypes-return-types: - -Return types -^^^^^^^^^^^^ - -By default functions are assumed to return the C :c:type:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the -function object. - -Here is a more advanced example, it uses the ``strchr`` function, which expects -a string pointer and a char, and returns a pointer to a string:: - - >>> strchr = libc.strchr - >>> strchr(b"abcdef", ord("d")) # doctest: +SKIP - 8059983 - >>> strchr.restype = c_char_p # c_char_p is a pointer to a string - >>> strchr(b"abcdef", ord("d")) - b'def' - >>> print(strchr(b"abcdef", ord("x"))) - None - >>> - -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a -single character Python bytes object into a C char:: - - >>> strchr.restype = c_char_p - >>> strchr.argtypes = [c_char_p, c_char] - >>> strchr(b"abcdef", b"d") - 'def' - >>> strchr(b"abcdef", b"def") - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: exceptions.TypeError: one character string expected - >>> print(strchr(b"abcdef", b"x")) - None - >>> strchr(b"abcdef", b"d") - 'def' - >>> - -You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The -callable will be called with the *integer* the C function returns, and the -result of this call will be used as the result of your function call. This is -useful to check for error return values and automatically raise an exception:: - - >>> GetModuleHandle = windll.kernel32.GetModuleHandleA # doctest: +WINDOWS - >>> def ValidHandle(value): - ... if value == 0: - ... raise WinError() - ... return value - ... - >>> - >>> GetModuleHandle.restype = ValidHandle # doctest: +WINDOWS - >>> GetModuleHandle(None) # doctest: +WINDOWS - 486539264 - >>> GetModuleHandle("something silly") # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "", line 3, in ValidHandle - OSError: [Errno 126] The specified module could not be found. - >>> - -``WinError`` is a function which will call Windows ``FormatMessage()`` api to -get the string representation of an error code, and *returns* an exception. -``WinError`` takes an optional error code parameter, if no one is used, it calls -:func:`GetLastError` to retrieve it. - -Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. - - -.. _ctypes-passing-pointers: - -Passing pointers (or: passing parameters by reference) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Sometimes a C api function expects a *pointer* to a data type as parameter, -probably to write into the corresponding location, or if the data is too large -to be passed by value. This is also known as *passing parameters by reference*. - -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters -by reference. The same effect can be achieved with the :func:`pointer` function, -although :func:`pointer` does a lot more work since it constructs a real pointer -object, so it is faster to use :func:`byref` if you don't need the pointer -object in Python itself:: - - >>> i = c_int() - >>> f = c_float() - >>> s = create_string_buffer(b'\000' * 32) - >>> print(i.value, f.value, repr(s.value)) - 0 0.0 b'' - >>> libc.sscanf(b"1 3.14 Hello", b"%d %f %s", - ... byref(i), byref(f), s) - 3 - >>> print(i.value, f.value, repr(s.value)) - 1 3.1400001049 b'Hello' - >>> - - -.. _ctypes-structures-unions: - -Structures and unions -^^^^^^^^^^^^^^^^^^^^^ - -Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of -*2-tuples*, containing a *field name* and a *field type*. - -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. - -Here is a simple example of a POINT structure, which contains two integers named -*x* and *y*, and also shows how to initialize a structure in the constructor:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = [("x", c_int), - ... ("y", c_int)] - ... - >>> point = POINT(10, 20) - >>> print(point.x, point.y) - 10 20 - >>> point = POINT(y=5) - >>> print(point.x, point.y) - 0 5 - >>> POINT(1, 2, 3) - Traceback (most recent call last): - File "", line 1, in - ValueError: too many initializers - >>> - -You can, however, build much more complicated structures. A structure can -itself contain other structures by using a structure as a field type. - -Here is a RECT structure which contains two POINTs named *upperleft* and -*lowerright*:: - - >>> class RECT(Structure): - ... _fields_ = [("upperleft", POINT), - ... ("lowerright", POINT)] - ... - >>> rc = RECT(point) - >>> print(rc.upperleft.x, rc.upperleft.y) - 0 5 - >>> print(rc.lowerright.x, rc.lowerright.y) - 0 0 - >>> - -Nested structures can also be initialized in the constructor in several ways:: - - >>> r = RECT(POINT(1, 2), POINT(3, 4)) - >>> r = RECT((1, 2), (3, 4)) - -Field :term:`descriptor`\s can be retrieved from the *class*, they are useful -for debugging because they can provide useful information:: - - >>> print(POINT.x) - - >>> print(POINT.y) - - >>> - - -.. _ctypes-structureunion-alignment-byte-order: - -.. warning:: - - :mod:`ctypes` does not support passing unions or structures with bit-fields - to functions by value. While this may work on 32-bit x86, it's not - guaranteed by the library to work in the general case. Unions and - structures with bit-fields should always be passed to functions by pointer. - -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior be specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. - -:mod:`ctypes` uses the native byte order for Structures and Unions. To build -structures with non-native byte order, you can use one of the -:class:`BigEndianStructure`, :class:`LittleEndianStructure`, -:class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These -classes cannot contain pointer fields. - - -.. _ctypes-bit-fields-in-structures-unions: - -Bit fields in structures and unions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to create structures and unions containing bit fields. Bit fields -are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: - - >>> class Int(Structure): - ... _fields_ = [("first_16", c_int, 16), - ... ("second_16", c_int, 16)] - ... - >>> print(Int.first_16) - - >>> print(Int.second_16) - - >>> - - -.. _ctypes-arrays: - -Arrays -^^^^^^ - -Arrays are sequences, containing a fixed number of instances of the same type. - -The recommended way to create array types is by multiplying a data type with a -positive integer:: - - TenPointsArrayType = POINT * 10 - -Here is an example of a somewhat artificial data type, a structure containing 4 -POINTs among other stuff:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = ("x", c_int), ("y", c_int) - ... - >>> class MyStruct(Structure): - ... _fields_ = [("a", c_int), - ... ("b", c_float), - ... ("point_array", POINT * 4)] - >>> - >>> print(len(MyStruct().point_array)) - 4 - >>> - -Instances are created in the usual way, by calling the class:: - - arr = TenPointsArrayType() - for pt in arr: - print(pt.x, pt.y) - -The above code print a series of ``0 0`` lines, because the array contents is -initialized to zeros. - -Initializers of the correct type can also be specified:: - - >>> from ctypes import * - >>> TenIntegers = c_int * 10 - >>> ii = TenIntegers(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - >>> print(ii) - - >>> for i in ii: print(i, end=" ") - ... - 1 2 3 4 5 6 7 8 9 10 - >>> - - -.. _ctypes-pointers: - -Pointers -^^^^^^^^ - -Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: - - >>> from ctypes import * - >>> i = c_int(42) - >>> pi = pointer(i) - >>> - -Pointer instances have a :attr:`~_Pointer.contents` attribute which -returns the object to which the pointer points, the ``i`` object above:: - - >>> pi.contents - c_long(42) - >>> - -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a -new, equivalent object each time you retrieve an attribute:: - - >>> pi.contents is i - False - >>> pi.contents is pi.contents - False - >>> - -Assigning another :class:`c_int` instance to the pointer's contents attribute -would cause the pointer to point to the memory location where this is stored:: - - >>> i = c_int(99) - >>> pi.contents = i - >>> pi.contents - c_long(99) - >>> - -.. XXX Document dereferencing pointers, and that it is preferred over the - .contents attribute. - -Pointer instances can also be indexed with integers:: - - >>> pi[0] - 99 - >>> - -Assigning to an integer index changes the pointed to value:: - - >>> print(i) - c_long(99) - >>> pi[0] = 22 - >>> print(i) - c_long(22) - >>> - -It is also possible to use indexes different from 0, but you must know what -you're doing, just as in C: You can access or change arbitrary memory locations. -Generally you only use this feature if you receive a pointer from a C function, -and you *know* that the pointer actually points to an array instead of a single -item. - -Behind the scenes, the :func:`pointer` function does more than simply create -pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a -new type:: - - >>> PI = POINTER(c_int) - >>> PI - - >>> PI(42) - Traceback (most recent call last): - File "", line 1, in - TypeError: expected c_long instead of int - >>> PI(c_int(42)) - - >>> - -Calling the pointer type without an argument creates a ``NULL`` pointer. -``NULL`` pointers have a ``False`` boolean value:: - - >>> null_ptr = POINTER(c_int)() - >>> print(bool(null_ptr)) - False - >>> - -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing -invalid non-\ ``NULL`` pointers would crash Python):: - - >>> null_ptr[0] - Traceback (most recent call last): - .... - ValueError: NULL pointer access - >>> - - >>> null_ptr[0] = 1234 - Traceback (most recent call last): - .... - ValueError: NULL pointer access - >>> - - -.. _ctypes-type-conversions: - -Type conversions -^^^^^^^^^^^^^^^^ - -Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of -a member field in a structure definition, only instances of exactly the same -type are accepted. There are some exceptions to this rule, where ctypes accepts -other objects. For example, you can pass compatible array instances instead of -pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: - - >>> class Bar(Structure): - ... _fields_ = [("count", c_int), ("values", POINTER(c_int))] - ... - >>> bar = Bar() - >>> bar.values = (c_int * 3)(1, 2, 3) - >>> bar.count = 3 - >>> for i in range(bar.count): - ... print(bar.values[i]) - ... - 1 - 2 - 3 - >>> - -In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed -type (``c_int`` in this case) can be passed to the function. ctypes will apply -the required :func:`byref` conversion in this case automatically. - -To set a POINTER type field to ``NULL``, you can assign ``None``:: - - >>> bar.values = None - >>> - -.. XXX list other conversions... - -Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be -used in the same way. The ``Bar`` structure defined above accepts -``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, -but not instances of other types:: - - >>> bar.values = (c_byte * 4)() - Traceback (most recent call last): - File "", line 1, in - TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance - >>> - -For these cases, the :func:`cast` function is handy. - -The :func:`cast` function can be used to cast a ctypes instance into a pointer -to a different ctypes data type. :func:`cast` takes two parameters, a ctypes -object that is or can be converted to a pointer of some kind, and a ctypes -pointer type. It returns an instance of the second argument, which references -the same memory block as the first argument:: - - >>> a = (c_byte * 4)() - >>> cast(a, POINTER(c_int)) - - >>> - -So, :func:`cast` can be used to assign to the ``values`` field of ``Bar`` the -structure:: - - >>> bar = Bar() - >>> bar.values = cast((c_byte * 4)(), POINTER(c_int)) - >>> print(bar.values[0]) - 0 - >>> - - -.. _ctypes-incomplete-types: - -Incomplete Types -^^^^^^^^^^^^^^^^ - -*Incomplete Types* are structures, unions or arrays whose members are not yet -specified. In C, they are specified by forward declarations, which are defined -later:: - - struct cell; /* forward declaration */ - - struct cell { - char *name; - struct cell *next; - }; - -The straightforward translation into ctypes code would be this, but it does not -work:: - - >>> class cell(Structure): - ... _fields_ = [("name", c_char_p), - ... ("next", POINTER(cell))] - ... - Traceback (most recent call last): - File "", line 1, in - File "", line 2, in cell - NameError: name 'cell' is not defined - >>> - -because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: - - >>> from ctypes import * - >>> class cell(Structure): - ... pass - ... - >>> cell._fields_ = [("name", c_char_p), - ... ("next", POINTER(cell))] - >>> - -Lets try it. We create two instances of ``cell``, and let them point to each -other, and finally follow the pointer chain a few times:: - - >>> c1 = cell() - >>> c1.name = "foo" - >>> c2 = cell() - >>> c2.name = "bar" - >>> c1.next = pointer(c2) - >>> c2.next = pointer(c1) - >>> p = c1 - >>> for i in range(8): - ... print(p.name, end=" ") - ... p = p.next[0] - ... - foo bar foo bar foo bar foo bar - >>> - - -.. _ctypes-callback-functions: - -Callback functions -^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` allows creating C callable function pointers from Python callables. -These are sometimes called *callback functions*. - -First, you must create a class for the callback function. The class knows the -calling convention, the return type, and the number and types of arguments this -function will receive. - -The :func:`CFUNCTYPE` factory function creates types for callback functions -using the ``cdecl`` calling convention. On Windows, the :func:`WINFUNCTYPE` -factory function creates types for callback functions using the ``stdcall`` -calling convention. - -Both of these factory functions are called with the result type as first -argument, and the callback functions expected argument types as the remaining -arguments. - -I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: - - >>> IntArray5 = c_int * 5 - >>> ia = IntArray5(5, 1, 7, 33, 99) - >>> qsort = libc.qsort - >>> qsort.restype = None - >>> - -:func:`qsort` must be called with a pointer to the data to sort, the number of -items in the data array, the size of one item, and a pointer to the comparison -function, the callback. The callback will then be called with two pointers to -items, and it must return a negative integer if the first item is smaller than -the second, a zero if they are equal, and a positive integer otherwise. - -So our callback function receives pointers to integers, and must return an -integer. First we create the ``type`` for the callback function:: - - >>> CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) - >>> - -To get started, here is a simple callback that shows the values it gets -passed:: - - >>> def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return 0 - ... - >>> cmp_func = CMPFUNC(py_cmp_func) - >>> - -The result:: - - >>> qsort(ia, len(ia), sizeof(c_int), cmp_func) # doctest: +LINUX - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 5 7 - py_cmp_func 1 7 - >>> - -Now we can actually compare the two items and return a useful result:: - - >>> def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return a[0] - b[0] - ... - >>> - >>> qsort(ia, len(ia), sizeof(c_int), CMPFUNC(py_cmp_func)) # doctest: +LINUX - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 1 7 - py_cmp_func 5 7 - >>> - -As we can easily check, our array is sorted now:: - - >>> for i in ia: print(i, end=" ") - ... - 1 5 7 33 99 - >>> - -The function factories can be used as decorator factories, so we may as well -write:: - - >>> @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) - ... def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return a[0] - b[0] - ... - >>> qsort(ia, len(ia), sizeof(c_int), py_cmp_func) - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 1 7 - py_cmp_func 5 7 - >>> - -.. note:: - - Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be - garbage collected, crashing your program when a callback is made. - - Also, note that if the callback function is called in a thread created - outside of Python's control (e.g. by the foreign code that calls the - callback), ctypes creates a new dummy Python thread on every invocation. This - behavior is correct for most purposes, but it means that values stored with - :class:`threading.local` will *not* survive across different callbacks, even when - those calls are made from the same C thread. - -.. _ctypes-accessing-values-exported-from-dlls: - -Accessing values exported from dlls -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some shared libraries not only export functions, they also export variables. An -example in the Python library itself is the :c:data:`Py_OptimizeFlag`, an integer -set to 0, 1, or 2, depending on the :option:`-O` or :option:`-OO` flag given on -startup. - -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of -the type. *pythonapi* is a predefined symbol giving access to the Python C -api:: - - >>> opt_flag = c_int.in_dll(pythonapi, "Py_OptimizeFlag") - >>> print(opt_flag) - c_long(0) - >>> - -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - -An extended example which also demonstrates the use of pointers accesses the -:c:data:`PyImport_FrozenModules` pointer exported by Python. - -Quoting the docs for that value: - - This pointer is initialized to point to an array of :c:type:`struct _frozen` - records, terminated by one whose members are all *NULL* or zero. When a frozen - module is imported, it is searched in this table. Third-party code could play - tricks with this to provide a dynamically created collection of frozen modules. - -So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: - - >>> from ctypes import * - >>> - >>> class struct_frozen(Structure): - ... _fields_ = [("name", c_char_p), - ... ("code", POINTER(c_ubyte)), - ... ("size", c_int)] - ... - >>> - -We have defined the :c:type:`struct _frozen` data type, so we can get the pointer -to the table:: - - >>> FrozenTable = POINTER(struct_frozen) - >>> table = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules") - >>> - -Since ``table`` is a ``pointer`` to the array of ``struct_frozen`` records, we -can iterate over it, but we just have to make sure that our loop terminates, -because pointers have no size. Sooner or later it would probably crash with an -access violation or whatever, so it's better to break out of the loop when we -hit the NULL entry:: - - >>> for item in table: - ... if item.name is None: - ... break - ... print(item.name.decode("ascii"), item.size) - ... - _frozen_importlib 31764 - _frozen_importlib_external 41499 - __hello__ 161 - __phello__ -161 - __phello__.spam 161 - >>> - -The fact that standard Python has a frozen module and a frozen package -(indicated by the negative size member) is not well known, it is only used for -testing. Try it out with ``import __hello__`` for example. - - -.. _ctypes-surprises: - -Surprises -^^^^^^^^^ - -There are some edges in :mod:`ctypes` where you might expect something other -than what actually happens. - -Consider the following example:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = ("x", c_int), ("y", c_int) - ... - >>> class RECT(Structure): - ... _fields_ = ("a", POINT), ("b", POINT) - ... - >>> p1 = POINT(1, 2) - >>> p2 = POINT(3, 4) - >>> rc = RECT(p1, p2) - >>> print(rc.a.x, rc.a.y, rc.b.x, rc.b.y) - 1 2 3 4 - >>> # now swap the two points - >>> rc.a, rc.b = rc.b, rc.a - >>> print(rc.a.x, rc.a.y, rc.b.x, rc.b.y) - 3 4 3 4 - >>> - -Hm. We certainly expected the last statement to print ``3 4 1 2``. What -happened? Here are the steps of the ``rc.a, rc.b = rc.b, rc.a`` line above:: - - >>> temp0, temp1 = rc.b, rc.a - >>> rc.a = temp0 - >>> rc.b = temp1 - >>> - -Note that ``temp0`` and ``temp1`` are objects still using the internal buffer of -the ``rc`` object above. So executing ``rc.a = temp0`` copies the buffer -contents of ``temp0`` into ``rc`` 's buffer. This, in turn, changes the -contents of ``temp1``. So, the last assignment ``rc.b = temp1``, doesn't have -the expected effect. - -Keep in mind that retrieving sub-objects from Structure, Unions, and Arrays -doesn't *copy* the sub-object, instead it retrieves a wrapper object accessing -the root-object's underlying buffer. - -Another example that may behave different from what one would expect is this:: - - >>> s = c_char_p() - >>> s.value = "abc def ghi" - >>> s.value - 'abc def ghi' - >>> s.value is s.value - False - >>> - -Why is it printing ``False``? ctypes instances are objects containing a memory -block plus some :term:`descriptor`\s accessing the contents of the memory. -Storing a Python object in the memory block does not store the object itself, -instead the ``contents`` of the object is stored. Accessing the contents again -constructs a new Python object each time! - - -.. _ctypes-variable-sized-data-types: - -Variable-sized data types -^^^^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` provides some support for variable-sized arrays and structures. - -The :func:`resize` function can be used to resize the memory buffer of an -existing ctypes object. The function takes the object as first argument, and -the requested size in bytes as the second argument. The memory block cannot be -made smaller than the natural memory block specified by the objects type, a -:exc:`ValueError` is raised if this is tried:: - - >>> short_array = (c_short * 4)() - >>> print(sizeof(short_array)) - 8 - >>> resize(short_array, 4) - Traceback (most recent call last): - ... - ValueError: minimum size is 8 - >>> resize(short_array, 32) - >>> sizeof(short_array) - 32 - >>> sizeof(type(short_array)) - 8 - >>> - -This is nice and fine, but how would one access the additional elements -contained in this array? Since the type still only knows about 4 elements, we -get errors accessing other elements:: - - >>> short_array[:] - [0, 0, 0, 0] - >>> short_array[7] - Traceback (most recent call last): - ... - IndexError: invalid index - >>> - -Another way to use variable-sized data types with :mod:`ctypes` is to use the -dynamic nature of Python, and (re-)define the data type after the required size -is already known, on a case by case basis. - - -.. _ctypes-ctypes-reference: - -ctypes reference ----------------- - - -.. _ctypes-finding-shared-libraries: - -Finding shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -When programming in a compiled language, shared libraries are accessed when -compiling/linking a program, and when the program is run. - -The purpose of the :func:`find_library` function is to locate a library in a way -similar to what the compiler or runtime loader does (on platforms with several -versions of a shared library the most recent should be loaded), while the ctypes -library loaders act like when a program is run, and call the runtime loader -directly. - -The :mod:`ctypes.util` module provides a function which can help to determine -the library to load. - - -.. data:: find_library(name) - :module: ctypes.util - :noindex: - - Try to find a library and return a pathname. *name* is the library name without - any prefix like *lib*, suffix like ``.so``, ``.dylib`` or version number (this - is the form used for the posix linker option :option:`!-l`). If no library can - be found, returns ``None``. - -The exact functionality is system dependent. - -On Linux, :func:`find_library` tries to run external programs -(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. -It returns the filename of the library file. - -.. versionchanged:: 3.6 - On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used - when searching for libraries, if a library cannot be found by any other means. - -Here are some examples:: - - >>> from ctypes.util import find_library - >>> find_library("m") - 'libm.so.6' - >>> find_library("c") - 'libc.so.6' - >>> find_library("bz2") - 'libbz2.so.1.0' - >>> - -On OS X, :func:`find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: - - >>> from ctypes.util import find_library - >>> find_library("c") - '/usr/lib/libc.dylib' - >>> find_library("m") - '/usr/lib/libm.dylib' - >>> find_library("bz2") - '/usr/lib/libbz2.dylib' - >>> find_library("AGL") - '/System/Library/Frameworks/AGL.framework/AGL' - >>> - -On Windows, :func:`find_library` searches along the system search path, and -returns the full pathname, but since there is no predefined naming scheme a call -like ``find_library("c")`` will fail and return ``None``. - -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. - - -.. _ctypes-loading-shared-libraries: - -Loading shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: - - -.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False) - - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:type:`int`. - - -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False) - - Windows only: Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` - values contain information specifying whether the function call failed or - succeeded, together with additional error code. If the return value signals a - failure, an :class:`OSError` is automatically raised. - - .. versionchanged:: 3.3 - :exc:`WindowsError` used to be raised. - - -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False) - - Windows only: Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return :c:type:`int` by default. - - On Windows CE only the standard calling convention is used, for convenience the - :class:`WinDLL` and :class:`OleDLL` use the standard calling convention on this - platform. - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the - Python GIL is *not* released during the function call, and after the function - execution the Python error flag is checked. If the error flag is set, a Python - exception is raised. - - Thus, this is only useful to call Python C api functions directly. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` -function is used to load the library into the process, and to get a handle to -it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -.. data:: RTLD_GLOBAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this flag is not available, - it is defined as the integer zero. - - -.. data:: RTLD_LOCAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this is not available, it - is the same as *RTLD_GLOBAL*. - - -.. data:: DEFAULT_MODE - :noindex: - - The default mode which is used to load shared libraries. On OSX 10.3, this is - *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. - -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time: - - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. - -Shared libraries can also be loaded by using one of the prefabricated objects, -which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. - - -.. class:: LibraryLoader(dlltype) - - Class which loads shared libraries. *dlltype* should be one of the - :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - - :meth:`__getattr__` has special behavior: It allows loading a shared library by - accessing it as attribute of a library loader instance. The result is cached, - so repeated attribute accesses return the same library each time. - - .. method:: LoadLibrary(name) - - Load a shared library into the process and return it. This method always - returns a new instance of the library. - - -These prefabricated library loaders are available: - -.. data:: cdll - :noindex: - - Creates :class:`CDLL` instances. - - -.. data:: windll - :noindex: - - Windows only: Creates :class:`WinDLL` instances. - - -.. data:: oledll - :noindex: - - Windows only: Creates :class:`OleDLL` instances. - - -.. data:: pydll - :noindex: - - Creates :class:`PyDLL` instances. - - -For accessing the C Python api directly, a ready-to-use Python shared library -object is available: - -.. data:: pythonapi - :noindex: - - An instance of :class:`PyDLL` that exposes Python C API functions as - attributes. Note that all these functions are assumed to return C - :c:type:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. - - -.. _ctypes-foreign-functions: - -Foreign functions -^^^^^^^^^^^^^^^^^ - -As explained in the previous section, foreign functions can be accessed as -attributes of loaded shared libraries. The function objects created in this way -by default accept any number of arguments, accept any ctypes data instances as -arguments, and return the default result type specified by the library loader. -They are instances of a private class: - - -.. class:: _FuncPtr - - Base class for C callable foreign functions. - - Instances of foreign functions are also C compatible data types; they - represent C function pointers. - - This behavior can be customized by assigning to special attributes of the - foreign function object. - - .. attribute:: restype - - Assign a ctypes type to specify the result type of the foreign function. - Use ``None`` for :c:type:`void`, a function not returning anything. - - It is possible to assign a callable Python object that is not a ctypes - type, in this case the function is assumed to return a C :c:type:`int`, and - the callable will be called with this integer, allowing further - processing or error checking. Using this is deprecated, for more flexible - post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. - - .. attribute:: argtypes - - Assign a tuple of ctypes types to specify the argument types that the - function accepts. Functions using the ``stdcall`` calling convention can - only be called with the same number of arguments as the length of this - tuple; functions using the C calling convention accept additional, - unspecified arguments as well. - - When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` - tuple, this method allows adapting the actual argument to an object that - the foreign function accepts. For example, a :class:`c_char_p` item in - the :attr:`argtypes` tuple will convert a string passed as argument into - a bytes object using ctypes conversion rules. - - New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a - value usable as argument (integer, string, ctypes instance). This allows - defining adapters that can adapt custom objects as function parameters. - - .. attribute:: errcheck - - Assign a Python function or another callable to this attribute. The - callable will be called with three or more arguments: - - .. function:: callable(result, func, arguments) - :noindex: - :module: - - *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. - - *func* is the foreign function object itself, this allows reusing the - same callable object to check or post process the results of several - functions. - - *arguments* is a tuple containing the parameters originally passed to - the function call, this allows specializing the behavior on the - arguments used. - - The object that this function returns will be returned from the - foreign function call, but it can also check the result value - and raise an exception if the foreign function call failed. - - -.. exception:: ArgumentError - - This exception is raised when a foreign function call cannot convert one of the - passed arguments. - - -.. _ctypes-function-prototypes: - -Function prototypes -^^^^^^^^^^^^^^^^^^^ - -Foreign functions can also be created by instantiating function prototypes. -Function prototypes are similar to function prototypes in C; they describe a -function (return type, argument types, calling convention) without defining an -implementation. The factory functions must be called with the desired result -type and the argument types of the function, and can be used as decorator -factories, and as such, be applied to functions through the ``@wrapper`` syntax. -See :ref:`ctypes-callback-functions` for examples. - - -.. function:: CFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - - The returned function prototype creates functions that use the standard C - calling convention. The function will release the GIL during the call. If - *use_errno* is set to true, the ctypes private copy of the system - :data:`errno` variable is exchanged with the real :data:`errno` value before - and after the call; *use_last_error* does the same for the Windows error - code. - - -.. function:: WINFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - - Windows only: The returned function prototype creates functions that use the - ``stdcall`` calling convention, except on Windows CE where - :func:`WINFUNCTYPE` is the same as :func:`CFUNCTYPE`. The function will - release the GIL during the call. *use_errno* and *use_last_error* have the - same meaning as above. - - -.. function:: PYFUNCTYPE(restype, *argtypes) - - The returned function prototype creates functions that use the Python calling - convention. The function will *not* release the GIL during the call. - -Function prototypes created by these factory functions can be instantiated in -different ways, depending on the type and number of the parameters in the call: - - - .. function:: prototype(address) - :noindex: - :module: - - Returns a foreign function at the specified address which must be an integer. - - - .. function:: prototype(callable) - :noindex: - :module: - - Create a C callable function (a callback function) from a Python *callable*. - - - .. function:: prototype(func_spec[, paramflags]) - :noindex: - :module: - - Returns a foreign function exported by a shared library. *func_spec* must - be a 2-tuple ``(name_or_ordinal, library)``. The first item is the name of - the exported function as string, or the ordinal of the exported function - as small integer. The second item is the shared library instance. - - - .. function:: prototype(vtbl_index, name[, paramflags[, iid]]) - :noindex: - :module: - - Returns a foreign function that will call a COM method. *vtbl_index* is - the index into the virtual function table, a small non-negative - integer. *name* is name of the COM method. *iid* is an optional pointer to - the interface identifier which is used in extended error reporting. - - COM methods use a special calling convention: They require a pointer to - the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. - - The optional *paramflags* parameter creates foreign function wrappers with much - more functionality than the features described above. - - *paramflags* must be a tuple of the same length as :attr:`argtypes`. - - Each item in this tuple contains further information about a parameter, it must - be a tuple containing one, two, or three items. - - The first item is an integer containing a combination of direction - flags for the parameter: - - 1 - Specifies an input parameter to the function. - - 2 - Output parameter. The foreign function fills in a value. - - 4 - Input parameter which defaults to the integer zero. - - The optional second item is the parameter name as string. If this is specified, - the foreign function can be called with named parameters. - - The optional third item is the default value for this parameter. - -This example demonstrates how to wrap the Windows ``MessageBoxW`` function so -that it supports default parameters and named arguments. The C declaration from -the windows header file is this:: - - WINUSERAPI int WINAPI - MessageBoxW( - HWND hWnd, - LPCWSTR lpText, - LPCWSTR lpCaption, - UINT uType); - -Here is the wrapping with :mod:`ctypes`:: - - >>> from ctypes import c_int, WINFUNCTYPE, windll - >>> from ctypes.wintypes import HWND, LPCWSTR, UINT - >>> prototype = WINFUNCTYPE(c_int, HWND, LPCWSTR, LPCWSTR, UINT) - >>> paramflags = (1, "hwnd", 0), (1, "text", "Hi"), (1, "caption", "Hello from ctypes"), (1, "flags", 0) - >>> MessageBox = prototype(("MessageBoxW", windll.user32), paramflags) - -The ``MessageBox`` foreign function can now be called in these ways:: - - >>> MessageBox() - >>> MessageBox(text="Spam, spam, spam") - >>> MessageBox(flags=2, text="foo bar") - -A second example demonstrates output parameters. The win32 ``GetWindowRect`` -function retrieves the dimensions of a specified window by copying them into -``RECT`` structure that the caller has to supply. Here is the C declaration:: - - WINUSERAPI BOOL WINAPI - GetWindowRect( - HWND hWnd, - LPRECT lpRect); - -Here is the wrapping with :mod:`ctypes`:: - - >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError - >>> from ctypes.wintypes import BOOL, HWND, RECT - >>> prototype = WINFUNCTYPE(BOOL, HWND, POINTER(RECT)) - >>> paramflags = (1, "hwnd"), (2, "lprect") - >>> GetWindowRect = prototype(("GetWindowRect", windll.user32), paramflags) - >>> - -Functions with output parameters will automatically return the output parameter -value if there is a single one, or a tuple containing the output parameter -values when there are more than one, so the GetWindowRect function now returns a -RECT instance, when called. - -Output parameters can be combined with the :attr:`errcheck` protocol to do -further output processing and error checking. The win32 ``GetWindowRect`` api -function returns a ``BOOL`` to signal success or failure, so this function could -do the error checking, and raises an exception when the api call failed:: - - >>> def errcheck(result, func, args): - ... if not result: - ... raise WinError() - ... return args - ... - >>> GetWindowRect.errcheck = errcheck - >>> - -If the :attr:`errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output -parameters. If you want to return a tuple of window coordinates instead of a -``RECT`` instance, you can retrieve the fields in the function and return them -instead, the normal processing will no longer take place:: - - >>> def errcheck(result, func, args): - ... if not result: - ... raise WinError() - ... rc = args[1] - ... return rc.left, rc.top, rc.bottom, rc.right - ... - >>> GetWindowRect.errcheck = errcheck - >>> - - -.. _ctypes-utility-functions: - -Utility functions -^^^^^^^^^^^^^^^^^ - -.. function:: addressof(obj) - - Returns the address of the memory buffer as integer. *obj* must be an - instance of a ctypes type. - - -.. function:: alignment(obj_or_type) - - Returns the alignment requirements of a ctypes type. *obj_or_type* must be a - ctypes type or instance. - - -.. function:: byref(obj[, offset]) - - Returns a light-weight pointer to *obj*, which must be an instance of a - ctypes type. *offset* defaults to zero, and must be an integer that will be - added to the internal pointer value. - - ``byref(obj, offset)`` corresponds to this C code:: - - (((char *)&obj) + offset) - - The returned object can only be used as a foreign function call parameter. - It behaves similar to ``pointer(obj)``, but the construction is a lot faster. - - -.. function:: cast(obj, type) - - This function is similar to the cast operator in C. It returns a new instance - of *type* which points to the same memory block as *obj*. *type* must be a - pointer type, and *obj* must be an object that can be interpreted as a - pointer. - - -.. function:: create_string_buffer(init_or_size, size=None) - - This function creates a mutable character buffer. The returned object is a - ctypes array of :class:`c_char`. - - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. - - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. - - - -.. function:: create_unicode_buffer(init_or_size, size=None) - - This function creates a mutable unicode character buffer. The returned object is - a ctypes array of :class:`c_wchar`. - - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. - - - -.. function:: DllCanUnloadNow() - - Windows only: This function is a hook which allows implementing in-process - COM servers with ctypes. It is called from the DllCanUnloadNow function that - the _ctypes extension dll exports. - - -.. function:: DllGetClassObject() - - Windows only: This function is a hook which allows implementing in-process - COM servers with ctypes. It is called from the DllGetClassObject function - that the ``_ctypes`` extension dll exports. - - -.. function:: find_library(name) - :module: ctypes.util - - Try to find a library and return a pathname. *name* is the library name - without any prefix like ``lib``, suffix like ``.so``, ``.dylib`` or version - number (this is the form used for the posix linker option :option:`!-l`). If - no library can be found, returns ``None``. - - The exact functionality is system dependent. - - -.. function:: find_msvcrt() - :module: ctypes.util - - Windows only: return the filename of the VC runtime library used by Python, - and by the extension modules. If the name of the library cannot be - determined, ``None`` is returned. - - If you need to free memory, for example, allocated by an extension module - with a call to the ``free(void *)``, it is important that you use the - function in the same library that allocated the memory. - - -.. function:: FormatError([code]) - - Windows only: Returns a textual description of the error code *code*. If no - error code is specified, the last error code is used by calling the Windows - api function GetLastError. - - -.. function:: GetLastError() - - Windows only: Returns the last error code set by Windows in the calling thread. - This function calls the Windows `GetLastError()` function directly, - it does not return the ctypes-private copy of the error code. - -.. function:: get_errno() - - Returns the current value of the ctypes-private copy of the system - :data:`errno` variable in the calling thread. - -.. function:: get_last_error() - - Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. - -.. function:: memmove(dst, src, count) - - Same as the standard C memmove library function: copies *count* bytes from - *src* to *dst*. *dst* and *src* must be integers or ctypes instances that can - be converted to pointers. - - -.. function:: memset(dst, c, count) - - Same as the standard C memset library function: fills the memory block at - address *dst* with *count* bytes of value *c*. *dst* must be an integer - specifying an address, or a ctypes instance. - - -.. function:: POINTER(type) - - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. - - -.. function:: pointer(obj) - - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. - - Note: If you just want to pass a pointer to an object to a foreign function - call, you should use ``byref(obj)`` which is much faster. - - -.. function:: resize(obj, size) - - This function resizes the internal memory buffer of *obj*, which must be an - instance of a ctypes type. It is not possible to make the buffer smaller - than the native size of the objects type, as given by ``sizeof(type(obj))``, - but it is possible to enlarge the buffer. - - -.. function:: set_errno(value) - - Set the current value of the ctypes-private copy of the system :data:`errno` - variable in the calling thread to *value* and return the previous value. - - - -.. function:: set_last_error(value) - - Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the - previous value. - - - -.. function:: sizeof(obj_or_type) - - Returns the size in bytes of a ctypes type or instance memory buffer. - Does the same as the C ``sizeof`` operator. - - -.. function:: string_at(address, size=-1) - - This function returns the C string starting at memory address *address* as a bytes - object. If size is specified, it is used as size, otherwise the string is assumed - to be zero-terminated. - - -.. function:: WinError(code=None, descr=None) - - Windows only: this function is probably the worst-named thing in ctypes. It - creates an instance of OSError. If *code* is not specified, - ``GetLastError`` is called to determine the error code. If *descr* is not - specified, :func:`FormatError` is called to get a textual description of the - error. - - .. versionchanged:: 3.3 - An instance of :exc:`WindowsError` used to be created. - - -.. function:: wstring_at(address, size=-1) - - This function returns the wide character string starting at memory address - *address* as a string. If *size* is specified, it is used as the number of - characters of the string, otherwise the string is assumed to be - zero-terminated. - - -.. _ctypes-data-types: - -Data types -^^^^^^^^^^ - - -.. class:: _CData - - This non-public class is the common base class of all ctypes data types. - Among other things, all ctypes type instances contain a memory block that - hold C compatible data; the address of the memory block is returned by the - :func:`addressof` helper function. Another instance variable is exposed as - :attr:`_objects`; this contains other Python objects that need to be kept - alive in case the memory block contains pointers. - - Common methods of ctypes data types, these are all class methods (to be - exact, they are methods of the :term:`metaclass`): - - .. method:: _CData.from_buffer(source[, offset]) - - This method returns a ctypes instance that shares the buffer of the - *source* object. The *source* object must support the writeable buffer - interface. The optional *offset* parameter specifies an offset into the - source buffer in bytes; the default is zero. If the source buffer is not - large enough a :exc:`ValueError` is raised. - - - .. method:: _CData.from_buffer_copy(source[, offset]) - - This method creates a ctypes instance, copying the buffer from the - *source* object buffer which must be readable. The optional *offset* - parameter specifies an offset into the source buffer in bytes; the default - is zero. If the source buffer is not large enough a :exc:`ValueError` is - raised. - - .. method:: from_address(address) - - This method returns a ctypes type instance using the memory specified by - *address* which must be an integer. - - .. method:: from_param(obj) - - This method adapts *obj* to a ctypes type. It is called with the actual - object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. - - All ctypes data types have a default implementation of this classmethod - that normally returns *obj* if that is an instance of the type. Some - types accept other objects as well. - - .. method:: in_dll(library, name) - - This method returns a ctypes type instance exported by a shared - library. *name* is the name of the symbol that exports the data, *library* - is the loaded shared library. - - Common instance variables of ctypes data types: - - .. attribute:: _b_base_ - - Sometimes ctypes data instances do not own the memory block they contain, - instead they share part of the memory block of a base object. The - :attr:`_b_base_` read-only member is the root ctypes object that owns the - memory block. - - .. attribute:: _b_needsfree_ - - This read-only variable is true when the ctypes data instance has - allocated the memory block itself, false otherwise. - - .. attribute:: _objects - - This member is either ``None`` or a dictionary containing Python objects - that need to be kept alive so that the memory block contents is kept - valid. This object is only exposed for debugging; never modify the - contents of this dictionary. - - -.. _ctypes-fundamental-data-types-2: - -Fundamental data types -^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: _SimpleCData - - This non-public class is the base class of all fundamental ctypes data - types. It is mentioned here because it contains the common attributes of the - fundamental ctypes data types. :class:`_SimpleCData` is a subclass of - :class:`_CData`, so it inherits their methods and attributes. ctypes data - types that are not and do not contain pointers can now be pickled. - - Instances have a single attribute: - - .. attribute:: value - - This attribute contains the actual value of the instance. For integer and - pointer types, it is an integer, for character types, it is a single - character bytes object or string, for character pointer types it is a - Python bytes object or string. - - When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement - original object return, always a new object is constructed. The same is - true for all other ctypes object instances. - - -Fundamental data types, when returned as foreign function call results, or, for -example, by retrieving structure field members or array items, are transparently -converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes -object, *not* a :class:`c_char_p` instance. - -.. XXX above is false, it actually returns a Unicode string - -Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will -receive an instance of this subclass from the function call. Of course, you can -get the value of the pointer by accessing the ``value`` attribute. - -These are the fundamental ctypes data types: - -.. class:: c_byte - - Represents the C :c:type:`signed char` datatype, and interprets the value as - small integer. The constructor accepts an optional integer initializer; no - overflow checking is done. - - -.. class:: c_char - - Represents the C :c:type:`char` datatype, and interprets the value as a single - character. The constructor accepts an optional string initializer, the - length of the string must be exactly one character. - - -.. class:: c_char_p - - Represents the C :c:type:`char *` datatype when it points to a zero-terminated - string. For a general character pointer that may also point to binary data, - ``POINTER(c_char)`` must be used. The constructor accepts an integer - address, or a bytes object. - - -.. class:: c_double - - Represents the C :c:type:`double` datatype. The constructor accepts an - optional float initializer. - - -.. class:: c_longdouble - - Represents the C :c:type:`long double` datatype. The constructor accepts an - optional float initializer. On platforms where ``sizeof(long double) == - sizeof(double)`` it is an alias to :class:`c_double`. - -.. class:: c_float - - Represents the C :c:type:`float` datatype. The constructor accepts an - optional float initializer. - - -.. class:: c_int - - Represents the C :c:type:`signed int` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. On platforms - where ``sizeof(int) == sizeof(long)`` it is an alias to :class:`c_long`. - - -.. class:: c_int8 - - Represents the C 8-bit :c:type:`signed int` datatype. Usually an alias for - :class:`c_byte`. - - -.. class:: c_int16 - - Represents the C 16-bit :c:type:`signed int` datatype. Usually an alias for - :class:`c_short`. - - -.. class:: c_int32 - - Represents the C 32-bit :c:type:`signed int` datatype. Usually an alias for - :class:`c_int`. - - -.. class:: c_int64 - - Represents the C 64-bit :c:type:`signed int` datatype. Usually an alias for - :class:`c_longlong`. - - -.. class:: c_long - - Represents the C :c:type:`signed long` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_longlong - - Represents the C :c:type:`signed long long` datatype. The constructor accepts - an optional integer initializer; no overflow checking is done. - - -.. class:: c_short - - Represents the C :c:type:`signed short` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_size_t - - Represents the C :c:type:`size_t` datatype. - - -.. class:: c_ssize_t - - Represents the C :c:type:`ssize_t` datatype. - - .. versionadded:: 3.2 - - -.. class:: c_ubyte - - Represents the C :c:type:`unsigned char` datatype, it interprets the value as - small integer. The constructor accepts an optional integer initializer; no - overflow checking is done. - - -.. class:: c_uint - - Represents the C :c:type:`unsigned int` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. On platforms - where ``sizeof(int) == sizeof(long)`` it is an alias for :class:`c_ulong`. - - -.. class:: c_uint8 - - Represents the C 8-bit :c:type:`unsigned int` datatype. Usually an alias for - :class:`c_ubyte`. - - -.. class:: c_uint16 - - Represents the C 16-bit :c:type:`unsigned int` datatype. Usually an alias for - :class:`c_ushort`. - - -.. class:: c_uint32 - - Represents the C 32-bit :c:type:`unsigned int` datatype. Usually an alias for - :class:`c_uint`. - - -.. class:: c_uint64 - - Represents the C 64-bit :c:type:`unsigned int` datatype. Usually an alias for - :class:`c_ulonglong`. - - -.. class:: c_ulong - - Represents the C :c:type:`unsigned long` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_ulonglong - - Represents the C :c:type:`unsigned long long` datatype. The constructor - accepts an optional integer initializer; no overflow checking is done. - - -.. class:: c_ushort - - Represents the C :c:type:`unsigned short` datatype. The constructor accepts - an optional integer initializer; no overflow checking is done. - - -.. class:: c_void_p - - Represents the C :c:type:`void *` type. The value is represented as integer. - The constructor accepts an optional integer initializer. - - -.. class:: c_wchar - - Represents the C :c:type:`wchar_t` datatype, and interprets the value as a - single character unicode string. The constructor accepts an optional string - initializer, the length of the string must be exactly one character. - - -.. class:: c_wchar_p - - Represents the C :c:type:`wchar_t *` datatype, which must be a pointer to a - zero-terminated wide character string. The constructor accepts an integer - address, or a string. - - -.. class:: c_bool - - Represent the C :c:type:`bool` datatype (more accurately, :c:type:`_Bool` from - C99). Its value can be ``True`` or ``False``, and the constructor accepts any object - that has a truth value. - - -.. class:: HRESULT - - Windows only: Represents a :c:type:`HRESULT` value, which contains success or - error information for a function or method call. - - -.. class:: py_object - - Represents the C :c:type:`PyObject *` datatype. Calling this without an - argument creates a ``NULL`` :c:type:`PyObject *` pointer. - -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. - - -.. _ctypes-structured-data-types: - -Structured data types -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: Union(*args, **kw) - - Abstract base class for unions in native byte order. - - -.. class:: BigEndianStructure(*args, **kw) - - Abstract base class for structures in *big endian* byte order. - - -.. class:: LittleEndianStructure(*args, **kw) - - Abstract base class for structures in *little endian* byte order. - -Structures with non-native byte order cannot contain pointer type fields, or any -other data types containing pointer type fields. - - -.. class:: Structure(*args, **kw) - - Abstract base class for structures in *native* byte order. - - Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will - create :term:`descriptor`\s which allow reading and writing the fields by direct - attribute accesses. These are the - - - .. attribute:: _fields_ - - A sequence defining the structure fields. The items must be 2-tuples or - 3-tuples. The first item is the name of the field, the second item - specifies the type of the field; it can be any ctypes data type. - - For integer type fields like :class:`c_int`, a third optional item can be - given. It must be a small positive integer defining the bit width of the - field. - - Field names must be unique within one structure or union. This is not - checked, only one field can be accessed when names are repeated. - - It is possible to define the :attr:`_fields_` class variable *after* the - class statement that defines the Structure subclass, this allows creating - data types that directly or indirectly reference themselves:: - - class List(Structure): - pass - List._fields_ = [("pnext", POINTER(List)), - ... - ] - - The :attr:`_fields_` class variable must, however, be defined before the - type is first used (an instance is created, :func:`sizeof` is called on it, - and so on). Later assignments to the :attr:`_fields_` class variable will - raise an AttributeError. - - It is possible to defined sub-subclasses of structure types, they inherit - the fields of the base class plus the :attr:`_fields_` defined in the - sub-subclass, if any. - - - .. attribute:: _pack_ - - An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - - - .. attribute:: _anonymous_ - - An optional sequence that lists the names of unnamed (anonymous) fields. - :attr:`_anonymous_` must be already defined when :attr:`_fields_` is - assigned, otherwise it will have no effect. - - The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows - accessing the nested fields directly, without the need to create the - structure or union field. - - Here is an example type (Windows):: - - class _U(Union): - _fields_ = [("lptdesc", POINTER(TYPEDESC)), - ("lpadesc", POINTER(ARRAYDESC)), - ("hreftype", HREFTYPE)] - - class TYPEDESC(Structure): - _anonymous_ = ("u",) - _fields_ = [("u", _U), - ("vt", VARTYPE)] - - - The ``TYPEDESC`` structure describes a COM data type, the ``vt`` field - specifies which one of the union fields is valid. Since the ``u`` field - is defined as anonymous field, it is now possible to access the members - directly off the TYPEDESC instance. ``td.lptdesc`` and ``td.u.lptdesc`` - are equivalent, but the former is faster since it does not need to create - a temporary union instance:: - - td = TYPEDESC() - td.vt = VT_PTR - td.lptdesc = POINTER(some_type) - td.u.lptdesc = POINTER(some_type) - - It is possible to defined sub-subclasses of structures, they inherit the - fields of the base class. If the subclass definition has a separate - :attr:`_fields_` variable, the fields specified in this are appended to the - fields of the base class. - - Structure and union constructors accept both positional and keyword - arguments. Positional arguments are used to initialize member fields in the - same order as they are appear in :attr:`_fields_`. Keyword arguments in the - constructor are interpreted as attribute assignments, so they will initialize - :attr:`_fields_` with the same name, or create new attributes for names not - present in :attr:`_fields_`. - - -.. _ctypes-arrays-pointers: - -Arrays and pointers -^^^^^^^^^^^^^^^^^^^ - -.. class:: Array(\*args) - - Abstract base class for arrays. - - The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a positive integer. Alternatively, you can subclass - this type and define :attr:`_length_` and :attr:`_type_` class variables. - Array elements can be read and written using standard - subscript and slice accesses; for slice reads, the resulting object is - *not* itself an :class:`Array`. - - - .. attribute:: _length_ - - A positive integer specifying the number of elements in the array. - Out-of-range subscripts result in an :exc:`IndexError`. Will be - returned by :func:`len`. - - - .. attribute:: _type_ - - Specifies the type of each element in the array. - - - Array subclass constructors accept positional arguments, used to - initialize the elements in order. - - -.. class:: _Pointer - - Private, abstract base class for pointers. - - Concrete pointer types are created by calling :func:`POINTER` with the - type that will be pointed to; this is done automatically by - :func:`pointer`. - - If a pointer points to an array, its elements can be read and - written using standard subscript and slice accesses. Pointer objects - have no size, so :func:`len` will raise :exc:`TypeError`. Negative - subscripts will read from the memory *before* the pointer (as in C), and - out-of-range subscripts will probably crash with an access violation (if - you're lucky). - - - .. attribute:: _type_ - - Specifies the type pointed to. - - .. attribute:: contents - - Returns the object to which to pointer points. Assigning to this - attribute changes the pointer to point to the assigned object. - diff --git a/third_party/python/Doc/library/curses.ascii.rst b/third_party/python/Doc/library/curses.ascii.rst deleted file mode 100644 index a69dbb2ac..000000000 --- a/third_party/python/Doc/library/curses.ascii.rst +++ /dev/null @@ -1,229 +0,0 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== - -.. module:: curses.ascii - :synopsis: Constants and set-membership functions for ASCII characters. - -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - --------------- - -The :mod:`curses.ascii` module supplies name constants for ASCII characters and -functions to test membership in various ASCII character classes. The constants -supplied are names for control characters as follows: - -+--------------+----------------------------------------------+ -| Name | Meaning | -+==============+==============================================+ -| :const:`NUL` | | -+--------------+----------------------------------------------+ -| :const:`SOH` | Start of heading, console interrupt | -+--------------+----------------------------------------------+ -| :const:`STX` | Start of text | -+--------------+----------------------------------------------+ -| :const:`ETX` | End of text | -+--------------+----------------------------------------------+ -| :const:`EOT` | End of transmission | -+--------------+----------------------------------------------+ -| :const:`ENQ` | Enquiry, goes with :const:`ACK` flow control | -+--------------+----------------------------------------------+ -| :const:`ACK` | Acknowledgement | -+--------------+----------------------------------------------+ -| :const:`BEL` | Bell | -+--------------+----------------------------------------------+ -| :const:`BS` | Backspace | -+--------------+----------------------------------------------+ -| :const:`TAB` | Tab | -+--------------+----------------------------------------------+ -| :const:`HT` | Alias for :const:`TAB`: "Horizontal tab" | -+--------------+----------------------------------------------+ -| :const:`LF` | Line feed | -+--------------+----------------------------------------------+ -| :const:`NL` | Alias for :const:`LF`: "New line" | -+--------------+----------------------------------------------+ -| :const:`VT` | Vertical tab | -+--------------+----------------------------------------------+ -| :const:`FF` | Form feed | -+--------------+----------------------------------------------+ -| :const:`CR` | Carriage return | -+--------------+----------------------------------------------+ -| :const:`SO` | Shift-out, begin alternate character set | -+--------------+----------------------------------------------+ -| :const:`SI` | Shift-in, resume default character set | -+--------------+----------------------------------------------+ -| :const:`DLE` | Data-link escape | -+--------------+----------------------------------------------+ -| :const:`DC1` | XON, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC2` | Device control 2, block-mode flow control | -+--------------+----------------------------------------------+ -| :const:`DC3` | XOFF, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC4` | Device control 4 | -+--------------+----------------------------------------------+ -| :const:`NAK` | Negative acknowledgement | -+--------------+----------------------------------------------+ -| :const:`SYN` | Synchronous idle | -+--------------+----------------------------------------------+ -| :const:`ETB` | End transmission block | -+--------------+----------------------------------------------+ -| :const:`CAN` | Cancel | -+--------------+----------------------------------------------+ -| :const:`EM` | End of medium | -+--------------+----------------------------------------------+ -| :const:`SUB` | Substitute | -+--------------+----------------------------------------------+ -| :const:`ESC` | Escape | -+--------------+----------------------------------------------+ -| :const:`FS` | File separator | -+--------------+----------------------------------------------+ -| :const:`GS` | Group separator | -+--------------+----------------------------------------------+ -| :const:`RS` | Record separator, block-mode terminator | -+--------------+----------------------------------------------+ -| :const:`US` | Unit separator | -+--------------+----------------------------------------------+ -| :const:`SP` | Space | -+--------------+----------------------------------------------+ -| :const:`DEL` | Delete | -+--------------+----------------------------------------------+ - -Note that many of these have little practical significance in modern usage. The -mnemonics derive from teleprinter conventions that predate digital computers. - -The module supplies the following functions, patterned on those in the standard -C library: - - -.. function:: isalnum(c) - - Checks for an ASCII alphanumeric character; it is equivalent to ``isalpha(c) or - isdigit(c)``. - - -.. function:: isalpha(c) - - Checks for an ASCII alphabetic character; it is equivalent to ``isupper(c) or - islower(c)``. - - -.. function:: isascii(c) - - Checks for a character value that fits in the 7-bit ASCII set. - - -.. function:: isblank(c) - - Checks for an ASCII whitespace character; space or horizontal tab. - - -.. function:: iscntrl(c) - - Checks for an ASCII control character (in the range 0x00 to 0x1f or 0x7f). - - -.. function:: isdigit(c) - - Checks for an ASCII decimal digit, ``'0'`` through ``'9'``. This is equivalent - to ``c in string.digits``. - - -.. function:: isgraph(c) - - Checks for ASCII any printable character except space. - - -.. function:: islower(c) - - Checks for an ASCII lower-case character. - - -.. function:: isprint(c) - - Checks for any ASCII printable character including space. - - -.. function:: ispunct(c) - - Checks for any printable ASCII character which is not a space or an alphanumeric - character. - - -.. function:: isspace(c) - - Checks for ASCII white-space characters; space, line feed, carriage return, form - feed, horizontal tab, vertical tab. - - -.. function:: isupper(c) - - Checks for an ASCII uppercase letter. - - -.. function:: isxdigit(c) - - Checks for an ASCII hexadecimal digit. This is equivalent to ``c in - string.hexdigits``. - - -.. function:: isctrl(c) - - Checks for an ASCII control character (ordinal values 0 to 31). - - -.. function:: ismeta(c) - - Checks for a non-ASCII character (ordinal values 0x80 and above). - -These functions accept either integers or single-character strings; when the argument is a -string, it is first converted using the built-in function :func:`ord`. - -Note that all these functions check ordinal bit values derived from the -character of the string you pass in; they do not actually know anything about -the host machine's character encoding. - -The following two functions take either a single-character string or integer -byte value; they return a value of the same type. - - -.. function:: ascii(c) - - Return the ASCII value corresponding to the low 7 bits of *c*. - - -.. function:: ctrl(c) - - Return the control character corresponding to the given character (the character - bit value is bitwise-anded with 0x1f). - - -.. function:: alt(c) - - Return the 8-bit character corresponding to the given ASCII character (the - character bit value is bitwise-ored with 0x80). - -The following function takes either a single-character string or integer value; -it returns a string. - - -.. index:: - single: ^ (caret); in curses module - single: ! (exclamation); in curses module - -.. function:: unctrl(c) - - Return a string representation of the ASCII character *c*. If *c* is printable, - this string is the character itself. If the character is a control character - (0x00--0x1f) the string consists of a caret (``'^'``) followed by the - corresponding uppercase letter. If the character is an ASCII delete (0x7f) the - string is ``'^?'``. If the character has its meta bit (0x80) set, the meta bit - is stripped, the preceding rules applied, and ``'!'`` prepended to the result. - - -.. data:: controlnames - - A 33-element string array that contains the ASCII mnemonics for the thirty-two - ASCII control characters from 0 (NUL) to 0x1f (US), in order, plus the mnemonic - ``SP`` for the space character. - diff --git a/third_party/python/Doc/library/curses.panel.rst b/third_party/python/Doc/library/curses.panel.rst deleted file mode 100644 index d770c03c8..000000000 --- a/third_party/python/Doc/library/curses.panel.rst +++ /dev/null @@ -1,120 +0,0 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== - -.. module:: curses.panel - :synopsis: A panel stack extension that adds depth to curses windows. - -.. sectionauthor:: A.M. Kuchling - --------------- - -Panels are windows with the added feature of depth, so they can be stacked on -top of each other, and only the visible portions of each window will be -displayed. Panels can be added, moved up or down in the stack, and removed. - - -.. _cursespanel-functions: - -Functions ---------- - -The module :mod:`curses.panel` defines the following functions: - - -.. function:: bottom_panel() - - Returns the bottom panel in the panel stack. - - -.. function:: new_panel(win) - - Returns a panel object, associating it with the given window *win*. Be aware - that you need to keep the returned panel object referenced explicitly. If you - don't, the panel object is garbage collected and removed from the panel stack. - - -.. function:: top_panel() - - Returns the top panel in the panel stack. - - -.. function:: update_panels() - - Updates the virtual screen after changes in the panel stack. This does not call - :func:`curses.doupdate`, so you'll have to do this yourself. - - -.. _curses-panel-objects: - -Panel Objects -------------- - -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. - -Panel objects have the following methods: - - -.. method:: Panel.above() - - Returns the panel above the current panel. - - -.. method:: Panel.below() - - Returns the panel below the current panel. - - -.. method:: Panel.bottom() - - Push the panel to the bottom of the stack. - - -.. method:: Panel.hidden() - - Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. - - -.. method:: Panel.hide() - - Hide the panel. This does not delete the object, it just makes the window on - screen invisible. - - -.. method:: Panel.move(y, x) - - Move the panel to the screen coordinates ``(y, x)``. - - -.. method:: Panel.replace(win) - - Change the window associated with the panel to the window *win*. - - -.. method:: Panel.set_userptr(obj) - - Set the panel's user pointer to *obj*. This is used to associate an arbitrary - piece of data with the panel, and can be any Python object. - - -.. method:: Panel.show() - - Display the panel (which might have been hidden). - - -.. method:: Panel.top() - - Push panel to the top of the stack. - - -.. method:: Panel.userptr() - - Returns the user pointer for the panel. This might be any Python object. - - -.. method:: Panel.window() - - Returns the window object associated with the panel. - diff --git a/third_party/python/Doc/library/curses.rst b/third_party/python/Doc/library/curses.rst deleted file mode 100644 index 9d87db64a..000000000 --- a/third_party/python/Doc/library/curses.rst +++ /dev/null @@ -1,1830 +0,0 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== - -.. module:: curses - :synopsis: An interface to the curses library, providing portable - terminal handling. - :platform: Unix - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Eric Raymond - --------------- - -The :mod:`curses` module provides an interface to the curses library, the -de-facto standard for portable advanced terminal handling. - -While curses is most widely used in the Unix environment, versions are available -for Windows, DOS, and possibly other systems as well. This extension module is -designed to match the API of ncurses, an open-source curses library hosted on -Linux and the BSD variants of Unix. - -.. note:: - - Whenever the documentation mentions a *character* it can be specified - as an integer, a one-character Unicode string or a one-byte byte string. - - Whenever the documentation mentions a *character string* it can be specified - as a Unicode string or a byte string. - -.. note:: - - Since version 5.4, the ncurses library decides how to interpret non-ASCII data - using the ``nl_langinfo`` function. That means that you have to call - :func:`locale.setlocale` in the application and encode Unicode strings - using one of the system's available encodings. This example uses the - system's default encoding:: - - import locale - locale.setlocale(locale.LC_ALL, '') - code = locale.getpreferredencoding() - - Then use *code* as the encoding for :meth:`str.encode` calls. - -.. seealso:: - - Module :mod:`curses.ascii` - Utilities for working with ASCII characters, regardless of your locale settings. - - Module :mod:`curses.panel` - A panel stack extension that adds depth to curses windows. - - Module :mod:`curses.textpad` - Editable text widget for curses supporting :program:`Emacs`\ -like bindings. - - :ref:`curses-howto` - Tutorial material on using curses with Python, by Andrew Kuchling and Eric - Raymond. - - The :source:`Tools/demo/` directory in the Python source distribution contains - some example programs using the curses bindings provided by this module. - - -.. _curses-functions: - -Functions ---------- - -The module :mod:`curses` defines the following exception: - - -.. exception:: error - - Exception raised when a curses library function returns an error. - -.. note:: - - Whenever *x* or *y* arguments to a function or a method are optional, they - default to the current cursor location. Whenever *attr* is optional, it defaults - to :const:`A_NORMAL`. - -The module :mod:`curses` defines the following functions: - - -.. function:: baudrate() - - Return the output speed of the terminal in bits per second. On software - terminal emulators it will have a fixed high value. Included for historical - reasons; in former times, it was used to write output loops for time delays and - occasionally to change interfaces depending on the line speed. - - -.. function:: beep() - - Emit a short attention sound. - - -.. function:: can_change_color() - - Return ``True`` or ``False``, depending on whether the programmer can change the colors - displayed by the terminal. - - -.. function:: cbreak() - - Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty - line buffering is turned off and characters are available to be read one by one. - However, unlike raw mode, special characters (interrupt, quit, suspend, and flow - control) retain their effects on the tty driver and calling program. Calling - first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. - - -.. function:: color_content(color_number) - - Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, - containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). - - -.. function:: color_pair(color_number) - - Return the attribute value for displaying text in the specified color. This - attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart - to this function. - - -.. function:: curs_set(visibility) - - Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, - normal, or very visible. If the terminal supports the visibility requested, return the - previous cursor state; otherwise raise an exception. On many - terminals, the "visible" mode is an underline cursor and the "very visible" mode - is a block cursor. - - -.. function:: def_prog_mode() - - Save the current terminal mode as the "program" mode, the mode when the running - program is using curses. (Its counterpart is the "shell" mode, for when the - program is not in curses.) Subsequent calls to :func:`reset_prog_mode` will - restore this mode. - - -.. function:: def_shell_mode() - - Save the current terminal mode as the "shell" mode, the mode when the running - program is not using curses. (Its counterpart is the "program" mode, when the - program is using curses capabilities.) Subsequent calls to - :func:`reset_shell_mode` will restore this mode. - - -.. function:: delay_output(ms) - - Insert an *ms* millisecond pause in output. - - -.. function:: doupdate() - - Update the physical screen. The curses library keeps two data structures, one - representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the - physical screen to match the virtual screen. - - The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write - operations such as :meth:`~window.addstr` have been performed on a window. The normal - :meth:`~window.refresh` call is simply :meth:`!noutrefresh` followed by :func:`!doupdate`; - if you have to update multiple windows, you can speed performance and perhaps - reduce screen flicker by issuing :meth:`!noutrefresh` calls on all windows, - followed by a single :func:`!doupdate`. - - -.. function:: echo() - - Enter echo mode. In echo mode, each character input is echoed to the screen as - it is entered. - - -.. function:: endwin() - - De-initialize the library, and return terminal to normal status. - - -.. function:: erasechar() - - Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this - is a property of the controlling tty of the curses program, and is not set by - the curses library itself. - - -.. function:: filter() - - The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the - capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` - string is set to the value of ``cr``. The effect is that the cursor is confined to - the current line, and so are screen updates. This may be used for enabling - character-at-a-time line editing without touching the rest of the screen. - - -.. function:: flash() - - Flash the screen. That is, change it to reverse-video and then change it back - in a short interval. Some people prefer such as 'visible bell' to the audible - attention signal produced by :func:`beep`. - - -.. function:: flushinp() - - Flush all input buffers. This throws away any typeahead that has been typed - by the user and has not yet been processed by the program. - - -.. function:: getmouse() - - After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be call to retrieve the queued mouse event, represented as a - 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish - multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is - currently unused.) *bstate* is an integer value whose bits will be set to - indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 4: - :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, - :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, - :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. - - -.. function:: getsyx() - - Return the current coordinates of the virtual screen cursor as a tuple - ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. - - -.. function:: getwin(file) - - Read window related data stored in the file by an earlier :func:`putwin` call. - The routine then creates and initializes a new window using that data, returning - the new window object. - - -.. function:: has_colors() - - Return ``True`` if the terminal can display colors; otherwise, return ``False``. - - -.. function:: has_ic() - - Return ``True`` if the terminal has insert- and delete-character capabilities. - This function is included for historical reasons only, as all modern software - terminal emulators have such capabilities. - - -.. function:: has_il() - - Return ``True`` if the terminal has insert- and delete-line capabilities, or can - simulate them using scrolling regions. This function is included for - historical reasons only, as all modern software terminal emulators have such - capabilities. - - -.. function:: has_key(ch) - - Take a key value *ch*, and return ``True`` if the current terminal type recognizes - a key with that value. - - -.. function:: halfdelay(tenths) - - Used for half-delay mode, which is similar to cbreak mode in that characters - typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, raise an exception if nothing has - been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use - :func:`nocbreak` to leave half-delay mode. - - -.. function:: init_color(color_number, r, g, b) - - Change the definition of a color, taking the number of the color to be changed - followed by three RGB values (for the amounts of red, green, and blue - components). The value of *color_number* must be between ``0`` and - :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and - ``1000``. When :func:`init_color` is used, all occurrences of that color on the - screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``True``. - - -.. function:: init_pair(pair_number, fg, bg) - - Change the definition of a color-pair. It takes three arguments: the number of - the color-pair to be changed, the foreground color number, and the background - color number. The value of *pair_number* must be between ``1`` and - ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot - be changed). The value of *fg* and *bg* arguments must be between ``0`` and - :const:`COLORS`. If the color-pair was previously initialized, the screen is - refreshed and all occurrences of that color-pair are changed to the new - definition. - - -.. function:: initscr() - - Initialize the library. Return a :ref:`window ` object - which represents the whole screen. - - .. note:: - - If there is an error opening the terminal, the underlying curses library may - cause the interpreter to exit. - - -.. function:: is_term_resized(nlines, ncols) - - Return ``True`` if :func:`resize_term` would modify the window structure, - ``False`` otherwise. - - -.. function:: isendwin() - - Return ``True`` if :func:`endwin` has been called (that is, the curses library has - been deinitialized). - - -.. function:: keyname(k) - - Return the name of the key numbered *k* as a bytes object. The name of a key generating printable - ASCII character is the key's character. The name of a control-key combination - is a two-byte bytes object consisting of a caret (``b'^'``) followed by the corresponding - printable ASCII character. The name of an alt-key combination (128--255) is a - bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding - ASCII character. - - -.. function:: killchar() - - Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems - this is a property of the controlling tty of the curses program, and is not set - by the curses library itself. - - -.. function:: longname() - - Return a bytes object containing the terminfo long name field describing the current - terminal. The maximum length of a verbose description is 128 characters. It is - defined only after the call to :func:`initscr`. - - -.. function:: meta(flag) - - If *flag* is ``True``, allow 8-bit characters to be input. If - *flag* is ``False``, allow only 7-bit chars. - - -.. function:: mouseinterval(interval) - - Set the maximum time in milliseconds that can elapse between press and release - events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 msec, or one fifth of a second. - - -.. function:: mousemask(mousemask) - - Set the mouse events to be reported, and return a tuple ``(availmask, - oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse - events are ever reported. - - -.. function:: napms(ms) - - Sleep for *ms* milliseconds. - - -.. function:: newpad(nlines, ncols) - - Create and return a pointer to a new pad data structure with the given number - of lines and columns. Return a pad as a window object. - - A pad is like a window, except that it is not restricted by the screen size, and - is not necessarily associated with a particular part of the screen. Pads can be - used when a large window is needed, and only a part of the window will be on the - screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` - methods of a pad require 6 arguments to specify the part of the pad to be - displayed and the location on the screen to be used for the display. The - arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and - the *s* arguments define a clipping box on the screen within which the pad region - is to be displayed. - - -.. function:: newwin(nlines, ncols) - newwin(nlines, ncols, begin_y, begin_x) - - Return a new :ref:`window `, whose left-upper corner - is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. - - By default, the window will extend from the specified position to the lower - right corner of the screen. - - -.. function:: nl() - - Enter newline mode. This mode translates the return key into newline on input, - and translates newline into return and line-feed on output. Newline mode is - initially on. - - -.. function:: nocbreak() - - Leave cbreak mode. Return to normal "cooked" mode with line buffering. - - -.. function:: noecho() - - Leave echo mode. Echoing of input characters is turned off. - - -.. function:: nonl() - - Leave newline mode. Disable translation of return into newline on input, and - disable low-level translation of newline into newline/return on output (but this - does not change the behavior of ``addch('\n')``, which always does the - equivalent of return and line feed on the virtual screen). With translation - off, curses can sometimes speed up vertical motion a little; also, it will be - able to detect the return key on input. - - -.. function:: noqiflush() - - When the :func:`!noqiflush` routine is used, normal flush of input and output queues - associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may - want to call :func:`!noqiflush` in a signal handler if you want output to - continue as though the interrupt had not occurred, after the handler exits. - - -.. function:: noraw() - - Leave raw mode. Return to normal "cooked" mode with line buffering. - - -.. function:: pair_content(pair_number) - - Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. - The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1``. - - -.. function:: pair_number(attr) - - Return the number of the color-pair set by the attribute value *attr*. - :func:`color_pair` is the counterpart to this function. - - -.. function:: putp(str) - - Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified - terminfo capability for the current terminal. Note that the output of :func:`putp` - always goes to standard output. - - -.. function:: qiflush([flag]) - - If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If - *flag* is ``True``, or no argument is provided, the queues will be flushed when - these control characters are read. - - -.. function:: raw() - - Enter raw mode. In raw mode, normal line buffering and processing of - interrupt, quit, suspend, and flow control keys are turned off; characters are - presented to curses input functions one by one. - - -.. function:: reset_prog_mode() - - Restore the terminal to "program" mode, as previously saved by - :func:`def_prog_mode`. - - -.. function:: reset_shell_mode() - - Restore the terminal to "shell" mode, as previously saved by - :func:`def_shell_mode`. - - -.. function:: resetty() - - Restore the state of the terminal modes to what it was at the last call to - :func:`savetty`. - - -.. function:: resize_term(nlines, ncols) - - Backend function used by :func:`resizeterm`, performing most of the work; - when resizing the windows, :func:`resize_term` blank-fills the areas that are - extended. The calling application should fill in these areas with - appropriate data. The :func:`!resize_term` function attempts to resize all - windows. However, due to the calling convention of pads, it is not possible - to resize these without additional interaction with the application. - - -.. function:: resizeterm(nlines, ncols) - - Resize the standard and current windows to the specified dimensions, and - adjusts other bookkeeping data used by the curses library that record the - window dimensions (in particular the SIGWINCH handler). - - -.. function:: savetty() - - Save the current state of the terminal modes in a buffer, usable by - :func:`resetty`. - - -.. function:: setsyx(y, x) - - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then - :meth:`leaveok ` is set ``True``. - - -.. function:: setupterm(term=None, fd=-1) - - Initialize the terminal. *term* is a string giving - the terminal name, or ``None``; if omitted or ``None``, the value of the - :envvar:`TERM` environment variable will be used. *fd* is the - file descriptor to which any initialization sequences will be sent; if not - supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. - - -.. function:: start_color() - - Must be called if the programmer wants to use colors, and before any other color - manipulation routine is called. It is good practice to call this routine right - after :func:`initscr`. - - :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` - module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number - of colors and color-pairs the terminal can support. It also restores the colors - on the terminal to the values they had when the terminal was just turned on. - - -.. function:: termattrs() - - Return a logical OR of all video attributes supported by the terminal. This - information is useful when a curses program needs complete control over the - appearance of the screen. - - -.. function:: termname() - - Return the value of the environment variable :envvar:`TERM`, as a bytes object, - truncated to 14 characters. - - -.. function:: tigetflag(capname) - - Return the value of the Boolean capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a - Boolean capability, or ``0`` if it is canceled or absent from the terminal - description. - - -.. function:: tigetnum(capname) - - Return the value of the numeric capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a - numeric capability, or ``-1`` if it is canceled or absent from the terminal - description. - - -.. function:: tigetstr(capname) - - Return the value of the string capability corresponding to the terminfo - capability name *capname* as a bytes object. Return ``None`` if *capname* - is not a terminfo "string capability", or is canceled or absent from the - terminal description. - - -.. function:: tparm(str[, ...]) - - Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. - ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. - - -.. function:: typeahead(fd) - - Specify that the file descriptor *fd* be used for typeahead checking. If *fd* - is ``-1``, then no typeahead checking is done. - - The curses library does "line-breakout optimization" by looking for typeahead - periodically while updating the screen. If input is found, and it is coming - from a tty, the current update is postponed until refresh or doupdate is called - again, allowing faster response to commands typed in advance. This function - allows specifying a different file descriptor for typeahead checking. - - -.. function:: unctrl(ch) - - Return a bytes object which is a printable representation of the character *ch*. - Control characters are represented as a caret followed by the character, for - example as ``b'^C'``. Printing characters are left as they are. - - -.. function:: ungetch(ch) - - Push *ch* so the next :meth:`~window.getch` will return it. - - .. note:: - - Only one *ch* can be pushed before :meth:`!getch` is called. - - -.. function:: update_lines_cols() - - Update :envvar:`LINES` and :envvar:`COLS`. Useful for detecting manual screen resize. - - .. versionadded:: 3.5 - - -.. function:: unget_wch(ch) - - Push *ch* so the next :meth:`~window.get_wch` will return it. - - .. note:: - - Only one *ch* can be pushed before :meth:`!get_wch` is called. - - .. versionadded:: 3.3 - - -.. function:: ungetmouse(id, x, y, z, bstate) - - Push a :const:`KEY_MOUSE` event onto the input queue, associating the given - state data with it. - - -.. function:: use_env(flag) - - If used, this function should be called before :func:`initscr` or newterm are - called. When *flag* is ``False``, the values of lines and columns specified in the - terminfo database will be used, even if environment variables :envvar:`LINES` - and :envvar:`COLUMNS` (used by default) are set, or if curses is running in a - window (in which case default behavior would be to use the window size if - :envvar:`LINES` and :envvar:`COLUMNS` are not set). - - -.. function:: use_default_colors() - - Allow use of default values for colors on terminals supporting this feature. Use - this to support transparency in your application. The default color is assigned - to the color number ``-1``. After calling this function, ``init_pair(x, - curses.COLOR_RED, -1)`` initializes, for instance, color pair *x* to a red - foreground color on the default background. - - -.. function:: wrapper(func, ...) - - Initialize curses and call another callable object, *func*, which should be the - rest of your curses-using application. If the application raises an exception, - this function will restore the terminal to a sane state before re-raising the - exception and generating a traceback. The callable object *func* is then passed - the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on - cbreak mode, turns off echo, enables the terminal keypad, and initializes colors - if the terminal has color support. On exit (whether normally or by exception) - it restores cooked mode, turns on echo, and disables the terminal keypad. - - -.. _curses-window-objects: - -Window Objects --------------- - -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: - - -.. method:: window.addch(ch[, attr]) - window.addch(y, x, ch[, attr]) - - Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painter at that location. By default, the character - position and attributes are the current settings for the window object. - - .. note:: - - Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the character is printed. - - -.. method:: window.addnstr(str, n[, attr]) - window.addnstr(y, x, str, n[, attr]) - - Paint at most *n* characters of the character string *str* at - ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. - - -.. method:: window.addstr(str[, attr]) - window.addstr(y, x, str[, attr]) - - Paint the character string *str* at ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. - - .. note:: - - Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. - - -.. method:: window.attroff(attr) - - Remove attribute *attr* from the "background" set applied to all writes to the - current window. - - -.. method:: window.attron(attr) - - Add attribute *attr* from the "background" set applied to all writes to the - current window. - - -.. method:: window.attrset(attr) - - Set the "background" set of attributes to *attr*. This set is initially - ``0`` (no attributes). - - -.. method:: window.bkgd(ch[, attr]) - - Set the background property of the window to the character *ch*, with - attributes *attr*. The change is then applied to every character position in - that window: - - * The attribute of every character in the window is changed to the new - background attribute. - - * Wherever the former background character appears, it is changed to the new - background character. - - -.. method:: window.bkgdset(ch[, attr]) - - Set the window's background. A window's background consists of a character and - any combination of attributes. The attribute part of the background is combined - (OR'ed) with all non-blank characters that are written into the window. Both - the character and attribute parts of the background are combined with the blank - characters. The background becomes a property of the character and moves with - the character through any scrolling and insert/delete line/character operations. - - -.. method:: window.border([ls[, rs[, ts[, bs[, tl[, tr[, bl[, br]]]]]]]]) - - Draw a border around the edges of the window. Each parameter specifies the - character to use for a specific part of the border; see the table below for more - details. - - .. note:: - - A ``0`` value for any parameter will cause the default character to be used for - that parameter. Keyword parameters can *not* be used. The defaults are listed - in this table: - - +-----------+---------------------+-----------------------+ - | Parameter | Description | Default value | - +===========+=====================+=======================+ - | *ls* | Left side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *rs* | Right side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *ts* | Top | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *bs* | Bottom | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *tl* | Upper-left corner | :const:`ACS_ULCORNER` | - +-----------+---------------------+-----------------------+ - | *tr* | Upper-right corner | :const:`ACS_URCORNER` | - +-----------+---------------------+-----------------------+ - | *bl* | Bottom-left corner | :const:`ACS_LLCORNER` | - +-----------+---------------------+-----------------------+ - | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | - +-----------+---------------------+-----------------------+ - - -.. method:: window.box([vertch, horch]) - - Similar to :meth:`border`, but both *ls* and *rs* are *vertch* and both *ts* and - *bs* are *horch*. The default corner characters are always used by this function. - - -.. method:: window.chgat(attr) - window.chgat(num, attr) - window.chgat(y, x, attr) - window.chgat(y, x, num, attr) - - Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, - the attribute will be set on all the characters to the end of the line. This - function moves cursor to position ``(y, x)`` if supplied. The changed line - will be touched using the :meth:`touchline` method so that the contents will - be redisplayed by the next window refresh. - - -.. method:: window.clear() - - Like :meth:`erase`, but also cause the whole window to be repainted upon next - call to :meth:`refresh`. - - -.. method:: window.clearok(flag) - - If *flag* is ``True``, the next call to :meth:`refresh` will clear the window - completely. - - -.. method:: window.clrtobot() - - Erase from cursor to the end of the window: all lines below the cursor are - deleted, and then the equivalent of :meth:`clrtoeol` is performed. - - -.. method:: window.clrtoeol() - - Erase from cursor to the end of the line. - - -.. method:: window.cursyncup() - - Update the current cursor position of all the ancestors of the window to - reflect the current cursor position of the window. - - -.. method:: window.delch([y, x]) - - Delete any character at ``(y, x)``. - - -.. method:: window.deleteln() - - Delete the line under the cursor. All following lines are moved up by one line. - - -.. method:: window.derwin(begin_y, begin_x) - window.derwin(nlines, ncols, begin_y, begin_x) - - An abbreviation for "derive window", :meth:`derwin` is the same as calling - :meth:`subwin`, except that *begin_y* and *begin_x* are relative to the origin - of the window, rather than relative to the entire screen. Return a window - object for the derived window. - - -.. method:: window.echochar(ch[, attr]) - - Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` - on the window. - - -.. method:: window.enclose(y, x) - - Test whether the given pair of screen-relative character-cell coordinates are - enclosed by the given window, returning ``True`` or ``False``. It is useful for - determining what subset of the screen windows enclose the location of a mouse - event. - - -.. attribute:: window.encoding - - Encoding used to encode method arguments (Unicode strings and characters). - The encoding attribute is inherited from the parent window when a subwindow - is created, for example with :meth:`window.subwin`. By default, the locale - encoding is used (see :func:`locale.getpreferredencoding`). - - .. versionadded:: 3.3 - - -.. method:: window.erase() - - Clear the window. - - -.. method:: window.getbegyx() - - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. - - -.. method:: window.getbkgd() - - Return the given window's current background character/attribute pair. - - -.. method:: window.getch([y, x]) - - Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on are represented by numbers higher - than 255. In no-delay mode, return ``-1`` if there is no input, otherwise - wait until a key is pressed. - - -.. method:: window.get_wch([y, x]) - - Get a wide character. Return a character for most keys, or an integer for - function keys, keypad keys, and other special keys. - In no-delay mode, raise an exception if there is no input. - - .. versionadded:: 3.3 - - -.. method:: window.getkey([y, x]) - - Get a character, returning a string instead of an integer, as :meth:`getch` - does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, raise an exception if - there is no input. - - -.. method:: window.getmaxyx() - - Return a tuple ``(y, x)`` of the height and width of the window. - - -.. method:: window.getparyx() - - Return the beginning coordinates of this window relative to its parent window - as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no - parent. - - -.. method:: window.getstr() - window.getstr(n) - window.getstr(y, x) - window.getstr(y, x, n) - - Read a bytes object from the user, with primitive line editing capacity. - - -.. method:: window.getyx() - - Return a tuple ``(y, x)`` of current cursor position relative to the window's - upper-left corner. - - -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) - - Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. - - -.. method:: window.idcok(flag) - - If *flag* is ``False``, curses no longer considers using the hardware insert/delete - character feature of the terminal; if *flag* is ``True``, use of character insertion - and deletion is enabled. When curses is first initialized, use of character - insert/delete is enabled by default. - - -.. method:: window.idlok(flag) - - If *flag* is ``True``, :mod:`curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. - - -.. method:: window.immedok(flag) - - If *flag* is ``True``, any change in the window image automatically causes the - window to be refreshed; you no longer have to call :meth:`refresh` yourself. - However, it may degrade performance considerably, due to repeated calls to - wrefresh. This option is disabled by default. - - -.. method:: window.inch([y, x]) - - Return the character at the given position in the window. The bottom 8 bits are - the character proper, and upper bits are the attributes. - - -.. method:: window.insch(ch[, attr]) - window.insch(y, x, ch[, attr]) - - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. - - -.. method:: window.insdelln(nlines) - - Insert *nlines* lines into the specified window above the current line. The - *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines - starting with the one under the cursor, and move the remaining lines up. The - bottom *nlines* lines are cleared. The current cursor position remains the - same. - - -.. method:: window.insertln() - - Insert a blank line under the cursor. All following lines are moved down by one - line. - - -.. method:: window.insnstr(str, n[, attr]) - window.insnstr(y, x, str, n[, attr]) - - Insert a character string (as many characters as will fit on the line) before - the character under the cursor, up to *n* characters. If *n* is zero or - negative, the entire string is inserted. All characters to the right of the - cursor are shifted right, with the rightmost characters on the line being lost. - The cursor position does not change (after moving to *y*, *x*, if specified). - - -.. method:: window.insstr(str[, attr]) - window.insstr(y, x, str[, attr]) - - Insert a character string (as many characters as will fit on the line) before - the character under the cursor. All characters to the right of the cursor are - shifted right, with the rightmost characters on the line being lost. The cursor - position does not change (after moving to *y*, *x*, if specified). - - -.. method:: window.instr([n]) - window.instr(y, x[, n]) - - Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped - from the characters. If *n* is specified, :meth:`instr` returns a string - at most *n* characters long (exclusive of the trailing NUL). - - -.. method:: window.is_linetouched(line) - - Return ``True`` if the specified line was modified since the last call to - :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` - exception if *line* is not valid for the given window. - - -.. method:: window.is_wintouched() - - Return ``True`` if the specified window was modified since the last call to - :meth:`refresh`; otherwise return ``False``. - - -.. method:: window.keypad(flag) - - If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be - left as is in the input stream. - - -.. method:: window.leaveok(flag) - - If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. - - If *flag* is ``False``, cursor will always be at "cursor position" after an update. - - -.. method:: window.move(new_y, new_x) - - Move cursor to ``(new_y, new_x)``. - - -.. method:: window.mvderwin(y, x) - - Move the window inside its parent window. The screen-relative parameters of - the window are not changed. This routine is used to display different parts of - the parent window at the same physical position on the screen. - - -.. method:: window.mvwin(new_y, new_x) - - Move the window so its upper-left corner is at ``(new_y, new_x)``. - - -.. method:: window.nodelay(flag) - - If *flag* is ``True``, :meth:`getch` will be non-blocking. - - -.. method:: window.notimeout(flag) - - If *flag* is ``True``, escape sequences will not be timed out. - - If *flag* is ``False``, after a few milliseconds, an escape sequence will not be - interpreted, and will be left in the input stream as is. - - -.. method:: window.noutrefresh() - - Mark for refresh but wait. This function updates the data structure - representing the desired state of the window, but does not force an update of - the physical screen. To accomplish that, call :func:`doupdate`. - - -.. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) - - Overlay the window on top of *destwin*. The windows need not be the same size, - only the overlapping region is copied. This copy is non-destructive, which means - that the current background character does not overwrite the old contents of - *destwin*. - - To get fine-grained control over the copied region, the second form of - :meth:`overlay` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, and the other variables mark a rectangle in - the destination window. - - -.. method:: window.overwrite(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) - - Overwrite the window on top of *destwin*. The windows need not be the same size, - in which case only the overlapping region is copied. This copy is destructive, - which means that the current background character overwrites the old contents of - *destwin*. - - To get fine-grained control over the copied region, the second form of - :meth:`overwrite` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, the other variables mark a rectangle in the - destination window. - - -.. method:: window.putwin(file) - - Write all data associated with the window into the provided file object. This - information can be later retrieved using the :func:`getwin` function. - - -.. method:: window.redrawln(beg, num) - - Indicate that the *num* screen lines, starting at line *beg*, are corrupted and - should be completely redrawn on the next :meth:`refresh` call. - - -.. method:: window.redrawwin() - - Touch the entire window, causing it to be completely redrawn on the next - :meth:`refresh` call. - - -.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) - - Update the display immediately (sync actual screen with previous - drawing/deleting methods). - - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, - *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be - displayed in the pad is calculated from the screen coordinates, since the - rectangles must be the same size. Both rectangles must be entirely contained - within their respective structures. Negative values of *pminrow*, *pmincol*, - *sminrow*, or *smincol* are treated as if they were zero. - - -.. method:: window.resize(nlines, ncols) - - Reallocate storage for a curses window to adjust its dimensions to the - specified values. If either dimension is larger than the current values, the - window's data is filled with blanks that have the current background - rendition (as set by :meth:`bkgdset`) merged into them. - - -.. method:: window.scroll([lines=1]) - - Scroll the screen or scrolling region upward by *lines* lines. - - -.. method:: window.scrollok(flag) - - Control what happens when the cursor of a window is moved off the edge of the - window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is ``False``, the - cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up - one line. Note that in order to get the physical scrolling effect on the - terminal, it is also necessary to call :meth:`idlok`. - - -.. method:: window.setscrreg(top, bottom) - - Set the scrolling region from line *top* to line *bottom*. All scrolling actions - will take place in this region. - - -.. method:: window.standend() - - Turn off the standout attribute. On some terminals this has the side effect of - turning off all attributes. - - -.. method:: window.standout() - - Turn on attribute *A_STANDOUT*. - - -.. method:: window.subpad(begin_y, begin_x) - window.subpad(nlines, ncols, begin_y, begin_x) - - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. - - -.. method:: window.subwin(begin_y, begin_x) - window.subwin(nlines, ncols, begin_y, begin_x) - - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. - - By default, the sub-window will extend from the specified position to the lower - right corner of the window. - - -.. method:: window.syncdown() - - Touch each location in the window that has been touched in any of its ancestor - windows. This routine is called by :meth:`refresh`, so it should almost never - be necessary to call it manually. - - -.. method:: window.syncok(flag) - - If *flag* is ``True``, then :meth:`syncup` is called automatically - whenever there is a change in the window. - - -.. method:: window.syncup() - - Touch all locations in ancestors of the window that have been changed in the - window. - - -.. method:: window.timeout(delay) - - Set blocking or non-blocking read behavior for the window. If *delay* is - negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and :meth:`getch` will - return ``-1`` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is - still no input at the end of that time. - - -.. method:: window.touchline(start, count[, changed]) - - Pretend *count* lines have been changed, starting with line *start*. If - *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). - - -.. method:: window.touchwin() - - Pretend the whole window has been changed, for purposes of drawing - optimizations. - - -.. method:: window.untouchwin() - - Mark all lines in the window as unchanged since the last call to - :meth:`refresh`. - - -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) - - Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch*. - - -Constants ---------- - -The :mod:`curses` module defines the following data members: - - -.. data:: ERR - - Some curses routines that return an integer, such as :func:`getch`, return - :const:`ERR` upon failure. - - -.. data:: OK - - Some curses routines that return an integer, such as :func:`napms`, return - :const:`OK` upon success. - - -.. data:: version - - A bytes object representing the current version of the module. Also available as - :const:`__version__`. - -Some constants are available to specify character cell attributes. -The exact constants available are system dependent. - -+------------------+-------------------------------+ -| Attribute | Meaning | -+==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode | -+------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode | -+------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode | -+------------------+-------------------------------+ -| ``A_DIM`` | Dim mode | -+------------------+-------------------------------+ -| ``A_INVIS`` | Invisible or blank mode | -+------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute | -+------------------+-------------------------------+ -| ``A_PROTECT`` | Protected mode | -+------------------+-------------------------------+ -| ``A_REVERSE`` | Reverse background and | -| | foreground colors | -+------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode | -+------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode | -+------------------+-------------------------------+ -| ``A_HORIZONTAL`` | Horizontal highlight | -+------------------+-------------------------------+ -| ``A_LEFT`` | Left highlight | -+------------------+-------------------------------+ -| ``A_LOW`` | Low highlight | -+------------------+-------------------------------+ -| ``A_RIGHT`` | Right highlight | -+------------------+-------------------------------+ -| ``A_TOP`` | Top highlight | -+------------------+-------------------------------+ -| ``A_VERTICAL`` | Vertical highlight | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ - -Several constants are available to extract corresponding attributes returned -by some methods. - -+------------------+-------------------------------+ -| Bit-mask | Meaning | -+==================+===============================+ -| ``A_ATTRIBUTES`` | Bit-mask to extract | -| | attributes | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ -| ``A_COLOR`` | Bit-mask to extract | -| | color-pair field information | -+------------------+-------------------------------+ - -Keys are referred to by integer constants with names starting with ``KEY_``. -The exact keycaps available are system dependent. - -.. XXX this table is far too large! should it be alphabetized? - -+-------------------+--------------------------------------------+ -| Key constant | Key | -+===================+============================================+ -| ``KEY_MIN`` | Minimum key value | -+-------------------+--------------------------------------------+ -| ``KEY_BREAK`` | Break key (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_DOWN`` | Down-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_UP`` | Up-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_LEFT`` | Left-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_RIGHT`` | Right-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_HOME`` | Home key (upward+left arrow) | -+-------------------+--------------------------------------------+ -| ``KEY_BACKSPACE`` | Backspace (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_F0`` | Function keys. Up to 64 function keys are | -| | supported. | -+-------------------+--------------------------------------------+ -| ``KEY_Fn`` | Value of function key *n* | -+-------------------+--------------------------------------------+ -| ``KEY_DL`` | Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_IL`` | Insert line | -+-------------------+--------------------------------------------+ -| ``KEY_DC`` | Delete character | -+-------------------+--------------------------------------------+ -| ``KEY_IC`` | Insert char or enter insert mode | -+-------------------+--------------------------------------------+ -| ``KEY_EIC`` | Exit insert char mode | -+-------------------+--------------------------------------------+ -| ``KEY_CLEAR`` | Clear screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOS`` | Clear to end of screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOL`` | Clear to end of line | -+-------------------+--------------------------------------------+ -| ``KEY_SF`` | Scroll 1 line forward | -+-------------------+--------------------------------------------+ -| ``KEY_SR`` | Scroll 1 line backward (reverse) | -+-------------------+--------------------------------------------+ -| ``KEY_NPAGE`` | Next page | -+-------------------+--------------------------------------------+ -| ``KEY_PPAGE`` | Previous page | -+-------------------+--------------------------------------------+ -| ``KEY_STAB`` | Set tab | -+-------------------+--------------------------------------------+ -| ``KEY_CTAB`` | Clear tab | -+-------------------+--------------------------------------------+ -| ``KEY_CATAB`` | Clear all tabs | -+-------------------+--------------------------------------------+ -| ``KEY_ENTER`` | Enter or send (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_SRESET`` | Soft (partial) reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_RESET`` | Reset or hard reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_PRINT`` | Print | -+-------------------+--------------------------------------------+ -| ``KEY_LL`` | Home down or bottom (lower left) | -+-------------------+--------------------------------------------+ -| ``KEY_A1`` | Upper left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_A3`` | Upper right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_B2`` | Center of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C1`` | Lower left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C3`` | Lower right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_BTAB`` | Back tab | -+-------------------+--------------------------------------------+ -| ``KEY_BEG`` | Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_CANCEL`` | Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_CLOSE`` | Close | -+-------------------+--------------------------------------------+ -| ``KEY_COMMAND`` | Cmd (command) | -+-------------------+--------------------------------------------+ -| ``KEY_COPY`` | Copy | -+-------------------+--------------------------------------------+ -| ``KEY_CREATE`` | Create | -+-------------------+--------------------------------------------+ -| ``KEY_END`` | End | -+-------------------+--------------------------------------------+ -| ``KEY_EXIT`` | Exit | -+-------------------+--------------------------------------------+ -| ``KEY_FIND`` | Find | -+-------------------+--------------------------------------------+ -| ``KEY_HELP`` | Help | -+-------------------+--------------------------------------------+ -| ``KEY_MARK`` | Mark | -+-------------------+--------------------------------------------+ -| ``KEY_MESSAGE`` | Message | -+-------------------+--------------------------------------------+ -| ``KEY_MOVE`` | Move | -+-------------------+--------------------------------------------+ -| ``KEY_NEXT`` | Next | -+-------------------+--------------------------------------------+ -| ``KEY_OPEN`` | Open | -+-------------------+--------------------------------------------+ -| ``KEY_OPTIONS`` | Options | -+-------------------+--------------------------------------------+ -| ``KEY_PREVIOUS`` | Prev (previous) | -+-------------------+--------------------------------------------+ -| ``KEY_REDO`` | Redo | -+-------------------+--------------------------------------------+ -| ``KEY_REFERENCE`` | Ref (reference) | -+-------------------+--------------------------------------------+ -| ``KEY_REFRESH`` | Refresh | -+-------------------+--------------------------------------------+ -| ``KEY_REPLACE`` | Replace | -+-------------------+--------------------------------------------+ -| ``KEY_RESTART`` | Restart | -+-------------------+--------------------------------------------+ -| ``KEY_RESUME`` | Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SAVE`` | Save | -+-------------------+--------------------------------------------+ -| ``KEY_SBEG`` | Shifted Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_SCANCEL`` | Shifted Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_SCOMMAND`` | Shifted Command | -+-------------------+--------------------------------------------+ -| ``KEY_SCOPY`` | Shifted Copy | -+-------------------+--------------------------------------------+ -| ``KEY_SCREATE`` | Shifted Create | -+-------------------+--------------------------------------------+ -| ``KEY_SDC`` | Shifted Delete char | -+-------------------+--------------------------------------------+ -| ``KEY_SDL`` | Shifted Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_SELECT`` | Select | -+-------------------+--------------------------------------------+ -| ``KEY_SEND`` | Shifted End | -+-------------------+--------------------------------------------+ -| ``KEY_SEOL`` | Shifted Clear line | -+-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Exit | -+-------------------+--------------------------------------------+ -| ``KEY_SFIND`` | Shifted Find | -+-------------------+--------------------------------------------+ -| ``KEY_SHELP`` | Shifted Help | -+-------------------+--------------------------------------------+ -| ``KEY_SHOME`` | Shifted Home | -+-------------------+--------------------------------------------+ -| ``KEY_SIC`` | Shifted Input | -+-------------------+--------------------------------------------+ -| ``KEY_SLEFT`` | Shifted Left arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SMESSAGE`` | Shifted Message | -+-------------------+--------------------------------------------+ -| ``KEY_SMOVE`` | Shifted Move | -+-------------------+--------------------------------------------+ -| ``KEY_SNEXT`` | Shifted Next | -+-------------------+--------------------------------------------+ -| ``KEY_SOPTIONS`` | Shifted Options | -+-------------------+--------------------------------------------+ -| ``KEY_SPREVIOUS`` | Shifted Prev | -+-------------------+--------------------------------------------+ -| ``KEY_SPRINT`` | Shifted Print | -+-------------------+--------------------------------------------+ -| ``KEY_SREDO`` | Shifted Redo | -+-------------------+--------------------------------------------+ -| ``KEY_SREPLACE`` | Shifted Replace | -+-------------------+--------------------------------------------+ -| ``KEY_SRIGHT`` | Shifted Right arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SRSUME`` | Shifted Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SSAVE`` | Shifted Save | -+-------------------+--------------------------------------------+ -| ``KEY_SSUSPEND`` | Shifted Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_SUNDO`` | Shifted Undo | -+-------------------+--------------------------------------------+ -| ``KEY_SUSPEND`` | Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_UNDO`` | Undo | -+-------------------+--------------------------------------------+ -| ``KEY_MOUSE`` | Mouse event has occurred | -+-------------------+--------------------------------------------+ -| ``KEY_RESIZE`` | Terminal resize event | -+-------------------+--------------------------------------------+ -| ``KEY_MAX`` | Maximum key value | -+-------------------+--------------------------------------------+ - -On VT100s and their software emulations, such as X terminal emulators, there are -normally at least four function keys (:const:`KEY_F1`, :const:`KEY_F2`, -:const:`KEY_F3`, :const:`KEY_F4`) available, and the arrow keys mapped to -:const:`KEY_UP`, :const:`KEY_DOWN`, :const:`KEY_LEFT` and :const:`KEY_RIGHT` in -the obvious way. If your machine has a PC keyboard, it is safe to expect arrow -keys and twelve function keys (older PC keyboards may have only ten function -keys); also, the following keypad mappings are standard: - -+------------------+-----------+ -| Keycap | Constant | -+==================+===========+ -| :kbd:`Insert` | KEY_IC | -+------------------+-----------+ -| :kbd:`Delete` | KEY_DC | -+------------------+-----------+ -| :kbd:`Home` | KEY_HOME | -+------------------+-----------+ -| :kbd:`End` | KEY_END | -+------------------+-----------+ -| :kbd:`Page Up` | KEY_PPAGE | -+------------------+-----------+ -| :kbd:`Page Down` | KEY_NPAGE | -+------------------+-----------+ - -The following table lists characters from the alternate character set. These are -inherited from the VT100 terminal, and will generally be available on software -emulations such as X terminals. When there is no graphic available, curses -falls back on a crude printable ASCII approximation. - -.. note:: - - These are available only after :func:`initscr` has been called. - -+------------------+------------------------------------------+ -| ACS code | Meaning | -+==================+==========================================+ -| ``ACS_BBSS`` | alternate name for upper right corner | -+------------------+------------------------------------------+ -| ``ACS_BLOCK`` | solid square block | -+------------------+------------------------------------------+ -| ``ACS_BOARD`` | board of squares | -+------------------+------------------------------------------+ -| ``ACS_BSBS`` | alternate name for horizontal line | -+------------------+------------------------------------------+ -| ``ACS_BSSB`` | alternate name for upper left corner | -+------------------+------------------------------------------+ -| ``ACS_BSSS`` | alternate name for top tee | -+------------------+------------------------------------------+ -| ``ACS_BTEE`` | bottom tee | -+------------------+------------------------------------------+ -| ``ACS_BULLET`` | bullet | -+------------------+------------------------------------------+ -| ``ACS_CKBOARD`` | checker board (stipple) | -+------------------+------------------------------------------+ -| ``ACS_DARROW`` | arrow pointing down | -+------------------+------------------------------------------+ -| ``ACS_DEGREE`` | degree symbol | -+------------------+------------------------------------------+ -| ``ACS_DIAMOND`` | diamond | -+------------------+------------------------------------------+ -| ``ACS_GEQUAL`` | greater-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_HLINE`` | horizontal line | -+------------------+------------------------------------------+ -| ``ACS_LANTERN`` | lantern symbol | -+------------------+------------------------------------------+ -| ``ACS_LARROW`` | left arrow | -+------------------+------------------------------------------+ -| ``ACS_LEQUAL`` | less-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_LLCORNER`` | lower left-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LRCORNER`` | lower right-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LTEE`` | left tee | -+------------------+------------------------------------------+ -| ``ACS_NEQUAL`` | not-equal sign | -+------------------+------------------------------------------+ -| ``ACS_PI`` | letter pi | -+------------------+------------------------------------------+ -| ``ACS_PLMINUS`` | plus-or-minus sign | -+------------------+------------------------------------------+ -| ``ACS_PLUS`` | big plus sign | -+------------------+------------------------------------------+ -| ``ACS_RARROW`` | right arrow | -+------------------+------------------------------------------+ -| ``ACS_RTEE`` | right tee | -+------------------+------------------------------------------+ -| ``ACS_S1`` | scan line 1 | -+------------------+------------------------------------------+ -| ``ACS_S3`` | scan line 3 | -+------------------+------------------------------------------+ -| ``ACS_S7`` | scan line 7 | -+------------------+------------------------------------------+ -| ``ACS_S9`` | scan line 9 | -+------------------+------------------------------------------+ -| ``ACS_SBBS`` | alternate name for lower right corner | -+------------------+------------------------------------------+ -| ``ACS_SBSB`` | alternate name for vertical line | -+------------------+------------------------------------------+ -| ``ACS_SBSS`` | alternate name for right tee | -+------------------+------------------------------------------+ -| ``ACS_SSBB`` | alternate name for lower left corner | -+------------------+------------------------------------------+ -| ``ACS_SSBS`` | alternate name for bottom tee | -+------------------+------------------------------------------+ -| ``ACS_SSSB`` | alternate name for left tee | -+------------------+------------------------------------------+ -| ``ACS_SSSS`` | alternate name for crossover or big plus | -+------------------+------------------------------------------+ -| ``ACS_STERLING`` | pound sterling | -+------------------+------------------------------------------+ -| ``ACS_TTEE`` | top tee | -+------------------+------------------------------------------+ -| ``ACS_UARROW`` | up arrow | -+------------------+------------------------------------------+ -| ``ACS_ULCORNER`` | upper left corner | -+------------------+------------------------------------------+ -| ``ACS_URCORNER`` | upper right corner | -+------------------+------------------------------------------+ -| ``ACS_VLINE`` | vertical line | -+------------------+------------------------------------------+ - -The following table lists the predefined colors: - -+-------------------+----------------------------+ -| Constant | Color | -+===================+============================+ -| ``COLOR_BLACK`` | Black | -+-------------------+----------------------------+ -| ``COLOR_BLUE`` | Blue | -+-------------------+----------------------------+ -| ``COLOR_CYAN`` | Cyan (light greenish blue) | -+-------------------+----------------------------+ -| ``COLOR_GREEN`` | Green | -+-------------------+----------------------------+ -| ``COLOR_MAGENTA`` | Magenta (purplish red) | -+-------------------+----------------------------+ -| ``COLOR_RED`` | Red | -+-------------------+----------------------------+ -| ``COLOR_WHITE`` | White | -+-------------------+----------------------------+ -| ``COLOR_YELLOW`` | Yellow | -+-------------------+----------------------------+ - - -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== - -.. module:: curses.textpad - :synopsis: Emacs-like input editing in a curses window. -.. moduleauthor:: Eric Raymond -.. sectionauthor:: Eric Raymond - - -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles -elementary text editing in a curses window, supporting a set of keybindings -resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, -FrameMaker, and many other programs). The module also provides a -rectangle-drawing function useful for framing text boxes or for other purposes. - -The module :mod:`curses.textpad` defines the following function: - - -.. function:: rectangle(win, uly, ulx, lry, lrx) - - Draw a rectangle. The first argument must be a window object; the remaining - arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the - rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using - VT100/IBM PC forms characters on terminals that make this possible (including - xterm and most other software terminal emulators). Otherwise it will be drawn - with ASCII dashes, vertical bars, and plus signs. - - -.. _curses-textpad-objects: - -Textbox objects ---------------- - -You can instantiate a :class:`Textbox` object as follows: - - -.. class:: Textbox(win) - - Return a textbox widget object. The *win* argument should be a curses - :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. - The instance's :attr:`stripspaces` flag is initially on. - - :class:`Textbox` objects have the following methods: - - - .. method:: edit([validator]) - - This is the entry point you will normally use. It accepts editing - keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for - each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a - string; whether blanks in the window are included is affected by the - :attr:`stripspaces` attribute. - - - .. method:: do_command(ch) - - Process a single command keystroke. Here are the supported special - keystrokes: - - +------------------+-------------------------------------------+ - | Keystroke | Action | - +==================+===========================================+ - | :kbd:`Control-A` | Go to left edge of window. | - +------------------+-------------------------------------------+ - | :kbd:`Control-B` | Cursor left, wrapping to previous line if | - | | appropriate. | - +------------------+-------------------------------------------+ - | :kbd:`Control-D` | Delete character under cursor. | - +------------------+-------------------------------------------+ - | :kbd:`Control-E` | Go to right edge (stripspaces off) or end | - | | of line (stripspaces on). | - +------------------+-------------------------------------------+ - | :kbd:`Control-F` | Cursor right, wrapping to next line when | - | | appropriate. | - +------------------+-------------------------------------------+ - | :kbd:`Control-G` | Terminate, returning the window contents. | - +------------------+-------------------------------------------+ - | :kbd:`Control-H` | Delete character backward. | - +------------------+-------------------------------------------+ - | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | - +------------------+-------------------------------------------+ - | :kbd:`Control-K` | If line is blank, delete it, otherwise | - | | clear to end of line. | - +------------------+-------------------------------------------+ - | :kbd:`Control-L` | Refresh screen. | - +------------------+-------------------------------------------+ - | :kbd:`Control-N` | Cursor down; move down one line. | - +------------------+-------------------------------------------+ - | :kbd:`Control-O` | Insert a blank line at cursor location. | - +------------------+-------------------------------------------+ - | :kbd:`Control-P` | Cursor up; move up one line. | - +------------------+-------------------------------------------+ - - Move operations do nothing if the cursor is at an edge where the movement - is not possible. The following synonyms are supported where possible: - - +------------------------+------------------+ - | Constant | Keystroke | - +========================+==================+ - | :const:`KEY_LEFT` | :kbd:`Control-B` | - +------------------------+------------------+ - | :const:`KEY_RIGHT` | :kbd:`Control-F` | - +------------------------+------------------+ - | :const:`KEY_UP` | :kbd:`Control-P` | - +------------------------+------------------+ - | :const:`KEY_DOWN` | :kbd:`Control-N` | - +------------------------+------------------+ - | :const:`KEY_BACKSPACE` | :kbd:`Control-h` | - +------------------------+------------------+ - - All other keystrokes are treated as a command to insert the given - character and move right (with line wrapping). - - - .. method:: gather() - - Return the window contents as a string; whether blanks in the - window are included is affected by the :attr:`stripspaces` member. - - - .. attribute:: stripspaces - - This attribute is a flag which controls the interpretation of blanks in - the window. When it is on, trailing blanks on each line are ignored; any - cursor motion that would land the cursor on a trailing blank goes to the - end of that line instead, and trailing blanks are stripped when the window - contents are gathered. diff --git a/third_party/python/Doc/library/custominterp.rst b/third_party/python/Doc/library/custominterp.rst deleted file mode 100644 index 5eeced20a..000000000 --- a/third_party/python/Doc/library/custominterp.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _custominterp: - -************************** -Custom Python Interpreters -************************** - -The modules described in this chapter allow writing interfaces similar to -Python's interactive interpreter. If you want a Python interpreter that -supports some special feature in addition to the Python language, you should -look at the :mod:`code` module. (The :mod:`codeop` module is lower-level, used -to support compiling a possibly-incomplete chunk of Python code.) - -The full list of modules described in this chapter is: - - -.. toctree:: - - code.rst - codeop.rst diff --git a/third_party/python/Doc/library/datatypes.rst b/third_party/python/Doc/library/datatypes.rst deleted file mode 100644 index 48af0823a..000000000 --- a/third_party/python/Doc/library/datatypes.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _datatypes: - -********** -Data Types -********** - -The modules described in this chapter provide a variety of specialized data -types such as dates and times, fixed-type arrays, heap queues, synchronized -queues, and sets. - -Python also provides some built-in data types, in particular, -:class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and -:class:`tuple`. The :class:`str` class is used to hold -Unicode strings, and the :class:`bytes` class is used to hold binary data. - -The following modules are documented in this chapter: - - -.. toctree:: - - datetime.rst - calendar.rst - collections.rst - collections.abc.rst - heapq.rst - bisect.rst - array.rst - weakref.rst - types.rst - copy.rst - pprint.rst - reprlib.rst - enum.rst diff --git a/third_party/python/Doc/library/datetime.rst b/third_party/python/Doc/library/datetime.rst deleted file mode 100644 index 25f2fd3f4..000000000 --- a/third_party/python/Doc/library/datetime.rst +++ /dev/null @@ -1,2172 +0,0 @@ -:mod:`datetime` --- Basic date and time types -============================================= - -.. module:: datetime - :synopsis: Basic date and time types. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: A.M. Kuchling - -**Source code:** :source:`Lib/datetime.py` - --------------- - -.. XXX what order should the types be discussed in? - -The :mod:`datetime` module supplies classes for manipulating dates and times in -both simple and complex ways. While date and time arithmetic is supported, the -focus of the implementation is on efficient attribute extraction for output -formatting and manipulation. For related functionality, see also the -:mod:`time` and :mod:`calendar` modules. - -There are two kinds of date and time objects: "naive" and "aware". - -An aware object has sufficient knowledge of applicable algorithmic and -political time adjustments, such as time zone and daylight saving time -information, to locate itself relative to other aware objects. An aware object -is used to represent a specific moment in time that is not open to -interpretation [#]_. - -A naive object does not contain enough information to unambiguously locate -itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is -purely up to the program, just like it is up to the program whether a -particular number represents metres, miles, or mass. Naive objects are easy to -understand and to work with, at the cost of ignoring some aspects of reality. - -For applications requiring aware objects, :class:`.datetime` and :class:`.time` -objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. -These :class:`tzinfo` objects capture information about the offset from UTC -time, the time zone name, and whether Daylight Saving Time is in effect. Note -that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offset from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of -detail is up to the application. The rules for time adjustment across the -world are more political than rational, change frequently, and there is no -standard suitable for every application aside from UTC. - -The :mod:`datetime` module exports the following constants: - -.. data:: MINYEAR - - The smallest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MINYEAR` is ``1``. - - -.. data:: MAXYEAR - - The largest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MAXYEAR` is ``9999``. - - -.. seealso:: - - Module :mod:`calendar` - General calendar related functions. - - Module :mod:`time` - Time access and conversions. - - -Available Types ---------------- - -.. class:: date - :noindex: - - An idealized naive date, assuming the current Gregorian calendar always was, and - always will be, in effect. Attributes: :attr:`year`, :attr:`month`, and - :attr:`day`. - - -.. class:: time - :noindex: - - An idealized time, independent of any particular day, assuming that every day - has exactly 24\*60\*60 seconds (there is no notion of "leap seconds" here). - Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, - and :attr:`.tzinfo`. - - -.. class:: datetime - :noindex: - - A combination of a date and a time. Attributes: :attr:`year`, :attr:`month`, - :attr:`day`, :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, - and :attr:`.tzinfo`. - - -.. class:: timedelta - :noindex: - - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. - - -.. class:: tzinfo - :noindex: - - An abstract base class for time zone information objects. These are used by the - :class:`.datetime` and :class:`.time` classes to provide a customizable notion of - time adjustment (for example, to account for time zone and/or daylight saving - time). - -.. class:: timezone - :noindex: - - A class that implements the :class:`tzinfo` abstract base class as a - fixed offset from the UTC. - - .. versionadded:: 3.2 - - -Objects of these types are immutable. - -Objects of the :class:`date` type are always naive. - -An object of type :class:`.time` or :class:`.datetime` may be naive or aware. -A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and -``d.tzinfo.utcoffset(d)`` does not return ``None``. If ``d.tzinfo`` is -``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)`` -returns ``None``, *d* is naive. A :class:`.time` object *t* is aware -if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return -``None``. Otherwise, *t* is naive. - -The distinction between naive and aware doesn't apply to :class:`timedelta` -objects. - -Subclass relationships:: - - object - timedelta - tzinfo - timezone - time - date - datetime - - -.. _datetime-timedelta: - -:class:`timedelta` Objects --------------------------- - -A :class:`timedelta` object represents a duration, the difference between two -dates or times. - -.. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - - All arguments are optional and default to ``0``. Arguments may be integers - or floats, and may be positive or negative. - - Only *days*, *seconds* and *microseconds* are stored internally. Arguments are - converted to those units: - - * A millisecond is converted to 1000 microseconds. - * A minute is converted to 60 seconds. - * An hour is converted to 3600 seconds. - * A week is converted to 7 days. - - and days, seconds and microseconds are then normalized so that the - representation is unique, with - - * ``0 <= microseconds < 1000000`` - * ``0 <= seconds < 3600*24`` (the number of seconds in one day) - * ``-999999999 <= days <= 999999999`` - - If any argument is a float and there are fractional microseconds, - the fractional microseconds left over from all arguments are - combined and their sum is rounded to the nearest microsecond using - round-half-to-even tiebreaker. If no argument is a float, the - conversion and normalization processes are exact (no information is - lost). - - If the normalized value of days lies outside the indicated range, - :exc:`OverflowError` is raised. - - Note that normalization of negative values may be surprising at first. For - example, - - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) - >>> (d.days, d.seconds, d.microseconds) - (-1, 86399, 999999) - - -Class attributes are: - -.. attribute:: timedelta.min - - The most negative :class:`timedelta` object, ``timedelta(-999999999)``. - - -.. attribute:: timedelta.max - - The most positive :class:`timedelta` object, ``timedelta(days=999999999, - hours=23, minutes=59, seconds=59, microseconds=999999)``. - - -.. attribute:: timedelta.resolution - - The smallest possible difference between non-equal :class:`timedelta` objects, - ``timedelta(microseconds=1)``. - -Note that, because of normalization, ``timedelta.max`` > ``-timedelta.min``. -``-timedelta.max`` is not representable as a :class:`timedelta` object. - -Instance attributes (read-only): - -+------------------+--------------------------------------------+ -| Attribute | Value | -+==================+============================================+ -| ``days`` | Between -999999999 and 999999999 inclusive | -+------------------+--------------------------------------------+ -| ``seconds`` | Between 0 and 86399 inclusive | -+------------------+--------------------------------------------+ -| ``microseconds`` | Between 0 and 999999 inclusive | -+------------------+--------------------------------------------+ - -Supported operations: - -.. XXX this table is too wide! - -+--------------------------------+-----------------------------------------------+ -| Operation | Result | -+================================+===============================================+ -| ``t1 = t2 + t3`` | Sum of *t2* and *t3*. Afterwards *t1*-*t2* == | -| | *t3* and *t1*-*t3* == *t2* are true. (1) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | -| | == *t2* - *t3* and *t2* == *t1* + *t3* are | -| | true. (1)(6) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | -| | Afterwards *t1* // i == *t2* is true, | -| | provided ``i != 0``. | -+--------------------------------+-----------------------------------------------+ -| | In general, *t1* \* i == *t1* \* (i-1) + *t1* | -| | is true. (1) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 * f or t1 = f * t2`` | Delta multiplied by a float. The result is | -| | rounded to the nearest multiple of | -| | timedelta.resolution using round-half-to-even.| -+--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of *t2* by *t3*. Returns a | -| | :class:`float` object. | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| -| | is rounded to the nearest multiple of | -| | timedelta.resolution using round-half-to-even.| -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 // i`` or | The floor is computed and the remainder (if | -| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | -| | integer is returned. (3) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 % t3`` | The remainder is computed as a | -| | :class:`timedelta` object. (3) | -+--------------------------------+-----------------------------------------------+ -| ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: | -| | ``q = t1 // t2`` (3) and ``r = t1 % t2``. | -| | q is an integer and r is a :class:`timedelta` | -| | object. | -+--------------------------------+-----------------------------------------------+ -| ``+t1`` | Returns a :class:`timedelta` object with the | -| | same value. (2) | -+--------------------------------+-----------------------------------------------+ -| ``-t1`` | equivalent to | -| | :class:`timedelta`\ (-*t1.days*, | -| | -*t1.seconds*, -*t1.microseconds*), | -| | and to *t1*\* -1. (1)(4) | -+--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, and| -| | to -*t* when ``t.days < 0``. (2) | -+--------------------------------+-----------------------------------------------+ -| ``str(t)`` | Returns a string in the form | -| | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | -| | is negative for negative ``t``. (5) | -+--------------------------------+-----------------------------------------------+ -| ``repr(t)`` | Returns a string in the form | -| | ``datetime.timedelta(D[, S[, U]])``, where D | -| | is negative for negative ``t``. (5) | -+--------------------------------+-----------------------------------------------+ - -Notes: - -(1) - This is exact, but may overflow. - -(2) - This is exact, and cannot overflow. - -(3) - Division by 0 raises :exc:`ZeroDivisionError`. - -(4) - -*timedelta.max* is not representable as a :class:`timedelta` object. - -(5) - String representations of :class:`timedelta` objects are normalized - similarly to their internal representation. This leads to somewhat - unusual results for negative timedeltas. For example: - - >>> timedelta(hours=-5) - datetime.timedelta(-1, 68400) - >>> print(_) - -1 day, 19:00:00 - -(6) - The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except - when t3 is equal to ``timedelta.max``; in that case the former will produce a result - while the latter will overflow. - -In addition to the operations listed above :class:`timedelta` objects support -certain additions and subtractions with :class:`date` and :class:`.datetime` -objects (see below). - -.. versionchanged:: 3.2 - Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and - the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. - - -Comparisons of :class:`timedelta` objects are supported with the -:class:`timedelta` object representing the smaller duration considered to be the -smaller timedelta. In order to stop mixed-type comparisons from falling back to -the default comparison by object address, when a :class:`timedelta` object is -compared to an object of a different type, :exc:`TypeError` is raised unless the -comparison is ``==`` or ``!=``. The latter cases return :const:`False` or -:const:`True`, respectively. - -:class:`timedelta` objects are :term:`hashable` (usable as dictionary keys), support -efficient pickling, and in Boolean contexts, a :class:`timedelta` object is -considered to be true if and only if it isn't equal to ``timedelta(0)``. - -Instance methods: - -.. method:: timedelta.total_seconds() - - Return the total number of seconds contained in the duration. Equivalent to - ``td / timedelta(seconds=1)``. - - Note that for very large time intervals (greater than 270 years on - most platforms) this method will lose microsecond accuracy. - - .. versionadded:: 3.2 - - -Example usage: - - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) # adds up to 365 days - >>> year.total_seconds() - 31536000.0 - >>> year == another_year - True - >>> ten_years = 10 * year - >>> ten_years, ten_years.days // 365 - (datetime.timedelta(3650), 10) - >>> nine_years = ten_years - year - >>> nine_years, nine_years.days // 365 - (datetime.timedelta(3285), 9) - >>> three_years = nine_years // 3; - >>> three_years, three_years.days // 365 - (datetime.timedelta(1095), 3) - >>> abs(three_years - ten_years) == 2 * three_years + year - True - - -.. _datetime-date: - -:class:`date` Objects ---------------------- - -A :class:`date` object represents a date (year, month and day) in an idealized -calendar, the current Gregorian calendar indefinitely extended in both -directions. January 1 of year 1 is called day number 1, January 2 of year 1 is -called day number 2, and so on. This matches the definition of the "proleptic -Gregorian" calendar in Dershowitz and Reingold's book Calendrical Calculations, -where it's the base calendar for all computations. See the book for algorithms -for converting between proleptic Gregorian ordinals and many other calendar -systems. - - -.. class:: date(year, month, day) - - All arguments are required. Arguments may be integers, in the following - ranges: - - * ``MINYEAR <= year <= MAXYEAR`` - * ``1 <= month <= 12`` - * ``1 <= day <= number of days in the given month and year`` - - If an argument outside those ranges is given, :exc:`ValueError` is raised. - - -Other constructors, all class methods: - -.. classmethod:: date.today() - - Return the current local date. This is equivalent to - ``date.fromtimestamp(time.time())``. - - -.. classmethod:: date.fromtimestamp(timestamp) - - Return the local date corresponding to the POSIX timestamp, such as is returned - by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out - of the range of values supported by the platform C :c:func:`localtime` function, - and :exc:`OSError` on :c:func:`localtime` failure. - It's common for this to be restricted to years from 1970 through 2038. Note - that on non-POSIX systems that include leap seconds in their notion of a - timestamp, leap seconds are ignored by :meth:`fromtimestamp`. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`localtime` function. Raise :exc:`OSError` instead of - :exc:`ValueError` on :c:func:`localtime` failure. - - -.. classmethod:: date.fromordinal(ordinal) - - Return the date corresponding to the proleptic Gregorian ordinal, where January - 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, ``date.fromordinal(d.toordinal()) == - d``. - - -Class attributes: - -.. attribute:: date.min - - The earliest representable date, ``date(MINYEAR, 1, 1)``. - - -.. attribute:: date.max - - The latest representable date, ``date(MAXYEAR, 12, 31)``. - - -.. attribute:: date.resolution - - The smallest possible difference between non-equal date objects, - ``timedelta(days=1)``. - - -Instance attributes (read-only): - -.. attribute:: date.year - - Between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - -.. attribute:: date.month - - Between 1 and 12 inclusive. - - -.. attribute:: date.day - - Between 1 and the number of days in the given month of the given year. - - -Supported operations: - -+-------------------------------+----------------------------------------------+ -| Operation | Result | -+===============================+==============================================+ -| ``date2 = date1 + timedelta`` | *date2* is ``timedelta.days`` days removed | -| | from *date1*. (1) | -+-------------------------------+----------------------------------------------+ -| ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | -| | timedelta == date1``. (2) | -+-------------------------------+----------------------------------------------+ -| ``timedelta = date1 - date2`` | \(3) | -+-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | -+-------------------------------+----------------------------------------------+ - -Notes: - -(1) - *date2* is moved forward in time if ``timedelta.days > 0``, or backward if - ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. - ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. - :exc:`OverflowError` is raised if ``date2.year`` would be smaller than - :const:`MINYEAR` or larger than :const:`MAXYEAR`. - -(2) - ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. - -(3) - This is exact, and cannot overflow. timedelta.seconds and - timedelta.microseconds are 0, and date2 + timedelta == date1 after. - -(4) - In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. - -Dates can be used as dictionary keys. In Boolean contexts, all :class:`date` -objects are considered to be true. - -Instance methods: - -.. method:: date.replace(year=self.year, month=self.month, day=self.day) - - Return a date with the same value, except for those parameters given new - values by whichever keyword arguments are specified. For example, if ``d == - date(2002, 12, 31)``, then ``d.replace(day=26) == date(2002, 12, 26)``. - - -.. method:: date.timetuple() - - Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - The hours, minutes and seconds are 0, and the DST flag is -1. ``d.timetuple()`` - is equivalent to ``time.struct_time((d.year, d.month, d.day, 0, 0, 0, - d.weekday(), yday, -1))``, where ``yday = d.toordinal() - date(d.year, 1, - 1).toordinal() + 1`` is the day number within the current year starting with - ``1`` for January 1st. - - -.. method:: date.toordinal() - - Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, - ``date.fromordinal(d.toordinal()) == d``. - - -.. method:: date.weekday() - - Return the day of the week as an integer, where Monday is 0 and Sunday is 6. - For example, ``date(2002, 12, 4).weekday() == 2``, a Wednesday. See also - :meth:`isoweekday`. - - -.. method:: date.isoweekday() - - Return the day of the week as an integer, where Monday is 1 and Sunday is 7. - For example, ``date(2002, 12, 4).isoweekday() == 3``, a Wednesday. See also - :meth:`weekday`, :meth:`isocalendar`. - - -.. method:: date.isocalendar() - - Return a 3-tuple, (ISO year, ISO week number, ISO weekday). - - The ISO calendar is a widely used variant of the Gregorian calendar. See - https://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good - explanation. - - The ISO year consists of 52 or 53 full weeks, and where a week starts on a - Monday and ends on a Sunday. The first week of an ISO year is the first - (Gregorian) calendar week of a year containing a Thursday. This is called week - number 1, and the ISO year of that Thursday is the same as its Gregorian year. - - For example, 2004 begins on a Thursday, so the first week of ISO year 2004 - begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004, so that - ``date(2003, 12, 29).isocalendar() == (2004, 1, 1)`` and ``date(2004, 1, - 4).isocalendar() == (2004, 1, 7)``. - - -.. method:: date.isoformat() - - Return a string representing the date in ISO 8601 format, 'YYYY-MM-DD'. For - example, ``date(2002, 12, 4).isoformat() == '2002-12-04'``. - - -.. method:: date.__str__() - - For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``. - - -.. method:: date.ctime() - - Return a string representing the date, for example ``date(2002, 12, - 4).ctime() == 'Wed Dec 4 00:00:00 2002'``. ``d.ctime()`` is equivalent to - ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the native C - :c:func:`ctime` function (which :func:`time.ctime` invokes, but which - :meth:`date.ctime` does not invoke) conforms to the C standard. - - -.. method:: date.strftime(format) - - Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: date.__format__(format) - - Same as :meth:`.date.strftime`. This makes it possible to specify a format - string for a :class:`.date` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -Example of counting days to an event:: - - >>> import time - >>> from datetime import date - >>> today = date.today() - >>> today - datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) - True - >>> my_birthday = date(today.year, 6, 24) - >>> if my_birthday < today: - ... my_birthday = my_birthday.replace(year=today.year + 1) - >>> my_birthday - datetime.date(2008, 6, 24) - >>> time_to_birthday = abs(my_birthday - today) - >>> time_to_birthday.days - 202 - -Example of working with :class:`date`: - -.. doctest:: - - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 - >>> d - datetime.date(2002, 3, 11) - >>> t = d.timetuple() - >>> for i in t: # doctest: +SKIP - ... print(i) - 2002 # year - 3 # month - 11 # day - 0 - 0 - 0 - 0 # weekday (0 = Monday) - 70 # 70th day in the year - -1 - >>> ic = d.isocalendar() - >>> for i in ic: # doctest: +SKIP - ... print(i) - 2002 # ISO year - 11 # ISO week number - 1 # ISO day number ( 1 = Monday ) - >>> d.isoformat() - '2002-03-11' - >>> d.strftime("%d/%m/%y") - '11/03/02' - >>> d.strftime("%A %d. %B %Y") - 'Monday 11. March 2002' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") - 'The day is 11, the month is March.' - - -.. _datetime-datetime: - -:class:`.datetime` Objects --------------------------- - -A :class:`.datetime` object is a single object containing all the information -from a :class:`date` object and a :class:`.time` object. Like a :class:`date` -object, :class:`.datetime` assumes the current Gregorian calendar extended in -both directions; like a time object, :class:`.datetime` assumes there are exactly -3600\*24 seconds in every day. - -Constructor: - -.. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - - The year, month and day arguments are required. *tzinfo* may be ``None``, or an - instance of a :class:`tzinfo` subclass. The remaining arguments may be integers, - in the following ranges: - - * ``MINYEAR <= year <= MAXYEAR``, - * ``1 <= month <= 12``, - * ``1 <= day <= number of days in the given month and year``, - * ``0 <= hour < 24``, - * ``0 <= minute < 60``, - * ``0 <= second < 60``, - * ``0 <= microsecond < 1000000``, - * ``fold in [0, 1]``. - - If an argument outside those ranges is given, :exc:`ValueError` is raised. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - -Other constructors, all class methods: - -.. classmethod:: datetime.today() - - Return the current local datetime, with :attr:`.tzinfo` ``None``. This is - equivalent to ``datetime.fromtimestamp(time.time())``. See also :meth:`now`, - :meth:`fromtimestamp`. - - -.. classmethod:: datetime.now(tz=None) - - Return the current local date and time. If optional argument *tz* is ``None`` - or not specified, this is like :meth:`today`, but, if possible, supplies more - precision than can be gotten from going through a :func:`time.time` timestamp - (for example, this may be possible on platforms supplying the C - :c:func:`gettimeofday` function). - - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - current date and time are converted to *tz*’s time zone. In this case the - result is equivalent to ``tz.fromutc(datetime.utcnow().replace(tzinfo=tz))``. - See also :meth:`today`, :meth:`utcnow`. - - -.. classmethod:: datetime.utcnow() - - Return the current UTC date and time, with :attr:`.tzinfo` ``None``. This is like - :meth:`now`, but returns the current UTC date and time, as a naive - :class:`.datetime` object. An aware current UTC datetime can be obtained by - calling ``datetime.now(timezone.utc)``. See also :meth:`now`. - -.. classmethod:: datetime.fromtimestamp(timestamp, tz=None) - - Return the local date and time corresponding to the POSIX timestamp, such as is - returned by :func:`time.time`. If optional argument *tz* is ``None`` or not - specified, the timestamp is converted to the platform's local date and time, and - the returned :class:`.datetime` object is naive. - - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - timestamp is converted to *tz*’s time zone. In this case the result is - equivalent to - ``tz.fromutc(datetime.utcfromtimestamp(timestamp).replace(tzinfo=tz))``. - - :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of - the range of values supported by the platform C :c:func:`localtime` or - :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or - :c:func:`gmtime` failure. - It's common for this to be restricted to years in - 1970 through 2038. Note that on non-POSIX systems that include leap seconds in - their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`, - and then it's possible to have two timestamps differing by a second that yield - identical :class:`.datetime` objects. See also :meth:`utcfromtimestamp`. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`localtime` or :c:func:`gmtime` functions. Raise :exc:`OSError` - instead of :exc:`ValueError` on :c:func:`localtime` or :c:func:`gmtime` - failure. - - .. versionchanged:: 3.6 - :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. - -.. classmethod:: datetime.utcfromtimestamp(timestamp) - - Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with - :attr:`.tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is - out of the range of values supported by the platform C :c:func:`gmtime` function, - and :exc:`OSError` on :c:func:`gmtime` failure. - It's common for this to be restricted to years in 1970 through 2038. - - To get an aware :class:`.datetime` object, call :meth:`fromtimestamp`:: - - datetime.fromtimestamp(timestamp, timezone.utc) - - On the POSIX compliant platforms, it is equivalent to the following - expression:: - - datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=timestamp) - - except the latter formula always supports the full years range: between - :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`gmtime` function. Raise :exc:`OSError` instead of - :exc:`ValueError` on :c:func:`gmtime` failure. - - -.. classmethod:: datetime.fromordinal(ordinal) - - Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, - where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and - microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. - - -.. classmethod:: datetime.combine(date, time, tzinfo=self.tzinfo) - - Return a new :class:`.datetime` object whose date components are equal to the - given :class:`date` object's, and whose time components - are equal to the given :class:`.time` object's. If the *tzinfo* - argument is provided, its value is used to set the :attr:`.tzinfo` attribute - of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. - - For any :class:`.datetime` object *d*, - ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a - :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes - are ignored. - - .. versionchanged:: 3.6 - Added the *tzinfo* argument. - - -.. classmethod:: datetime.strptime(date_string, format) - - Return a :class:`.datetime` corresponding to *date_string*, parsed according to - *format*. This is equivalent to ``datetime(*(time.strptime(date_string, - format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format - can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - - -Class attributes: - -.. attribute:: datetime.min - - The earliest representable :class:`.datetime`, ``datetime(MINYEAR, 1, 1, - tzinfo=None)``. - - -.. attribute:: datetime.max - - The latest representable :class:`.datetime`, ``datetime(MAXYEAR, 12, 31, 23, 59, - 59, 999999, tzinfo=None)``. - - -.. attribute:: datetime.resolution - - The smallest possible difference between non-equal :class:`.datetime` objects, - ``timedelta(microseconds=1)``. - - -Instance attributes (read-only): - -.. attribute:: datetime.year - - Between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - -.. attribute:: datetime.month - - Between 1 and 12 inclusive. - - -.. attribute:: datetime.day - - Between 1 and the number of days in the given month of the given year. - - -.. attribute:: datetime.hour - - In ``range(24)``. - - -.. attribute:: datetime.minute - - In ``range(60)``. - - -.. attribute:: datetime.second - - In ``range(60)``. - - -.. attribute:: datetime.microsecond - - In ``range(1000000)``. - - -.. attribute:: datetime.tzinfo - - The object passed as the *tzinfo* argument to the :class:`.datetime` constructor, - or ``None`` if none was passed. - - -.. attribute:: datetime.fold - - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A - repeated interval occurs when clocks are rolled back at the end of daylight saving - time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. - - .. versionadded:: 3.6 - -Supported operations: - -+---------------------------------------+--------------------------------+ -| Operation | Result | -+=======================================+================================+ -| ``datetime2 = datetime1 + timedelta`` | \(1) | -+---------------------------------------+--------------------------------+ -| ``datetime2 = datetime1 - timedelta`` | \(2) | -+---------------------------------------+--------------------------------+ -| ``timedelta = datetime1 - datetime2`` | \(3) | -+---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | -+---------------------------------------+--------------------------------+ - -(1) - datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The - result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and - datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if - datetime2.year would be smaller than :const:`MINYEAR` or larger than - :const:`MAXYEAR`. Note that no time zone adjustments are done even if the - input is an aware object. - -(2) - Computes the datetime2 such that datetime2 + timedelta == datetime1. As for - addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input - datetime, and no time zone adjustments are done even if the input is aware. - -(3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if - both operands are naive, or if both are aware. If one is aware and the other is - naive, :exc:`TypeError` is raised. - - If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, - the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments - are done in this case. - - If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The - result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - - b.utcoffset())`` except that the implementation never overflows. - -(4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. - - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). - - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`.datetime` - instances don't raise :exc:`TypeError`. - - .. note:: - - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. - -:class:`.datetime` objects can be used as dictionary keys. In Boolean contexts, -all :class:`.datetime` objects are considered to be true. - -Instance methods: - -.. method:: datetime.date() - - Return :class:`date` object with same year, month and day. - - -.. method:: datetime.time() - - Return :class:`.time` object with same hour, minute, second, microsecond and fold. - :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. - - .. versionchanged:: 3.6 - The fold value is copied to the returned :class:`.time` object. - - -.. method:: datetime.timetz() - - Return :class:`.time` object with same hour, minute, second, microsecond, fold, and - tzinfo attributes. See also method :meth:`time`. - - .. versionchanged:: 3.6 - The fold value is copied to the returned :class:`.time` object. - - -.. method:: datetime.replace(year=self.year, month=self.month, day=self.day, \ - hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ - tzinfo=self.tzinfo, * fold=0) - - Return a datetime with the same attributes, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive datetime from an aware - datetime with no conversion of date and time data. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - - -.. method:: datetime.astimezone(tz=None) - - Return a :class:`.datetime` object with new :attr:`.tzinfo` attribute *tz*, - adjusting the date and time data so the result is the same UTC time as - *self*, but in *tz*'s local time. - - If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its - :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* - is naive, it is presumed to represent time in the system timezone. - - If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted - datetime instance will be set to an instance of :class:`timezone` - with the zone name and offset obtained from the OS. - - If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no - adjustment of date or time data is performed. Else the result is local - time in the timezone *tz*, representing the same UTC time as *self*: after - ``astz = dt.astimezone(tz)``, ``astz - astz.utcoffset()`` will have - the same date and time data as ``dt - dt.utcoffset()``. - - If you merely want to attach a time zone object *tz* to a datetime *dt* without - adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you - merely want to remove the time zone object from an aware datetime *dt* without - conversion of date and time data, use ``dt.replace(tzinfo=None)``. - - Note that the default :meth:`tzinfo.fromutc` method can be overridden in a - :class:`tzinfo` subclass to affect the result returned by :meth:`astimezone`. - Ignoring error cases, :meth:`astimezone` acts like:: - - def astimezone(self, tz): - if self.tzinfo is tz: - return self - # Convert self to UTC, and attach the new time zone object. - utc = (self - self.utcoffset()).replace(tzinfo=tz) - # Convert from UTC to tz's local time. - return tz.fromutc(utc) - - .. versionchanged:: 3.3 - *tz* now can be omitted. - - .. versionchanged:: 3.6 - The :meth:`astimezone` method can now be called on naive instances that - are presumed to represent system local time. - - -.. method:: datetime.utcoffset() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.utcoffset(self)``, and raises an exception if the latter doesn't - return ``None``, or a :class:`timedelta` object representing a whole number of - minutes with magnitude less than one day. - - -.. method:: datetime.dst() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.dst(self)``, and raises an exception if the latter doesn't return - ``None``, or a :class:`timedelta` object representing a whole number of minutes - with magnitude less than one day. - - -.. method:: datetime.tzname() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.tzname(self)``, raises an exception if the latter doesn't return - ``None`` or a string object, - - -.. method:: datetime.timetuple() - - Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - ``d.timetuple()`` is equivalent to ``time.struct_time((d.year, d.month, d.day, - d.hour, d.minute, d.second, d.weekday(), yday, dst))``, where ``yday = - d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` is the day number within - the current year starting with ``1`` for January 1st. The :attr:`tm_isdst` flag - of the result is set according to the :meth:`dst` method: :attr:`.tzinfo` is - ``None`` or :meth:`dst` returns ``None``, :attr:`tm_isdst` is set to ``-1``; - else if :meth:`dst` returns a non-zero value, :attr:`tm_isdst` is set to ``1``; - else :attr:`tm_isdst` is set to ``0``. - - -.. method:: datetime.utctimetuple() - - If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what - ``d.dst()`` returns. DST is never in effect for a UTC time. - - If *d* is aware, *d* is normalized to UTC time, by subtracting - ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note - that an :exc:`OverflowError` may be raised if *d*.year was - ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year - boundary. - - -.. method:: datetime.toordinal() - - Return the proleptic Gregorian ordinal of the date. The same as - ``self.date().toordinal()``. - -.. method:: datetime.timestamp() - - Return POSIX timestamp corresponding to the :class:`.datetime` - instance. The return value is a :class:`float` similar to that - returned by :func:`time.time`. - - Naive :class:`.datetime` instances are assumed to represent local - time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` - supports wider range of values than :c:func:`mktime` on many - platforms, this method may raise :exc:`OverflowError` for times far - in the past or far in the future. - - For aware :class:`.datetime` instances, the return value is computed - as:: - - (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. note:: - - There is no method to obtain the POSIX timestamp directly from a - naive :class:`.datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not - set to UTC, you can obtain the POSIX timestamp by supplying - ``tzinfo=timezone.utc``:: - - timestamp = dt.replace(tzinfo=timezone.utc).timestamp() - - or by calculating the timestamp directly:: - - timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) - -.. method:: datetime.weekday() - - Return the day of the week as an integer, where Monday is 0 and Sunday is 6. - The same as ``self.date().weekday()``. See also :meth:`isoweekday`. - - -.. method:: datetime.isoweekday() - - Return the day of the week as an integer, where Monday is 1 and Sunday is 7. - The same as ``self.date().isoweekday()``. See also :meth:`weekday`, - :meth:`isocalendar`. - - -.. method:: datetime.isocalendar() - - Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The same as - ``self.date().isocalendar()``. - - -.. method:: datetime.isoformat(sep='T', timespec='auto') - - Return a string representing the date and time in ISO 8601 format, - YYYY-MM-DDTHH:MM:SS.mmmmmm or, if :attr:`microsecond` is 0, - YYYY-MM-DDTHH:MM:SS - - If :meth:`utcoffset` does not return ``None``, a 6-character string is - appended, giving the UTC offset in (signed) hours and minutes: - YYYY-MM-DDTHH:MM:SS.mmmmmm+HH:MM or, if :attr:`microsecond` is 0 - YYYY-MM-DDTHH:MM:SS+HH:MM - - The optional argument *sep* (default ``'T'``) is a one-character separator, - placed between the date and time portions of the result. For example, - - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): - ... def utcoffset(self, dt): return timedelta(minutes=-399) - ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') - '2002-12-25 00:00:00-06:39' - - The optional argument *timespec* specifies the number of additional - components of the time to include (the default is ``'auto'``). - It can be one of the following: - - - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, - same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. - - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. - - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. - - .. note:: - - Excluded time components are truncated, not rounded. - - :exc:`ValueError` will be raised on an invalid *timespec* argument. - - - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') - '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') - '2015-01-01T12:30:59.000000' - - .. versionadded:: 3.6 - Added the *timespec* argument. - - -.. method:: datetime.__str__() - - For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to - ``d.isoformat(' ')``. - - -.. method:: datetime.ctime() - - Return a string representing the date and time, for example ``datetime(2002, 12, - 4, 20, 30, 40).ctime() == 'Wed Dec 4 20:30:40 2002'``. ``d.ctime()`` is - equivalent to ``time.ctime(time.mktime(d.timetuple()))`` on platforms where the - native C :c:func:`ctime` function (which :func:`time.ctime` invokes, but which - :meth:`datetime.ctime` does not invoke) conforms to the C standard. - - -.. method:: datetime.strftime(format) - - Return a string representing the date and time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: datetime.__format__(format) - - Same as :meth:`.datetime.strftime`. This makes it possible to specify a format - string for a :class:`.datetime` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -Examples of working with datetime objects: - -.. doctest:: - - >>> from datetime import datetime, date, time - >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) - datetime.datetime(2005, 7, 14, 12, 30) - >>> # Using datetime.now() or datetime.utcnow() - >>> datetime.now() # doctest: +SKIP - datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.utcnow() # doctest: +SKIP - datetime.datetime(2007, 12, 6, 15, 29, 43, 79060) - >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt - datetime.datetime(2006, 11, 21, 16, 30) - >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() - >>> for it in tt: # doctest: +SKIP - ... print(it) - ... - 2006 # year - 11 # month - 21 # day - 16 # hour - 30 # minute - 0 # second - 1 # weekday (0 = Monday) - 325 # number of days since 1st January - -1 # dst - method tzinfo.dst() returned None - >>> # Date in ISO format - >>> ic = dt.isocalendar() - >>> for it in ic: # doctest: +SKIP - ... print(it) - ... - 2006 # ISO year - 47 # ISO week - 2 # ISO weekday - >>> # Formatting datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") - 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") - 'The day is 21, the month is November, the time is 04:30PM.' - -Using datetime with tzinfo: - - >>> from datetime import timedelta, datetime, tzinfo - >>> class GMT1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March - ... d = datetime(dt.year, 4, 1) # ends last Sunday in October - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) - ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +1" - ... - >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) - ... def dst(self, dt): - ... d = datetime(dt.year, 4, 1) - ... self.dston = d - timedelta(days=d.weekday() + 1) - ... d = datetime(dt.year, 11, 1) - ... self.dstoff = d - timedelta(days=d.weekday() + 1) - ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) - ... else: - ... return timedelta(0) - ... def tzname(self,dt): - ... return "GMT +2" - ... - >>> gmt1 = GMT1() - >>> # Daylight Saving Time - >>> dt1 = datetime(2006, 11, 21, 16, 30, tzinfo=gmt1) - >>> dt1.dst() - datetime.timedelta(0) - >>> dt1.utcoffset() - datetime.timedelta(0, 3600) - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=gmt1) - >>> dt2.dst() - datetime.timedelta(0, 3600) - >>> dt2.utcoffset() - datetime.timedelta(0, 7200) - >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(GMT2()) - >>> dt3 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 14, 0, tzinfo=) - >>> dt2 # doctest: +ELLIPSIS - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=) - >>> dt2.utctimetuple() == dt3.utctimetuple() - True - - - -.. _datetime-time: - -:class:`.time` Objects ----------------------- - -A time object represents a (local) time of day, independent of any particular -day, and subject to adjustment via a :class:`tzinfo` object. - -.. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - - All arguments are optional. *tzinfo* may be ``None``, or an instance of a - :class:`tzinfo` subclass. The remaining arguments may be integers, in the - following ranges: - - * ``0 <= hour < 24``, - * ``0 <= minute < 60``, - * ``0 <= second < 60``, - * ``0 <= microsecond < 1000000``, - * ``fold in [0, 1]``. - - If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to ``0`` except *tzinfo*, which defaults to :const:`None`. - -Class attributes: - - -.. attribute:: time.min - - The earliest representable :class:`.time`, ``time(0, 0, 0, 0)``. - - -.. attribute:: time.max - - The latest representable :class:`.time`, ``time(23, 59, 59, 999999)``. - - -.. attribute:: time.resolution - - The smallest possible difference between non-equal :class:`.time` objects, - ``timedelta(microseconds=1)``, although note that arithmetic on - :class:`.time` objects is not supported. - - -Instance attributes (read-only): - -.. attribute:: time.hour - - In ``range(24)``. - - -.. attribute:: time.minute - - In ``range(60)``. - - -.. attribute:: time.second - - In ``range(60)``. - - -.. attribute:: time.microsecond - - In ``range(1000000)``. - - -.. attribute:: time.tzinfo - - The object passed as the tzinfo argument to the :class:`.time` constructor, or - ``None`` if none was passed. - - -.. attribute:: time.fold - - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A - repeated interval occurs when clocks are rolled back at the end of daylight saving - time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. - - .. versionadded:: 3.6 - - -Supported operations: - -* comparison of :class:`.time` to :class:`.time`, where *a* is considered less - than *b* when *a* precedes *b* in time. If one comparand is naive and the other - is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have - the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is - ignored and the base times are compared. If both comparands are aware and - have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by - subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order - to stop mixed-type comparisons from falling back to the default comparison by - object address, when a :class:`.time` object is compared to an object of a - different type, :exc:`TypeError` is raised unless the comparison is ``==`` or - ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. - - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`~datetime.time` instances - don't raise :exc:`TypeError`. - -* hash, use as dict key - -* efficient pickling - -In boolean contexts, a :class:`.time` object is always considered to be true. - -.. versionchanged:: 3.5 - Before Python 3.5, a :class:`.time` object was considered to be false if it - represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. - -Instance methods: - -.. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ - microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0) - - Return a :class:`.time` with the same value, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive :class:`.time` from an - aware :class:`.time`, without conversion of the time data. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - - -.. method:: time.isoformat(timespec='auto') - - Return a string representing the time in ISO 8601 format, HH:MM:SS.mmmmmm or, if - :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a - 6-character string is appended, giving the UTC offset in (signed) hours and - minutes: HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM - - The optional argument *timespec* specifies the number of additional - components of the time to include (the default is ``'auto'``). - It can be one of the following: - - - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, - same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. - - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in HH:MM:SS format. - - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. HH:MM:SS.sss format. - - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. - - .. note:: - - Excluded time components are truncated, not rounded. - - :exc:`ValueError` will be raised on an invalid *timespec* argument. - - - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') - '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') - '12:34:56.000000' - >>> dt.isoformat(timespec='auto') - '12:34:56' - - .. versionadded:: 3.6 - Added the *timespec* argument. - - -.. method:: time.__str__() - - For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``. - - -.. method:: time.strftime(format) - - Return a string representing the time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: time.__format__(format) - - Same as :meth:`.time.strftime`. This makes it possible to specify a format string - for a :class:`.time` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: time.utcoffset() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.utcoffset(None)``, and raises an exception if the latter doesn't - return ``None`` or a :class:`timedelta` object representing a whole number of - minutes with magnitude less than one day. - - -.. method:: time.dst() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.dst(None)``, and raises an exception if the latter doesn't return - ``None``, or a :class:`timedelta` object representing a whole number of minutes - with magnitude less than one day. - - -.. method:: time.tzname() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't - return ``None`` or a string object. - - -Example: - - >>> from datetime import time, tzinfo, timedelta - >>> class GMT1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): - ... return "Europe/Prague" - ... - >>> t = time(12, 10, 30, tzinfo=GMT1()) - >>> t # doctest: +ELLIPSIS - datetime.time(12, 10, 30, tzinfo=) - >>> gmt = GMT1() - >>> t.isoformat() - '12:10:30+01:00' - >>> t.dst() - datetime.timedelta(0) - >>> t.tzname() - 'Europe/Prague' - >>> t.strftime("%H:%M:%S %Z") - '12:10:30 Europe/Prague' - >>> 'The {} is {:%H:%M}.'.format("time", t) - 'The time is 12:10.' - - -.. _datetime-tzinfo: - -:class:`tzinfo` Objects ------------------------ - -.. class:: tzinfo() - - This is an abstract base class, meaning that this class should not be - instantiated directly. You need to derive a concrete subclass, and (at least) - supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module supplies - a simple concrete subclass of :class:`tzinfo`, :class:`timezone`, which can represent - timezones with fixed offset from UTC such as UTC itself or North American EST and - EDT. - - An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the - constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object - supports methods revealing offset of local time from UTC, the name of the time - zone, and DST offset, all relative to a date or time object passed to them. - - Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, else it can be - pickled but possibly not unpickled again. This is a technical requirement that - may be relaxed in the future. - - A concrete subclass of :class:`tzinfo` may need to implement the following - methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. - - -.. method:: tzinfo.utcoffset(dt) - - Return offset of local time from UTC, in minutes east of UTC. If local time is - west of UTC, this should be negative. Note that this is intended to be the - total offset from UTC; for example, if a :class:`tzinfo` object represents both - time zone and DST adjustments, :meth:`utcoffset` should return their sum. If - the UTC offset isn't known, return ``None``. Else the value returned must be a - :class:`timedelta` object specifying a whole number of minutes in the range - -1439 to 1439 inclusive (1440 = 24\*60; the magnitude of the offset must be less - than one day). Most implementations of :meth:`utcoffset` will probably look - like one of these two:: - - return CONSTANT # fixed-offset class - return CONSTANT + self.dst(dt) # daylight-aware class - - If :meth:`utcoffset` does not return ``None``, :meth:`dst` should not return - ``None`` either. - - The default implementation of :meth:`utcoffset` raises - :exc:`NotImplementedError`. - - -.. method:: tzinfo.dst(dt) - - Return the daylight saving time (DST) adjustment, in minutes east of UTC, or - ``None`` if DST information isn't known. Return ``timedelta(0)`` if DST is not - in effect. If DST is in effect, return the offset as a :class:`timedelta` object - (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has - already been added to the UTC offset returned by :meth:`utcoffset`, so there's - no need to consult :meth:`dst` unless you're interested in obtaining DST info - separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag - should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for - DST changes when crossing time zones. - - An instance *tz* of a :class:`tzinfo` subclass that models both standard and - daylight times must be consistent in this sense: - - ``tz.utcoffset(dt) - tz.dst(dt)`` - - must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz`` For sane :class:`tzinfo` subclasses, this expression yields the time - zone's "standard offset", which should not depend on the date or the time, but - only on geographic location. The implementation of :meth:`datetime.astimezone` - relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee - this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. - - Most implementations of :meth:`dst` will probably look like one of these two:: - - def dst(self, dt): - # a fixed-offset class: doesn't account for DST - return timedelta(0) - - or :: - - def dst(self, dt): - # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed - # in standard local time. Then - - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) - else: - return timedelta(0) - - The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. - - -.. method:: tzinfo.tzname(dt) - - Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, - and there's no requirement that it mean anything in particular. For example, - "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all - valid replies. Return ``None`` if a string name isn't known. Note that this is - a method rather than a fixed string primarily because some :class:`tzinfo` - subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for - daylight time. - - The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. - - -These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the -argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. - -When ``None`` is passed, it's up to the class designer to decide the best -response. For example, returning ``None`` is appropriate if the class wishes to -say that time objects don't participate in the :class:`tzinfo` protocols. It -may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as -there is no other convention for discovering the standard offset. - -When a :class:`.datetime` object is passed in response to a :class:`.datetime` -method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local -time, and not need worry about objects in other timezones. - -There is one more :class:`tzinfo` method that a subclass may wish to override: - - -.. method:: tzinfo.fromutc(dt) - - This is called from the default :class:`datetime.astimezone()` - implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s - date and time data are to be viewed as expressing a UTC time. The purpose - of :meth:`fromutc` is to adjust the date and time data, returning an - equivalent datetime in *self*'s local time. - - Most :class:`tzinfo` subclasses should be able to inherit the default - :meth:`fromutc` implementation without problems. It's strong enough to handle - fixed-offset time zones, and time zones accounting for both standard and - daylight time, and the latter even if the DST transition times differ in - different years. An example of a time zone the default :meth:`fromutc` - implementation may not handle correctly in all cases is one where the standard - offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and - :meth:`fromutc` may not produce the result you want if the result is one of the - hours straddling the moment the standard offset changes. - - Skipping code for error cases, the default :meth:`fromutc` implementation acts - like:: - - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() - # raise ValueError if dtoff is None or dtdst is None - delta = dtoff - dtdst # this is self's standard offset - if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() - # raise ValueError if dtdst is None - if dtdst: - return dt + dtdst - else: - return dt - -Example :class:`tzinfo` classes: - -.. literalinclude:: ../includes/tzinfo-examples.py - -Note that there are unavoidable subtleties twice per year in a :class:`tzinfo` -subclass accounting for both standard and daylight time, at the DST transition -points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the -minute after 1:59 (EST) on the second Sunday in March, and ends the minute after -1:59 (EDT) on the first Sunday in November:: - - UTC 3:MM 4:MM 5:MM 6:MM 7:MM 8:MM - EST 22:MM 23:MM 0:MM 1:MM 2:MM 3:MM - EDT 23:MM 0:MM 1:MM 2:MM 3:MM 4:MM - - start 22:MM 23:MM 0:MM 1:MM 3:MM 4:MM - - end 23:MM 0:MM 1:MM 1:MM 2:MM 3:MM - -When DST starts (the "start" line), the local wall clock leaps from 1:59 to -3:00. A wall time of the form 2:MM doesn't really make sense on that day, so -``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST -begins. For example, at the Spring forward transition of 2016, we get - - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) - >>> for i in range(4): - ... u = u0 + i*HOUR - ... t = u.astimezone(Eastern) - ... print(u.time(), 'UTC =', t.time(), t.tzname()) - ... - 05:00:00 UTC = 00:00:00 EST - 06:00:00 UTC = 01:00:00 EST - 07:00:00 UTC = 03:00:00 EDT - 08:00:00 UTC = 04:00:00 EDT - - -When DST ends (the "end" line), there's a potentially worse problem: there's an -hour that can't be spelled unambiguously in local wall time: the last hour of -daylight time. In Eastern, that's times of the form 5:MM UTC on the day -daylight time ends. The local wall clock leaps from 1:59 (daylight time) back -to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC -hours into the same local hour then. In the Eastern example, UTC times of the -form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. -For example, at the Fall back transition of 2016, we get - - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) - >>> for i in range(4): - ... u = u0 + i*HOUR - ... t = u.astimezone(Eastern) - ... print(u.time(), 'UTC =', t.time(), t.tzname(), t.fold) - ... - 04:00:00 UTC = 00:00:00 EDT 0 - 05:00:00 UTC = 01:00:00 EDT 0 - 06:00:00 UTC = 01:00:00 EST 1 - 07:00:00 UTC = 02:00:00 EST 0 - -Note that the :class:`datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. - -Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid -:class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing -only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). - -.. seealso:: - - `dateutil.tz `_ - The standard library has :class:`timezone` class for handling arbitrary - fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. - - *dateutil.tz* library brings the *IANA timezone database* (also known as the - Olson database) to Python and its usage is recommended. - - `IANA timezone database `_ - The Time Zone Database (often called tz, tzdata or zoneinfo) contains code and - data that represent the history of local time for many representative - locations around the globe. It is updated periodically to reflect changes - made by political bodies to time zone boundaries, UTC offsets, and - daylight-saving rules. - - -.. _datetime-timezone: - -:class:`timezone` Objects --------------------------- - -The :class:`timezone` class is a subclass of :class:`tzinfo`, each -instance of which represents a timezone defined by a fixed offset from -UTC. Note that objects of this class cannot be used to represent -timezone information in the locations where different offsets are used -in different days of the year or where historical changes have been -made to civil time. - - -.. class:: timezone(offset, name=None) - - The *offset* argument must be specified as a :class:`timedelta` - object representing the difference between the local time and UTC. It must - be strictly between ``-timedelta(hours=24)`` and - ``timedelta(hours=24)`` and represent a whole number of minutes, - otherwise :exc:`ValueError` is raised. - - The *name* argument is optional. If specified it must be a string that - will be used as the value returned by the :meth:`datetime.tzname` method. - - .. versionadded:: 3.2 - -.. method:: timezone.utcoffset(dt) - - Return the fixed value specified when the :class:`timezone` instance is - constructed. The *dt* argument is ignored. The return value is a - :class:`timedelta` instance equal to the difference between the - local time and UTC. - -.. method:: timezone.tzname(dt) - - Return the fixed value specified when the :class:`timezone` instance - is constructed. If *name* is not provided in the constructor, the - name returned by ``tzname(dt)`` is generated from the value of the - ``offset`` as follows. If *offset* is ``timedelta(0)``, the name - is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign - of ``offset``, HH and MM are two digits of ``offset.hours`` and - ``offset.minutes`` respectively. - - .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. - - -.. method:: timezone.dst(dt) - - Always returns ``None``. - -.. method:: timezone.fromutc(dt) - - Return ``dt + offset``. The *dt* argument must be an aware - :class:`.datetime` instance, with ``tzinfo`` set to ``self``. - -Class attributes: - -.. attribute:: timezone.utc - - The UTC timezone, ``timezone(timedelta(0))``. - - -.. index:: - single: % (percent); datetime format - -.. _strftime-strptime-behavior: - -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- - -:class:`date`, :class:`.datetime`, and :class:`.time` objects all support a -``strftime(format)`` method, to create a string representing the time under the -control of an explicit format string. Broadly speaking, ``d.strftime(fmt)`` -acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` -although not all objects support a :meth:`timetuple` method. - -Conversely, the :meth:`datetime.strptime` class method creates a -:class:`.datetime` object from a string representing a date and time and a -corresponding format string. ``datetime.strptime(date_string, format)`` is -equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``, except -when the format includes sub-second components or timezone offset information, -which are supported in ``datetime.strptime`` but are discarded by ``time.strptime``. - -For :class:`.time` objects, the format codes for year, month, and day should not -be used, as time objects have no such values. If they're used anyway, ``1900`` -is substituted for the year, and ``1`` for the month and day. - -For :class:`date` objects, the format codes for hours, minutes, seconds, and -microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. - -The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform -variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. - -The following is a list of all the format codes that the C standard (1989 -version) requires, and these work on all platforms with a standard C -implementation. Note that the 1999 version of the C standard added additional -format codes. - -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | -| | abbreviated name. | (en_US); | | -| | || So, Mo, ..., Sa | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | -| | | Saturday (en_US); | | -| | || Sonntag, Montag, ..., | | -| | | Samstag (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | -| | name. | (en_US); | | -| | || Jan, Feb, ..., Dez | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | -| | | ..., December (en_US);| | -| | || Januar, Februar, ..., | | -| | | Dezember (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | -+-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE) | \(3) | -+-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded on the | 999999 | | -| | left. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(6) | -| | or -HHMM (empty string if the | +1030 | | -| | object is naive). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | -| | if the object is naive). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | -| | the week) as a zero padded | | | -| | decimal number. All days in a | | | -| | new year preceding the first | | | -| | Sunday are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | -| | the week) as a decimal number. | | | -| | All days in a new year | | | -| | preceding the first Monday | | | -| | are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | -| | representation. || 08/16/1988 (en_US); | | -| | || 16.08.1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | -| | representation. || 21:30:00 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | -+-----------+--------------------------------+------------------------+-------+ - -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. These -may not be available on all platforms when used with the :meth:`strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | -+-----------+--------------------------------+------------------------+-------+ - -.. versionadded:: 3.6 - ``%G``, ``%u`` and ``%V`` were added. - -Notes: - -(1) - Because the format depends on the current locale, care should be taken when - making assumptions about the output value. Field orderings will vary (for - example, "month/day/year" versus "day/month/year"), and the output may - contain Unicode characters encoded using the locale's default encoding (for - example, if the current locale is ``ja_JP``, the default encoding could be - any one of ``eucJP``, ``SJIS``, or ``utf-8``; use :meth:`locale.getlocale` - to determine the current locale's encoding). - -(2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but - years < 1000 must be zero-filled to 4-digit width. - - .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to - years >= 1900. - - .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to - years >= 1000. - -(3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects - the output hour field if the ``%I`` directive is used to parse the hour. - -(4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support - leap seconds. - -(5) - When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is - an extension to the set of format characters in the C standard (but - implemented separately in datetime objects, and therefore always - available). - -(6) - For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty - strings. - - For an aware object: - - ``%z`` - :meth:`utcoffset` is transformed into a 5-character string of the form - +HHMM or -HHMM, where HH is a 2-digit string giving the number of UTC - offset hours, and MM is a 2-digit string giving the number of UTC offset - minutes. For example, if :meth:`utcoffset` returns - ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string - ``'-0330'``. - - ``%Z`` - If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty - string. Otherwise ``%Z`` is replaced by the returned value, which must - be a string. - - .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an - aware :class:`.datetime` object will be produced. The ``tzinfo`` of the - result will be set to a :class:`timezone` instance. - -(7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used - in calculations when the day of the week and the calendar year (``%Y``) - are specified. - -(8) - Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the - day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not - interchangeable. - -.. rubric:: Footnotes - -.. [#] If, that is, we ignore the effects of Relativity diff --git a/third_party/python/Doc/library/dbm.rst b/third_party/python/Doc/library/dbm.rst deleted file mode 100644 index 32e80b2cf..000000000 --- a/third_party/python/Doc/library/dbm.rst +++ /dev/null @@ -1,370 +0,0 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= - -.. module:: dbm - :synopsis: Interfaces to various Unix "database" formats. - -**Source code:** :source:`Lib/dbm/__init__.py` - --------------- - -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the -slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There -is a `third party interface `_ to -the Oracle Berkeley DB. - - -.. exception:: error - - A tuple containing the exceptions that can be raised by each of the supported - modules, with a unique exception also named :exc:`dbm.error` as the first - item --- the latter is used when :exc:`dbm.error` is raised. - - -.. function:: whichdb(filename) - - This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. - - Returns one of the following values: ``None`` if the file can't be opened - because it's unreadable or doesn't exist; the empty string (``''``) if the - file's format can't be guessed; or a string containing the required module - name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. - - -.. function:: open(file, flag='r', mode=0o666) - - Open the database file *file* and return a corresponding object. - - If the database file already exists, the :func:`whichdb` function is used to - determine its type and the appropriate module is used; if it does not exist, - the first module listed above that can be imported is used. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). - - -The object returned by :func:`.open` supports the same basic functionality as -dictionaries; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`keys` method are -available, as well as :meth:`get` and :meth:`setdefault`. - -.. versionchanged:: 3.2 - :meth:`get` and :meth:`setdefault` are now available in all database modules. - -Key and values are always stored as bytes. This means that when -strings are used they are implicitly converted to the default encoding before -being stored. - -These objects also support being used in a :keyword:`with` statement, which -will automatically close them when done. - -.. versionchanged:: 3.4 - Added native support for the context management protocol to the objects - returned by :func:`.open`. - -The following example records some hostnames and a corresponding title, and -then prints out the contents of the database:: - - import dbm - - # Open database, creating it if necessary. - with dbm.open('cache', 'c') as db: - - # Record some values - db[b'hello'] = b'there' - db['www.python.org'] = 'Python Website' - db['www.cnn.com'] = 'Cable News Network' - - # Note that the keys are considered bytes now. - assert db[b'www.python.org'] == b'Python Website' - # Notice how the value is now in bytes. - assert db['www.cnn.com'] == b'Cable News Network' - - # Often-used methods of the dict interface work too. - print(db.get('python.org', b'not present')) - - # Storing a non-string key or value will raise an exception (most - # likely a TypeError). - db['www.yahoo.com'] = 4 - - # db is automatically closed when leaving the with statement. - - -.. seealso:: - - Module :mod:`shelve` - Persistence module which stores non-string data. - - -The individual submodules are described in the following sections. - - -:mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------- - -.. module:: dbm.gnu - :platform: Unix - :synopsis: GNU's reinterpretation of dbm. - -**Source code:** :source:`Lib/dbm/gnu.py` - --------------- - -This module is quite similar to the :mod:`dbm` module, but uses the GNU library -``gdbm`` instead to provide some additional functionality. Please note that the -file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible. - -The :mod:`dbm.gnu` module provides an interface to the GNU DBM library. -``dbm.gnu.gdbm`` objects behave like mappings (dictionaries), except that keys and -values are always converted to bytes before storing. Printing a ``gdbm`` -object doesn't print the -keys and values, and the :meth:`items` and :meth:`values` methods are not -supported. - -.. exception:: error - - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is - raised for general mapping errors like specifying an incorrect key. - - -.. function:: open(filename[, flag[, mode]]) - - Open a ``gdbm`` database and return a :class:`gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The following additional characters may be appended to the flag to control - how the database is opened: - - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ - - Not all flags are valid for all versions of ``gdbm``. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. - - In addition to the dictionary-like methods, ``gdbm`` objects have the - following methods: - - .. method:: gdbm.firstkey() - - It's possible to loop over every key in the database using this method and the - :meth:`nextkey` method. The traversal is ordered by ``gdbm``'s internal - hash values, and won't be sorted by the key values. This method returns - the starting key. - - .. method:: gdbm.nextkey(key) - - Returns the key that follows *key* in the traversal. The following code prints - every key in the database ``db``, without having to create a list in memory that - contains them all:: - - k = db.firstkey() - while k != None: - print(k) - k = db.nextkey(k) - - .. method:: gdbm.reorganize() - - If you have carried out a lot of deletions and would like to shrink the space - used by the ``gdbm`` file, this routine will reorganize the database. ``gdbm`` - objects will not shorten the length of a database file except by using this - reorganization; otherwise, deleted file space will be kept and reused as new - (key, value) pairs are added. - - .. method:: gdbm.sync() - - When the database has been opened in fast mode, this method forces any - unwritten data to be written to the disk. - - .. method:: gdbm.close() - - Close the ``gdbm`` database. - -:mod:`dbm.ndbm` --- Interface based on ndbm -------------------------------------------- - -.. module:: dbm.ndbm - :platform: Unix - :synopsis: The standard "database" interface, based on ndbm. - -**Source code:** :source:`Lib/dbm/ndbm.py` - --------------- - -The :mod:`dbm.ndbm` module provides an interface to the Unix "(n)dbm" library. -Dbm objects behave like mappings (dictionaries), except that keys and values are -always stored as bytes. Printing a ``dbm`` object doesn't print the keys and -values, and the :meth:`items` and :meth:`values` methods are not supported. - -This module can be used with the "classic" ndbm interface or the GNU GDBM -compatibility interface. On Unix, the :program:`configure` script will attempt -to locate the appropriate header file to simplify building this module. - -.. exception:: error - - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised - for general mapping errors like specifying an incorrect key. - - -.. data:: library - - Name of the ``ndbm`` implementation library used. - - -.. function:: open(filename[, flag[, mode]]) - - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the - name of the database file (without the :file:`.dir` or :file:`.pag` extensions). - - The optional *flag* argument must be one of these values: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). - - In addition to the dictionary-like methods, ``ndbm`` objects - provide the following method: - - .. method:: ndbm.close() - - Close the ``ndbm`` database. - - -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ - -.. module:: dbm.dumb - :synopsis: Portable implementation of the simple DBM interface. - -**Source code:** :source:`Lib/dbm/dumb.py` - -.. index:: single: databases - -.. note:: - - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` - module is not written for speed and is not nearly as heavily used as the other - database modules. - --------------- - -The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which -is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no -external library is required. As with other persistent mappings, the keys and -values are always stored as bytes. - -The module defines the following: - - -.. exception:: error - - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is - raised for general mapping errors like specifying an incorrect key. - - -.. function:: open(filename[, flag[, mode]]) - - Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is - the basename of the database file (without any specific extensions). When a - dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions - are created. - - The optional *flag* argument supports only the semantics of ``'c'`` - and ``'n'`` values. Other values will default to database being always - opened for update, and will be created if it does not exist. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be modified - by the prevailing umask). - - .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. - - .. deprecated-removed:: 3.6 3.8 - Creating database in ``'r'`` and ``'w'`` modes. Modifying database in - ``'r'`` mode. - - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: - - .. method:: dumbdbm.sync() - - Synchronize the on-disk directory and data files. This method is called - by the :meth:`Shelve.sync` method. - - .. method:: dumbdbm.close() - - Close the ``dumbdbm`` database. - diff --git a/third_party/python/Doc/library/debug.rst b/third_party/python/Doc/library/debug.rst deleted file mode 100644 index 88a2fa62a..000000000 --- a/third_party/python/Doc/library/debug.rst +++ /dev/null @@ -1,18 +0,0 @@ -*********************** -Debugging and Profiling -*********************** - -These libraries help you with Python development: the debugger enables you to -step through code, analyze stack frames and set breakpoints etc., and the -profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. - -.. toctree:: - - bdb.rst - faulthandler.rst - pdb.rst - profile.rst - timeit.rst - trace.rst - tracemalloc.rst diff --git a/third_party/python/Doc/library/decimal.rst b/third_party/python/Doc/library/decimal.rst deleted file mode 100644 index e984edcb7..000000000 --- a/third_party/python/Doc/library/decimal.rst +++ /dev/null @@ -1,2112 +0,0 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== - -.. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. - -.. moduleauthor:: Eric Price -.. moduleauthor:: Facundo Batista -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Aahz -.. moduleauthor:: Tim Peters -.. moduleauthor:: Stefan Krah -.. sectionauthor:: Raymond D. Hettinger - -**Source code:** :source:`Lib/decimal.py` - -.. import modules for testing inline doctests with the Sphinx doctest builder -.. testsetup:: * - - import decimal - import math - from decimal import * - # make sure each group gets a fresh context - setcontext(Context()) - --------------- - -The :mod:`decimal` module provides support for fast correctly-rounded -decimal floating point arithmetic. It offers several advantages over the -:class:`float` datatype: - -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. - -* Decimal numbers can be represented exactly. In contrast, numbers like - :const:`1.1` and :const:`2.2` do not have exact representations in binary - floating point. End users typically would not expect ``1.1 + 2.2`` to display - as :const:`3.3000000000000003` as it does with binary floating point. - -* The exactness carries over into arithmetic. In decimal floating point, ``0.1 - + 0.1 + 0.1 - 0.3`` is exactly equal to zero. In binary floating point, the result - is :const:`5.5511151231257827e-017`. While near to zero, the differences - prevent reliable equality testing and differences can accumulate. For this - reason, decimal is preferred in accounting applications which have strict - equality invariants. - -* The decimal module incorporates a notion of significant places so that ``1.30 - + 1.20`` is :const:`2.50`. The trailing zero is kept to indicate significance. - This is the customary presentation for monetary applications. For - multiplication, the "schoolbook" approach uses all the figures in the - multiplicands. For instance, ``1.3 * 1.2`` gives :const:`1.56` while ``1.30 * - 1.20`` gives :const:`1.5600`. - -* Unlike hardware based binary floating point, the decimal module has a user - alterable precision (defaulting to 28 places) which can be as large as needed for - a given problem: - - >>> from decimal import * - >>> getcontext().prec = 6 - >>> Decimal(1) / Decimal(7) - Decimal('0.142857') - >>> getcontext().prec = 28 - >>> Decimal(1) / Decimal(7) - Decimal('0.1428571428571428571428571429') - -* Both binary and decimal floating point are implemented in terms of published - standards. While the built-in float type exposes only a modest portion of its - capabilities, the decimal module exposes all required parts of the standard. - When needed, the programmer has full control over rounding and signal handling. - This includes an option to enforce exact arithmetic by using exceptions - to block any inexact operations. - -* The decimal module was designed to support "without prejudice, both exact - unrounded decimal arithmetic (sometimes called fixed-point arithmetic) - and rounded floating-point arithmetic." -- excerpt from the decimal - arithmetic specification. - -The module design is centered around three concepts: the decimal number, the -context for arithmetic, and signals. - -A decimal number is immutable. It has a sign, coefficient digits, and an -exponent. To preserve significance, the coefficient digits do not truncate -trailing zeros. Decimals also include special values such as -:const:`Infinity`, :const:`-Infinity`, and :const:`NaN`. The standard also -differentiates :const:`-0` from :const:`+0`. - -The context for arithmetic is an environment specifying precision, rounding -rules, limits on exponents, flags indicating the results of operations, and trap -enablers which determine whether signals are treated as exceptions. Rounding -options include :const:`ROUND_CEILING`, :const:`ROUND_DOWN`, -:const:`ROUND_FLOOR`, :const:`ROUND_HALF_DOWN`, :const:`ROUND_HALF_EVEN`, -:const:`ROUND_HALF_UP`, :const:`ROUND_UP`, and :const:`ROUND_05UP`. - -Signals are groups of exceptional conditions arising during the course of -computation. Depending on the needs of the application, signals may be ignored, -considered as informational, or treated as exceptions. The signals in the -decimal module are: :const:`Clamped`, :const:`InvalidOperation`, -:const:`DivisionByZero`, :const:`Inexact`, :const:`Rounded`, :const:`Subnormal`, -:const:`Overflow`, :const:`Underflow` and :const:`FloatOperation`. - -For each signal there is a flag and a trap enabler. When a signal is -encountered, its flag is set to one, then, if the trap enabler is -set to one, an exception is raised. Flags are sticky, so the user needs to -reset them before monitoring a calculation. - - -.. seealso:: - - * IBM's General Decimal Arithmetic Specification, `The General Decimal Arithmetic - Specification `_. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-tutorial: - -Quick-start Tutorial --------------------- - -The usual start to using decimals is importing the module, viewing the current -context with :func:`getcontext` and, if necessary, setting new values for -precision, rounding, or enabled traps:: - - >>> from decimal import * - >>> getcontext() - Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[], traps=[Overflow, DivisionByZero, - InvalidOperation]) - - >>> getcontext().prec = 7 # Set a new precision - -Decimal instances can be constructed from integers, strings, floats, or tuples. -Construction from an integer or a float performs an exact conversion of the -value of that integer or float. Decimal numbers include special values such as -:const:`NaN` which stands for "Not a number", positive and negative -:const:`Infinity`, and :const:`-0`:: - - >>> getcontext().prec = 28 - >>> Decimal(10) - Decimal('10') - >>> Decimal('3.14') - Decimal('3.14') - >>> Decimal(3.14) - Decimal('3.140000000000000124344978758017532527446746826171875') - >>> Decimal((0, (3, 1, 4), -2)) - Decimal('3.14') - >>> Decimal(str(2.0 ** 0.5)) - Decimal('1.4142135623730951') - >>> Decimal(2) ** Decimal('0.5') - Decimal('1.414213562373095048801688724') - >>> Decimal('NaN') - Decimal('NaN') - >>> Decimal('-Infinity') - Decimal('-Infinity') - -If the :exc:`FloatOperation` signal is trapped, accidental mixing of -decimals and floats in constructors or ordering comparisons raises -an exception:: - - >>> c = getcontext() - >>> c.traps[FloatOperation] = True - >>> Decimal(3.14) - Traceback (most recent call last): - File "", line 1, in - decimal.FloatOperation: [] - >>> Decimal('3.5') < 3.7 - Traceback (most recent call last): - File "", line 1, in - decimal.FloatOperation: [] - >>> Decimal('3.5') == 3.5 - True - -.. versionadded:: 3.3 - -The significance of a new Decimal is determined solely by the number of digits -input. Context precision and rounding only come into play during arithmetic -operations. - -.. doctest:: newcontext - - >>> getcontext().prec = 6 - >>> Decimal('3.0') - Decimal('3.0') - >>> Decimal('3.1415926535') - Decimal('3.1415926535') - >>> Decimal('3.1415926535') + Decimal('2.7182818285') - Decimal('5.85987') - >>> getcontext().rounding = ROUND_UP - >>> Decimal('3.1415926535') + Decimal('2.7182818285') - Decimal('5.85988') - -If the internal limits of the C version are exceeded, constructing -a decimal raises :class:`InvalidOperation`:: - - >>> Decimal("1e9999999999999999999") - Traceback (most recent call last): - File "", line 1, in - decimal.InvalidOperation: [] - -.. versionchanged:: 3.3 - -Decimals interact well with much of the rest of Python. Here is a small decimal -floating point flying circus: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> data = list(map(Decimal, '1.34 1.87 3.45 2.35 1.00 0.03 9.25'.split())) - >>> max(data) - Decimal('9.25') - >>> min(data) - Decimal('0.03') - >>> sorted(data) - [Decimal('0.03'), Decimal('1.00'), Decimal('1.34'), Decimal('1.87'), - Decimal('2.35'), Decimal('3.45'), Decimal('9.25')] - >>> sum(data) - Decimal('19.29') - >>> a,b,c = data[:3] - >>> str(a) - '1.34' - >>> float(a) - 1.34 - >>> round(a, 1) - Decimal('1.3') - >>> int(a) - 1 - >>> a * 5 - Decimal('6.70') - >>> a * b - Decimal('2.5058') - >>> c % a - Decimal('0.77') - -And some mathematical functions are also available to Decimal: - - >>> getcontext().prec = 28 - >>> Decimal(2).sqrt() - Decimal('1.414213562373095048801688724') - >>> Decimal(1).exp() - Decimal('2.718281828459045235360287471') - >>> Decimal('10').ln() - Decimal('2.302585092994045684017991455') - >>> Decimal('10').log10() - Decimal('1') - -The :meth:`quantize` method rounds a number to a fixed exponent. This method is -useful for monetary applications that often round results to a fixed number of -places: - - >>> Decimal('7.325').quantize(Decimal('.01'), rounding=ROUND_DOWN) - Decimal('7.32') - >>> Decimal('7.325').quantize(Decimal('1.'), rounding=ROUND_UP) - Decimal('8') - -As shown above, the :func:`getcontext` function accesses the current context and -allows the settings to be changed. This approach meets the needs of most -applications. - -For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` -function. - -In accordance with the standard, the :mod:`decimal` module provides two ready to -use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The -former is especially useful for debugging because many of the traps are -enabled: - -.. doctest:: newcontext - :options: +NORMALIZE_WHITESPACE - - >>> myothercontext = Context(prec=60, rounding=ROUND_HALF_DOWN) - >>> setcontext(myothercontext) - >>> Decimal(1) / Decimal(7) - Decimal('0.142857142857142857142857142857142857142857142857142857142857') - - >>> ExtendedContext - Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[], traps=[]) - >>> setcontext(ExtendedContext) - >>> Decimal(1) / Decimal(7) - Decimal('0.142857143') - >>> Decimal(42) / Decimal(0) - Decimal('Infinity') - - >>> setcontext(BasicContext) - >>> Decimal(42) / Decimal(0) - Traceback (most recent call last): - File "", line 1, in -toplevel- - Decimal(42) / Decimal(0) - DivisionByZero: x / 0 - -Contexts also have signal flags for monitoring exceptional conditions -encountered during computations. The flags remain set until explicitly cleared, -so it is best to clear the flags before each set of monitored computations by -using the :meth:`clear_flags` method. :: - - >>> setcontext(ExtendedContext) - >>> getcontext().clear_flags() - >>> Decimal(355) / Decimal(113) - Decimal('3.14159292') - >>> getcontext() - Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[Inexact, Rounded], traps=[]) - -The *flags* entry shows that the rational approximation to :const:`Pi` was -rounded (digits beyond the context precision were thrown away) and that the -result is inexact (some of the discarded digits were non-zero). - -Individual traps are set using the dictionary in the :attr:`traps` field of a -context: - -.. doctest:: newcontext - - >>> setcontext(ExtendedContext) - >>> Decimal(1) / Decimal(0) - Decimal('Infinity') - >>> getcontext().traps[DivisionByZero] = 1 - >>> Decimal(1) / Decimal(0) - Traceback (most recent call last): - File "", line 1, in -toplevel- - Decimal(1) / Decimal(0) - DivisionByZero: x / 0 - -Most programs adjust the current context only once, at the beginning of the -program. And, in many applications, data is converted to :class:`Decimal` with -a single cast inside a loop. With context set and decimals created, the bulk of -the program manipulates the data no differently than with other Python numeric -types. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-decimal: - -Decimal objects ---------------- - - -.. class:: Decimal(value="0", context=None) - - Construct a new :class:`Decimal` object based from *value*. - - *value* can be an integer, string, tuple, :class:`float`, or another :class:`Decimal` - object. If no *value* is given, returns ``Decimal('0')``. If *value* is a - string, it should conform to the decimal numeric string syntax after leading - and trailing whitespace characters, as well as underscores throughout, are removed:: - - sign ::= '+' | '-' - digit ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' - indicator ::= 'e' | 'E' - digits ::= digit [digit]... - decimal-part ::= digits '.' [digits] | ['.'] digits - exponent-part ::= indicator [sign] digits - infinity ::= 'Infinity' | 'Inf' - nan ::= 'NaN' [digits] | 'sNaN' [digits] - numeric-value ::= decimal-part [exponent-part] | infinity - numeric-string ::= [sign] numeric-value | [sign] nan - - Other Unicode decimal digits are also permitted where ``digit`` - appears above. These include decimal digits from various other - alphabets (for example, Arabic-Indic and Devanāgarī digits) along - with the fullwidth digits ``'\uff10'`` through ``'\uff19'``. - - If *value* is a :class:`tuple`, it should have three components, a sign - (:const:`0` for positive or :const:`1` for negative), a :class:`tuple` of - digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` - returns ``Decimal('1.414')``. - - If *value* is a :class:`float`, the binary floating point value is losslessly - converted to its exact decimal equivalent. This conversion can often require - 53 or more digits of precision. For example, ``Decimal(float('1.1'))`` - converts to - ``Decimal('1.100000000000000088817841970012523233890533447265625')``. - - The *context* precision does not affect how many digits are stored. That is - determined exclusively by the number of digits in *value*. For example, - ``Decimal('3.00000')`` records all five zeros even if the context precision is - only three. - - The purpose of the *context* argument is determining what to do if *value* is a - malformed string. If the context traps :const:`InvalidOperation`, an exception - is raised; otherwise, the constructor returns a new Decimal with the value of - :const:`NaN`. - - Once constructed, :class:`Decimal` objects are immutable. - - .. versionchanged:: 3.2 - The argument to the constructor is now permitted to be a :class:`float` - instance. - - .. versionchanged:: 3.3 - :class:`float` arguments raise an exception if the :exc:`FloatOperation` - trap is set. By default the trap is off. - - .. versionchanged:: 3.6 - Underscores are allowed for grouping, as with integral and floating-point - literals in code. - - Decimal floating point objects share many properties with the other built-in - numeric types such as :class:`float` and :class:`int`. All of the usual math - operations and special methods apply. Likewise, decimal objects can be - copied, pickled, printed, used as dictionary keys, used as set elements, - compared, sorted, and coerced to another type (such as :class:`float` or - :class:`int`). - - There are some small differences between arithmetic on Decimal objects and - arithmetic on integers and floats. When the remainder operator ``%`` is - applied to Decimal objects, the sign of the result is the sign of the - *dividend* rather than the sign of the divisor:: - - >>> (-7) % 4 - 1 - >>> Decimal(-7) % Decimal(4) - Decimal('-3') - - The integer division operator ``//`` behaves analogously, returning the - integer part of the true quotient (truncating towards zero) rather than its - floor, so as to preserve the usual identity ``x == (x // y) * y + x % y``:: - - >>> -7 // 4 - -2 - >>> Decimal(-7) // Decimal(4) - Decimal('-1') - - The ``%`` and ``//`` operators implement the ``remainder`` and - ``divide-integer`` operations (respectively) as described in the - specification. - - Decimal objects cannot generally be combined with floats or - instances of :class:`fractions.Fraction` in arithmetic operations: - an attempt to add a :class:`Decimal` to a :class:`float`, for - example, will raise a :exc:`TypeError`. However, it is possible to - use Python's comparison operators to compare a :class:`Decimal` - instance ``x`` with another number ``y``. This avoids confusing results - when doing equality comparisons between numbers of different types. - - .. versionchanged:: 3.2 - Mixed-type comparisons between :class:`Decimal` instances and other - numeric types are now fully supported. - - In addition to the standard numeric properties, decimal floating point - objects also have a number of specialized methods: - - - .. method:: adjusted() - - Return the adjusted exponent after shifting out the coefficient's - rightmost digits until only the lead digit remains: - ``Decimal('321e+5').adjusted()`` returns seven. Used for determining the - position of the most significant digit with respect to the decimal point. - - .. method:: as_integer_ratio() - - Return a pair ``(n, d)`` of integers that represent the given - :class:`Decimal` instance as a fraction, in lowest terms and - with a positive denominator:: - - >>> Decimal('-3.14').as_integer_ratio() - (-157, 50) - - The conversion is exact. Raise OverflowError on infinities and ValueError - on NaNs. - - .. versionadded:: 3.6 - - .. method:: as_tuple() - - Return a :term:`named tuple` representation of the number: - ``DecimalTuple(sign, digits, exponent)``. - - - .. method:: canonical() - - Return the canonical encoding of the argument. Currently, the encoding of - a :class:`Decimal` instance is always canonical, so this operation returns - its argument unchanged. - - .. method:: compare(other, context=None) - - Compare the values of two Decimal instances. :meth:`compare` returns a - Decimal instance, and if either operand is a NaN then the result is a - NaN:: - - a or b is a NaN ==> Decimal('NaN') - a < b ==> Decimal('-1') - a == b ==> Decimal('0') - a > b ==> Decimal('1') - - .. method:: compare_signal(other, context=None) - - This operation is identical to the :meth:`compare` method, except that all - NaNs signal. That is, if neither operand is a signaling NaN then any - quiet NaN operand is treated as though it were a signaling NaN. - - .. method:: compare_total(other, context=None) - - Compare two operands using their abstract representation rather than their - numerical value. Similar to the :meth:`compare` method, but the result - gives a total ordering on :class:`Decimal` instances. Two - :class:`Decimal` instances with the same numeric value but different - representations compare unequal in this ordering: - - >>> Decimal('12.0').compare_total(Decimal('12')) - Decimal('-1') - - Quiet and signaling NaNs are also included in the total ordering. The - result of this function is ``Decimal('0')`` if both operands have the same - representation, ``Decimal('-1')`` if the first operand is lower in the - total order than the second, and ``Decimal('1')`` if the first operand is - higher in the total order than the second operand. See the specification - for details of the total order. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: compare_total_mag(other, context=None) - - Compare two operands using their abstract representation rather than their - value as in :meth:`compare_total`, but ignoring the sign of each operand. - ``x.compare_total_mag(y)`` is equivalent to - ``x.copy_abs().compare_total(y.copy_abs())``. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: conjugate() - - Just returns self, this method is only to comply with the Decimal - Specification. - - .. method:: copy_abs() - - Return the absolute value of the argument. This operation is unaffected - by the context and is quiet: no flags are changed and no rounding is - performed. - - .. method:: copy_negate() - - Return the negation of the argument. This operation is unaffected by the - context and is quiet: no flags are changed and no rounding is performed. - - .. method:: copy_sign(other, context=None) - - Return a copy of the first operand with the sign set to be the same as the - sign of the second operand. For example: - - >>> Decimal('2.3').copy_sign(Decimal('-1.5')) - Decimal('-2.3') - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: exp(context=None) - - Return the value of the (natural) exponential function ``e**x`` at the - given number. The result is correctly rounded using the - :const:`ROUND_HALF_EVEN` rounding mode. - - >>> Decimal(1).exp() - Decimal('2.718281828459045235360287471') - >>> Decimal(321).exp() - Decimal('2.561702493119680037517373933E+139') - - .. method:: from_float(f) - - Classmethod that converts a float to a decimal number, exactly. - - Note `Decimal.from_float(0.1)` is not the same as `Decimal('0.1')`. - Since 0.1 is not exactly representable in binary floating point, the - value is stored as the nearest representable value which is - `0x1.999999999999ap-4`. That equivalent value in decimal is - `0.1000000000000000055511151231257827021181583404541015625`. - - .. note:: From Python 3.2 onwards, a :class:`Decimal` instance - can also be constructed directly from a :class:`float`. - - .. doctest:: - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - >>> Decimal.from_float(float('nan')) - Decimal('NaN') - >>> Decimal.from_float(float('inf')) - Decimal('Infinity') - >>> Decimal.from_float(float('-inf')) - Decimal('-Infinity') - - .. versionadded:: 3.1 - - .. method:: fma(other, third, context=None) - - Fused multiply-add. Return self*other+third with no rounding of the - intermediate product self*other. - - >>> Decimal(2).fma(3, 5) - Decimal('11') - - .. method:: is_canonical() - - Return :const:`True` if the argument is canonical and :const:`False` - otherwise. Currently, a :class:`Decimal` instance is always canonical, so - this operation always returns :const:`True`. - - .. method:: is_finite() - - Return :const:`True` if the argument is a finite number, and - :const:`False` if the argument is an infinity or a NaN. - - .. method:: is_infinite() - - Return :const:`True` if the argument is either positive or negative - infinity and :const:`False` otherwise. - - .. method:: is_nan() - - Return :const:`True` if the argument is a (quiet or signaling) NaN and - :const:`False` otherwise. - - .. method:: is_normal(context=None) - - Return :const:`True` if the argument is a *normal* finite number. Return - :const:`False` if the argument is zero, subnormal, infinite or a NaN. - - .. method:: is_qnan() - - Return :const:`True` if the argument is a quiet NaN, and - :const:`False` otherwise. - - .. method:: is_signed() - - Return :const:`True` if the argument has a negative sign and - :const:`False` otherwise. Note that zeros and NaNs can both carry signs. - - .. method:: is_snan() - - Return :const:`True` if the argument is a signaling NaN and :const:`False` - otherwise. - - .. method:: is_subnormal(context=None) - - Return :const:`True` if the argument is subnormal, and :const:`False` - otherwise. - - .. method:: is_zero() - - Return :const:`True` if the argument is a (positive or negative) zero and - :const:`False` otherwise. - - .. method:: ln(context=None) - - Return the natural (base e) logarithm of the operand. The result is - correctly rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - - .. method:: log10(context=None) - - Return the base ten logarithm of the operand. The result is correctly - rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - - .. method:: logb(context=None) - - For a nonzero number, return the adjusted exponent of its operand as a - :class:`Decimal` instance. If the operand is a zero then - ``Decimal('-Infinity')`` is returned and the :const:`DivisionByZero` flag - is raised. If the operand is an infinity then ``Decimal('Infinity')`` is - returned. - - .. method:: logical_and(other, context=None) - - :meth:`logical_and` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise ``and`` of the two operands. - - .. method:: logical_invert(context=None) - - :meth:`logical_invert` is a logical operation. The - result is the digit-wise inversion of the operand. - - .. method:: logical_or(other, context=None) - - :meth:`logical_or` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise ``or`` of the two operands. - - .. method:: logical_xor(other, context=None) - - :meth:`logical_xor` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise exclusive or of the two operands. - - .. method:: max(other, context=None) - - Like ``max(self, other)`` except that the context rounding rule is applied - before returning and that :const:`NaN` values are either signaled or - ignored (depending on the context and whether they are signaling or - quiet). - - .. method:: max_mag(other, context=None) - - Similar to the :meth:`.max` method, but the comparison is done using the - absolute values of the operands. - - .. method:: min(other, context=None) - - Like ``min(self, other)`` except that the context rounding rule is applied - before returning and that :const:`NaN` values are either signaled or - ignored (depending on the context and whether they are signaling or - quiet). - - .. method:: min_mag(other, context=None) - - Similar to the :meth:`.min` method, but the comparison is done using the - absolute values of the operands. - - .. method:: next_minus(context=None) - - Return the largest number representable in the given context (or in the - current thread's context if no context is given) that is smaller than the - given operand. - - .. method:: next_plus(context=None) - - Return the smallest number representable in the given context (or in the - current thread's context if no context is given) that is larger than the - given operand. - - .. method:: next_toward(other, context=None) - - If the two operands are unequal, return the number closest to the first - operand in the direction of the second operand. If both operands are - numerically equal, return a copy of the first operand with the sign set to - be the same as the sign of the second operand. - - .. method:: normalize(context=None) - - Normalize the number by stripping the rightmost trailing zeros and - converting any result equal to :const:`Decimal('0')` to - :const:`Decimal('0e0')`. Used for producing canonical values for attributes - of an equivalence class. For example, ``Decimal('32.100')`` and - ``Decimal('0.321000e+2')`` both normalize to the equivalent value - ``Decimal('32.1')``. - - .. method:: number_class(context=None) - - Return a string describing the *class* of the operand. The returned value - is one of the following ten strings. - - * ``"-Infinity"``, indicating that the operand is negative infinity. - * ``"-Normal"``, indicating that the operand is a negative normal number. - * ``"-Subnormal"``, indicating that the operand is negative and subnormal. - * ``"-Zero"``, indicating that the operand is a negative zero. - * ``"+Zero"``, indicating that the operand is a positive zero. - * ``"+Subnormal"``, indicating that the operand is positive and subnormal. - * ``"+Normal"``, indicating that the operand is a positive normal number. - * ``"+Infinity"``, indicating that the operand is positive infinity. - * ``"NaN"``, indicating that the operand is a quiet NaN (Not a Number). - * ``"sNaN"``, indicating that the operand is a signaling NaN. - - .. method:: quantize(exp, rounding=None, context=None) - - Return a value equal to the first operand after rounding and having the - exponent of the second operand. - - >>> Decimal('1.41421356').quantize(Decimal('1.000')) - Decimal('1.414') - - Unlike other operations, if the length of the coefficient after the - quantize operation would be greater than precision, then an - :const:`InvalidOperation` is signaled. This guarantees that, unless there - is an error condition, the quantized exponent is always equal to that of - the right-hand operand. - - Also unlike other operations, quantize never signals Underflow, even if - the result is subnormal and inexact. - - If the exponent of the second operand is larger than that of the first - then rounding may be necessary. In this case, the rounding mode is - determined by the ``rounding`` argument if given, else by the given - ``context`` argument; if neither argument is given the rounding mode of - the current thread's context is used. - - An error is returned whenever the resulting exponent is greater than - :attr:`Emax` or less than :attr:`Etiny`. - - .. method:: radix() - - Return ``Decimal(10)``, the radix (base) in which the :class:`Decimal` - class does all its arithmetic. Included for compatibility with the - specification. - - .. method:: remainder_near(other, context=None) - - Return the remainder from dividing *self* by *other*. This differs from - ``self % other`` in that the sign of the remainder is chosen so as to - minimize its absolute value. More precisely, the return value is - ``self - n * other`` where ``n`` is the integer nearest to the exact - value of ``self / other``, and if two integers are equally near then the - even one is chosen. - - If the result is zero then its sign will be the sign of *self*. - - >>> Decimal(18).remainder_near(Decimal(10)) - Decimal('-2') - >>> Decimal(25).remainder_near(Decimal(10)) - Decimal('5') - >>> Decimal(35).remainder_near(Decimal(10)) - Decimal('-5') - - .. method:: rotate(other, context=None) - - Return the result of rotating the digits of the first operand by an amount - specified by the second operand. The second operand must be an integer in - the range -precision through precision. The absolute value of the second - operand gives the number of places to rotate. If the second operand is - positive then rotation is to the left; otherwise rotation is to the right. - The coefficient of the first operand is padded on the left with zeros to - length precision if necessary. The sign and exponent of the first operand - are unchanged. - - .. method:: same_quantum(other, context=None) - - Test whether self and other have the same exponent or whether both are - :const:`NaN`. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: scaleb(other, context=None) - - Return the first operand with exponent adjusted by the second. - Equivalently, return the first operand multiplied by ``10**other``. The - second operand must be an integer. - - .. method:: shift(other, context=None) - - Return the result of shifting the digits of the first operand by an amount - specified by the second operand. The second operand must be an integer in - the range -precision through precision. The absolute value of the second - operand gives the number of places to shift. If the second operand is - positive then the shift is to the left; otherwise the shift is to the - right. Digits shifted into the coefficient are zeros. The sign and - exponent of the first operand are unchanged. - - .. method:: sqrt(context=None) - - Return the square root of the argument to full precision. - - - .. method:: to_eng_string(context=None) - - Convert to a string, using engineering notation if an exponent is needed. - - Engineering notation has an exponent which is a multiple of 3. This - can leave up to 3 digits to the left of the decimal place and may - require the addition of either one or two trailing zeros. - - For example, this converts ``Decimal('123E+1')`` to ``Decimal('1.23E+3')``. - - .. method:: to_integral(rounding=None, context=None) - - Identical to the :meth:`to_integral_value` method. The ``to_integral`` - name has been kept for compatibility with older versions. - - .. method:: to_integral_exact(rounding=None, context=None) - - Round to the nearest integer, signaling :const:`Inexact` or - :const:`Rounded` as appropriate if rounding occurs. The rounding mode is - determined by the ``rounding`` parameter if given, else by the given - ``context``. If neither parameter is given then the rounding mode of the - current context is used. - - .. method:: to_integral_value(rounding=None, context=None) - - Round to the nearest integer without signaling :const:`Inexact` or - :const:`Rounded`. If given, applies *rounding*; otherwise, uses the - rounding method in either the supplied *context* or the current context. - - -.. _logical_operands_label: - -Logical operands -^^^^^^^^^^^^^^^^ - -The :meth:`logical_and`, :meth:`logical_invert`, :meth:`logical_or`, -and :meth:`logical_xor` methods expect their arguments to be *logical -operands*. A *logical operand* is a :class:`Decimal` instance whose -exponent and sign are both zero, and whose digits are all either -:const:`0` or :const:`1`. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-context: - -Context objects ---------------- - -Contexts are environments for arithmetic operations. They govern precision, set -rules for rounding, determine which signals are treated as exceptions, and limit -the range for exponents. - -Each thread has its own current context which is accessed or changed using the -:func:`getcontext` and :func:`setcontext` functions: - - -.. function:: getcontext() - - Return the current context for the active thread. - - -.. function:: setcontext(c) - - Set the current context for the active thread to *c*. - -You can also use the :keyword:`with` statement and the :func:`localcontext` -function to temporarily change the active context. - -.. function:: localcontext(ctx=None) - - Return a context manager that will set the current context for the active thread - to a copy of *ctx* on entry to the with-statement and restore the previous context - when exiting the with-statement. If no context is specified, a copy of the - current context is used. - - For example, the following code sets the current decimal precision to 42 places, - performs a calculation, and then automatically restores the previous context:: - - from decimal import localcontext - - with localcontext() as ctx: - ctx.prec = 42 # Perform a high precision calculation - s = calculate_something() - s = +s # Round the final result back to the default precision - -New contexts can also be created using the :class:`Context` constructor -described below. In addition, the module provides three pre-made contexts: - - -.. class:: BasicContext - - This is a standard context defined by the General Decimal Arithmetic - Specification. Precision is set to nine. Rounding is set to - :const:`ROUND_HALF_UP`. All flags are cleared. All traps are enabled (treated - as exceptions) except :const:`Inexact`, :const:`Rounded`, and - :const:`Subnormal`. - - Because many of the traps are enabled, this context is useful for debugging. - - -.. class:: ExtendedContext - - This is a standard context defined by the General Decimal Arithmetic - Specification. Precision is set to nine. Rounding is set to - :const:`ROUND_HALF_EVEN`. All flags are cleared. No traps are enabled (so that - exceptions are not raised during computations). - - Because the traps are disabled, this context is useful for applications that - prefer to have result value of :const:`NaN` or :const:`Infinity` instead of - raising exceptions. This allows an application to complete a run in the - presence of conditions that would otherwise halt the program. - - -.. class:: DefaultContext - - This context is used by the :class:`Context` constructor as a prototype for new - contexts. Changing a field (such a precision) has the effect of changing the - default for new contexts created by the :class:`Context` constructor. - - This context is most useful in multi-threaded environments. Changing one of the - fields before threads are started has the effect of setting system-wide - defaults. Changing the fields after threads have started is not recommended as - it would require thread synchronization to prevent race conditions. - - In single threaded environments, it is preferable to not use this context at - all. Instead, simply create contexts explicitly as described below. - - The default values are :attr:`prec`\ =\ :const:`28`, - :attr:`rounding`\ =\ :const:`ROUND_HALF_EVEN`, - and enabled traps for :class:`Overflow`, :class:`InvalidOperation`, and - :class:`DivisionByZero`. - -In addition to the three supplied contexts, new contexts can be created with the -:class:`Context` constructor. - - -.. class:: Context(prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None) - - Creates a new context. If a field is not specified or is :const:`None`, the - default values are copied from the :const:`DefaultContext`. If the *flags* - field is not specified or is :const:`None`, all flags are cleared. - - *prec* is an integer in the range [:const:`1`, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. - - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. - - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. - - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, :const:`0`], - *Emax* in the range [:const:`0`, :const:`MAX_EMAX`]. - - The *capitals* field is either :const:`0` or :const:`1` (the default). If set to - :const:`1`, exponents are printed with a capital :const:`E`; otherwise, a - lowercase :const:`e` is used: :const:`Decimal('6.02e+23')`. - - The *clamp* field is either :const:`0` (the default) or :const:`1`. - If set to :const:`1`, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - :const:`0` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most ``Emax``. When *clamp* is - :const:`1`, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: - - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') - - A *clamp* value of :const:`1` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. - - The :class:`Context` class defines several general purpose methods as well as - a large number of methods for doing arithmetic directly in a given context. - In addition, for each of the :class:`Decimal` methods described above (with - the exception of the :meth:`adjusted` and :meth:`as_tuple` methods) there is - a corresponding :class:`Context` method. For example, for a :class:`Context` - instance ``C`` and :class:`Decimal` instance ``x``, ``C.exp(x)`` is - equivalent to ``x.exp(context=C)``. Each :class:`Context` method accepts a - Python integer (an instance of :class:`int`) anywhere that a - Decimal instance is accepted. - - - .. method:: clear_flags() - - Resets all of the flags to :const:`0`. - - .. method:: clear_traps() - - Resets all of the traps to :const:`0`. - - .. versionadded:: 3.3 - - .. method:: copy() - - Return a duplicate of the context. - - .. method:: copy_decimal(num) - - Return a copy of the Decimal instance num. - - .. method:: create_decimal(num) - - Creates a new Decimal instance from *num* but using *self* as - context. Unlike the :class:`Decimal` constructor, the context precision, - rounding method, flags, and traps are applied to the conversion. - - This is useful because constants are often given to a greater precision - than is needed by the application. Another benefit is that rounding - immediately eliminates unintended effects from digits beyond the current - precision. In the following example, using unrounded inputs means that - adding zero to a sum can change the result: - - .. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> Decimal('3.4445') + Decimal('1.0023') - Decimal('4.45') - >>> Decimal('3.4445') + Decimal(0) + Decimal('1.0023') - Decimal('4.44') - - This method implements the to-number operation of the IBM specification. - If the argument is a string, no leading or trailing whitespace or - underscores are permitted. - - .. method:: create_decimal_from_float(f) - - Creates a new Decimal instance from a float *f* but rounding using *self* - as the context. Unlike the :meth:`Decimal.from_float` class method, - the context precision, rounding method, flags, and traps are applied to - the conversion. - - .. doctest:: - - >>> context = Context(prec=5, rounding=ROUND_DOWN) - >>> context.create_decimal_from_float(math.pi) - Decimal('3.1415') - >>> context = Context(prec=5, traps=[Inexact]) - >>> context.create_decimal_from_float(math.pi) - Traceback (most recent call last): - ... - decimal.Inexact: None - - .. versionadded:: 3.1 - - .. method:: Etiny() - - Returns a value equal to ``Emin - prec + 1`` which is the minimum exponent - value for subnormal results. When underflow occurs, the exponent is set - to :const:`Etiny`. - - .. method:: Etop() - - Returns a value equal to ``Emax - prec + 1``. - - The usual approach to working with decimals is to create :class:`Decimal` - instances and then apply arithmetic operations which take place within the - current context for the active thread. An alternative approach is to use - context methods for calculating within a specific context. The methods are - similar to those for the :class:`Decimal` class and are only briefly - recounted here. - - - .. method:: abs(x) - - Returns the absolute value of *x*. - - - .. method:: add(x, y) - - Return the sum of *x* and *y*. - - - .. method:: canonical(x) - - Returns the same Decimal object *x*. - - - .. method:: compare(x, y) - - Compares *x* and *y* numerically. - - - .. method:: compare_signal(x, y) - - Compares the values of the two operands numerically. - - - .. method:: compare_total(x, y) - - Compares two operands using their abstract representation. - - - .. method:: compare_total_mag(x, y) - - Compares two operands using their abstract representation, ignoring sign. - - - .. method:: copy_abs(x) - - Returns a copy of *x* with the sign set to 0. - - - .. method:: copy_negate(x) - - Returns a copy of *x* with the sign inverted. - - - .. method:: copy_sign(x, y) - - Copies the sign from *y* to *x*. - - - .. method:: divide(x, y) - - Return *x* divided by *y*. - - - .. method:: divide_int(x, y) - - Return *x* divided by *y*, truncated to an integer. - - - .. method:: divmod(x, y) - - Divides two numbers and returns the integer part of the result. - - - .. method:: exp(x) - - Returns `e ** x`. - - - .. method:: fma(x, y, z) - - Returns *x* multiplied by *y*, plus *z*. - - - .. method:: is_canonical(x) - - Returns ``True`` if *x* is canonical; otherwise returns ``False``. - - - .. method:: is_finite(x) - - Returns ``True`` if *x* is finite; otherwise returns ``False``. - - - .. method:: is_infinite(x) - - Returns ``True`` if *x* is infinite; otherwise returns ``False``. - - - .. method:: is_nan(x) - - Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - - - .. method:: is_normal(x) - - Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - - - .. method:: is_qnan(x) - - Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - - - .. method:: is_signed(x) - - Returns ``True`` if *x* is negative; otherwise returns ``False``. - - - .. method:: is_snan(x) - - Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - - - .. method:: is_subnormal(x) - - Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - - - .. method:: is_zero(x) - - Returns ``True`` if *x* is a zero; otherwise returns ``False``. - - - .. method:: ln(x) - - Returns the natural (base e) logarithm of *x*. - - - .. method:: log10(x) - - Returns the base 10 logarithm of *x*. - - - .. method:: logb(x) - - Returns the exponent of the magnitude of the operand's MSD. - - - .. method:: logical_and(x, y) - - Applies the logical operation *and* between each operand's digits. - - - .. method:: logical_invert(x) - - Invert all the digits in *x*. - - - .. method:: logical_or(x, y) - - Applies the logical operation *or* between each operand's digits. - - - .. method:: logical_xor(x, y) - - Applies the logical operation *xor* between each operand's digits. - - - .. method:: max(x, y) - - Compares two values numerically and returns the maximum. - - - .. method:: max_mag(x, y) - - Compares the values numerically with their sign ignored. - - - .. method:: min(x, y) - - Compares two values numerically and returns the minimum. - - - .. method:: min_mag(x, y) - - Compares the values numerically with their sign ignored. - - - .. method:: minus(x) - - Minus corresponds to the unary prefix minus operator in Python. - - - .. method:: multiply(x, y) - - Return the product of *x* and *y*. - - - .. method:: next_minus(x) - - Returns the largest representable number smaller than *x*. - - - .. method:: next_plus(x) - - Returns the smallest representable number larger than *x*. - - - .. method:: next_toward(x, y) - - Returns the number closest to *x*, in direction towards *y*. - - - .. method:: normalize(x) - - Reduces *x* to its simplest form. - - - .. method:: number_class(x) - - Returns an indication of the class of *x*. - - - .. method:: plus(x) - - Plus corresponds to the unary prefix plus operator in Python. This - operation applies the context precision and rounding, so it is *not* an - identity operation. - - - .. method:: power(x, y, modulo=None) - - Return ``x`` to the power of ``y``, reduced modulo ``modulo`` if given. - - With two arguments, compute ``x**y``. If ``x`` is negative then ``y`` - must be integral. The result will be inexact unless ``y`` is integral and - the result is finite and can be expressed exactly in 'precision' digits. - The rounding mode of the context is used. Results are always correctly-rounded - in the Python version. - - .. versionchanged:: 3.3 - The C module computes :meth:`power` in terms of the correctly-rounded - :meth:`exp` and :meth:`ln` functions. The result is well-defined but - only "almost always correctly-rounded". - - With three arguments, compute ``(x**y) % modulo``. For the three argument - form, the following restrictions on the arguments hold: - - - all three arguments must be integral - - ``y`` must be nonnegative - - at least one of ``x`` or ``y`` must be nonzero - - ``modulo`` must be nonzero and have at most 'precision' digits - - The value resulting from ``Context.power(x, y, modulo)`` is - equal to the value that would be obtained by computing ``(x**y) - % modulo`` with unbounded precision, but is computed more - efficiently. The exponent of the result is zero, regardless of - the exponents of ``x``, ``y`` and ``modulo``. The result is - always exact. - - - .. method:: quantize(x, y) - - Returns a value equal to *x* (rounded), having the exponent of *y*. - - - .. method:: radix() - - Just returns 10, as this is Decimal, :) - - - .. method:: remainder(x, y) - - Returns the remainder from integer division. - - The sign of the result, if non-zero, is the same as that of the original - dividend. - - - .. method:: remainder_near(x, y) - - Returns ``x - y * n``, where *n* is the integer nearest the exact value - of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - - - .. method:: rotate(x, y) - - Returns a rotated copy of *x*, *y* times. - - - .. method:: same_quantum(x, y) - - Returns ``True`` if the two operands have the same exponent. - - - .. method:: scaleb (x, y) - - Returns the first operand after adding the second value its exp. - - - .. method:: shift(x, y) - - Returns a shifted copy of *x*, *y* times. - - - .. method:: sqrt(x) - - Square root of a non-negative number to context precision. - - - .. method:: subtract(x, y) - - Return the difference between *x* and *y*. - - - .. method:: to_eng_string(x) - - Convert to a string, using engineering notation if an exponent is needed. - - Engineering notation has an exponent which is a multiple of 3. This - can leave up to 3 digits to the left of the decimal place and may - require the addition of either one or two trailing zeros. - - - .. method:: to_integral_exact(x) - - Rounds to an integer. - - - .. method:: to_sci_string(x) - - Converts a number to a string using scientific notation. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -.. _decimal-rounding-modes: - -Constants ---------- - -The constants in this section are only relevant for the C module. They -are also included in the pure Python version for compatibility. - -+---------------------+---------------------+-------------------------------+ -| | 32-bit | 64-bit | -+=====================+=====================+===============================+ -| .. data:: MAX_PREC | :const:`425000000` | :const:`999999999999999999` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MAX_EMAX | :const:`425000000` | :const:`999999999999999999` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_EMIN | :const:`-425000000` | :const:`-999999999999999999` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_ETINY | :const:`-849999999` | :const:`-1999999999999999997` | -+---------------------+---------------------+-------------------------------+ - - -.. data:: HAVE_THREADS - - The default value is ``True``. If Python is compiled without threads, the - C version automatically disables the expensive thread local context - machinery. In this case, the value is ``False``. - -Rounding modes --------------- - -.. data:: ROUND_CEILING - - Round towards :const:`Infinity`. - -.. data:: ROUND_DOWN - - Round towards zero. - -.. data:: ROUND_FLOOR - - Round towards :const:`-Infinity`. - -.. data:: ROUND_HALF_DOWN - - Round to nearest with ties going towards zero. - -.. data:: ROUND_HALF_EVEN - - Round to nearest with ties going to nearest even integer. - -.. data:: ROUND_HALF_UP - - Round to nearest with ties going away from zero. - -.. data:: ROUND_UP - - Round away from zero. - -.. data:: ROUND_05UP - - Round away from zero if last digit after rounding towards zero would have - been 0 or 5; otherwise round towards zero. - - -.. _decimal-signals: - -Signals -------- - -Signals represent conditions that arise during computation. Each corresponds to -one context flag and one context trap enabler. - -The context flag is set whenever the condition is encountered. After the -computation, flags may be checked for informational purposes (for instance, to -determine whether a computation was exact). After checking the flags, be sure to -clear all flags before starting the next computation. - -If the context's trap enabler is set for the signal, then the condition causes a -Python exception to be raised. For example, if the :class:`DivisionByZero` trap -is set, then a :exc:`DivisionByZero` exception is raised upon encountering the -condition. - - -.. class:: Clamped - - Altered an exponent to fit representation constraints. - - Typically, clamping occurs when an exponent falls outside the context's - :attr:`Emin` and :attr:`Emax` limits. If possible, the exponent is reduced to - fit by adding zeros to the coefficient. - - -.. class:: DecimalException - - Base class for other signals and a subclass of :exc:`ArithmeticError`. - - -.. class:: DivisionByZero - - Signals the division of a non-infinite number by zero. - - Can occur with division, modulo division, or when raising a number to a negative - power. If this signal is not trapped, returns :const:`Infinity` or - :const:`-Infinity` with the sign determined by the inputs to the calculation. - - -.. class:: Inexact - - Indicates that rounding occurred and the result is not exact. - - Signals when non-zero digits were discarded during rounding. The rounded result - is returned. The signal flag or trap is used to detect when results are - inexact. - - -.. class:: InvalidOperation - - An invalid operation was performed. - - Indicates that an operation was requested that does not make sense. If not - trapped, returns :const:`NaN`. Possible causes include:: - - Infinity - Infinity - 0 * Infinity - Infinity / Infinity - x % 0 - Infinity % x - sqrt(-x) and x > 0 - 0 ** 0 - x ** (non-integer) - x ** Infinity - - -.. class:: Overflow - - Numerical overflow. - - Indicates the exponent is larger than :attr:`Emax` after rounding has - occurred. If not trapped, the result depends on the rounding mode, either - pulling inward to the largest representable finite number or rounding outward - to :const:`Infinity`. In either case, :class:`Inexact` and :class:`Rounded` - are also signaled. - - -.. class:: Rounded - - Rounding occurred though possibly no information was lost. - - Signaled whenever rounding discards digits; even if those digits are zero - (such as rounding :const:`5.00` to :const:`5.0`). If not trapped, returns - the result unchanged. This signal is used to detect loss of significant - digits. - - -.. class:: Subnormal - - Exponent was lower than :attr:`Emin` prior to rounding. - - Occurs when an operation result is subnormal (the exponent is too small). If - not trapped, returns the result unchanged. - - -.. class:: Underflow - - Numerical underflow with result rounded to zero. - - Occurs when a subnormal result is pushed to zero by rounding. :class:`Inexact` - and :class:`Subnormal` are also signaled. - - -.. class:: FloatOperation - - Enable stricter semantics for mixing floats and Decimals. - - If the signal is not trapped (default), mixing floats and Decimals is - permitted in the :class:`~decimal.Decimal` constructor, - :meth:`~decimal.Context.create_decimal` and all comparison operators. - Both conversion and comparisons are exact. Any occurrence of a mixed - operation is silently recorded by setting :exc:`FloatOperation` in the - context flags. Explicit conversions with :meth:`~decimal.Decimal.from_float` - or :meth:`~decimal.Context.create_decimal_from_float` do not set the flag. - - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise :exc:`FloatOperation`. - - -The following table summarizes the hierarchy of signals:: - - exceptions.ArithmeticError(exceptions.Exception) - DecimalException - Clamped - DivisionByZero(DecimalException, exceptions.ZeroDivisionError) - Inexact - Overflow(Inexact, Rounded) - Underflow(Inexact, Rounded, Subnormal) - InvalidOperation - Rounded - Subnormal - FloatOperation(DecimalException, exceptions.TypeError) - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - -.. _decimal-notes: - -Floating Point Notes --------------------- - - -Mitigating round-off error with increased precision -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The use of decimal floating point eliminates decimal representation error -(making it possible to represent :const:`0.1` exactly); however, some operations -can still incur round-off error when non-zero digits exceed the fixed precision. - -The effects of round-off error can be amplified by the addition or subtraction -of nearly offsetting quantities resulting in loss of significance. Knuth -provides two instructive examples where rounded floating point arithmetic with -insufficient precision causes the breakdown of the associative and distributive -properties of addition: - -.. doctest:: newcontext - - # Examples from Seminumerical Algorithms, Section 4.2.2. - >>> from decimal import Decimal, getcontext - >>> getcontext().prec = 8 - - >>> u, v, w = Decimal(11111113), Decimal(-11111111), Decimal('7.51111111') - >>> (u + v) + w - Decimal('9.5111111') - >>> u + (v + w) - Decimal('10') - - >>> u, v, w = Decimal(20000), Decimal(-6), Decimal('6.0000003') - >>> (u*v) + (u*w) - Decimal('0.01') - >>> u * (v+w) - Decimal('0.0060000') - -The :mod:`decimal` module makes it possible to restore the identities by -expanding the precision sufficiently to avoid loss of significance: - -.. doctest:: newcontext - - >>> getcontext().prec = 20 - >>> u, v, w = Decimal(11111113), Decimal(-11111111), Decimal('7.51111111') - >>> (u + v) + w - Decimal('9.51111111') - >>> u + (v + w) - Decimal('9.51111111') - >>> - >>> u, v, w = Decimal(20000), Decimal(-6), Decimal('6.0000003') - >>> (u*v) + (u*w) - Decimal('0.0060000') - >>> u * (v+w) - Decimal('0.0060000') - - -Special values -^^^^^^^^^^^^^^ - -The number system for the :mod:`decimal` module provides special values -including :const:`NaN`, :const:`sNaN`, :const:`-Infinity`, :const:`Infinity`, -and two zeros, :const:`+0` and :const:`-0`. - -Infinities can be constructed directly with: ``Decimal('Infinity')``. Also, -they can arise from dividing by zero when the :exc:`DivisionByZero` signal is -not trapped. Likewise, when the :exc:`Overflow` signal is not trapped, infinity -can result from rounding beyond the limits of the largest representable number. - -The infinities are signed (affine) and can be used in arithmetic operations -where they get treated as very large, indeterminate numbers. For instance, -adding a constant to infinity gives another infinite result. - -Some operations are indeterminate and return :const:`NaN`, or if the -:exc:`InvalidOperation` signal is trapped, raise an exception. For example, -``0/0`` returns :const:`NaN` which means "not a number". This variety of -:const:`NaN` is quiet and, once created, will flow through other computations -always resulting in another :const:`NaN`. This behavior can be useful for a -series of computations that occasionally have missing inputs --- it allows the -calculation to proceed while flagging specific results as invalid. - -A variant is :const:`sNaN` which signals rather than remaining quiet after every -operation. This is a useful return value when an invalid result needs to -interrupt a calculation for special handling. - -The behavior of Python's comparison operators can be a little surprising where a -:const:`NaN` is involved. A test for equality where one of the operands is a -quiet or signaling :const:`NaN` always returns :const:`False` (even when doing -``Decimal('NaN')==Decimal('NaN')``), while a test for inequality always returns -:const:`True`. An attempt to compare two Decimals using any of the ``<``, -``<=``, ``>`` or ``>=`` operators will raise the :exc:`InvalidOperation` signal -if either operand is a :const:`NaN`, and return :const:`False` if this signal is -not trapped. Note that the General Decimal Arithmetic specification does not -specify the behavior of direct comparisons; these rules for comparisons -involving a :const:`NaN` were taken from the IEEE 854 standard (see Table 3 in -section 5.7). To ensure strict standards-compliance, use the :meth:`compare` -and :meth:`compare-signal` methods instead. - -The signed zeros can result from calculations that underflow. They keep the sign -that would have resulted if the calculation had been carried out to greater -precision. Since their magnitude is zero, both positive and negative zeros are -treated as equal and their sign is informational. - -In addition to the two signed zeros which are distinct yet equal, there are -various representations of zero with differing precisions yet equivalent in -value. This takes a bit of getting used to. For an eye accustomed to -normalized floating point representations, it is not immediately obvious that -the following calculation returns a value equal to zero: - - >>> 1 / Decimal('Infinity') - Decimal('0E-1000026') - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-threads: - -Working with threads --------------------- - -The :func:`getcontext` function accesses a different :class:`Context` object for -each thread. Having separate thread contexts means that threads may make -changes (such as ``getcontext().prec=10``) without interfering with other threads. - -Likewise, the :func:`setcontext` function automatically assigns its target to -the current thread. - -If :func:`setcontext` has not been called before :func:`getcontext`, then -:func:`getcontext` will automatically create a new context for use in the -current thread. - -The new context is copied from a prototype context called *DefaultContext*. To -control the defaults so that each thread will use the same values throughout the -application, directly modify the *DefaultContext* object. This should be done -*before* any threads are started so that there won't be a race condition between -threads calling :func:`getcontext`. For example:: - - # Set applicationwide defaults for all threads about to be launched - DefaultContext.prec = 12 - DefaultContext.rounding = ROUND_DOWN - DefaultContext.traps = ExtendedContext.traps.copy() - DefaultContext.traps[InvalidOperation] = 1 - setcontext(DefaultContext) - - # Afterwards, the threads can be started - t1.start() - t2.start() - t3.start() - . . . - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-recipes: - -Recipes -------- - -Here are a few recipes that serve as utility functions and that demonstrate ways -to work with the :class:`Decimal` class:: - - def moneyfmt(value, places=2, curr='', sep=',', dp='.', - pos='', neg='-', trailneg=''): - """Convert Decimal to a money formatted string. - - places: required number of places after the decimal point - curr: optional currency symbol before the sign (may be blank) - sep: optional grouping separator (comma, period, space, or blank) - dp: decimal point indicator (comma or period) - only specify as blank when places is zero - pos: optional sign for positive numbers: '+', space or blank - neg: optional sign for negative numbers: '-', '(', space or blank - trailneg:optional trailing minus indicator: '-', ')', space or blank - - >>> d = Decimal('-1234567.8901') - >>> moneyfmt(d, curr='$') - '-$1,234,567.89' - >>> moneyfmt(d, places=0, sep='.', dp='', neg='', trailneg='-') - '1.234.568-' - >>> moneyfmt(d, curr='$', neg='(', trailneg=')') - '($1,234,567.89)' - >>> moneyfmt(Decimal(123456789), sep=' ') - '123 456 789.00' - >>> moneyfmt(Decimal('-0.02'), neg='<', trailneg='>') - '<0.02>' - - """ - q = Decimal(10) ** -places # 2 places --> '0.01' - sign, digits, exp = value.quantize(q).as_tuple() - result = [] - digits = list(map(str, digits)) - build, next = result.append, digits.pop - if sign: - build(trailneg) - for i in range(places): - build(next() if digits else '0') - if places: - build(dp) - if not digits: - build('0') - i = 0 - while digits: - build(next()) - i += 1 - if i == 3 and digits: - i = 0 - build(sep) - build(curr) - build(neg if sign else pos) - return ''.join(reversed(result)) - - def pi(): - """Compute Pi to the current precision. - - >>> print(pi()) - 3.141592653589793238462643383 - - """ - getcontext().prec += 2 # extra digits for intermediate steps - three = Decimal(3) # substitute "three=3.0" for regular floats - lasts, t, s, n, na, d, da = 0, three, 3, 1, 0, 0, 24 - while s != lasts: - lasts = s - n, na = n+na, na+8 - d, da = d+da, da+32 - t = (t * n) / d - s += t - getcontext().prec -= 2 - return +s # unary plus applies the new precision - - def exp(x): - """Return e raised to the power of x. Result type matches input type. - - >>> print(exp(Decimal(1))) - 2.718281828459045235360287471 - >>> print(exp(Decimal(2))) - 7.389056098930650227230427461 - >>> print(exp(2.0)) - 7.38905609893 - >>> print(exp(2+0j)) - (7.38905609893+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num = 0, 0, 1, 1, 1 - while s != lasts: - lasts = s - i += 1 - fact *= i - num *= x - s += num / fact - getcontext().prec -= 2 - return +s - - def cos(x): - """Return the cosine of x as measured in radians. - - The Taylor series approximation works best for a small value of x. - For larger values, first compute x = x % (2 * pi). - - >>> print(cos(Decimal('0.5'))) - 0.8775825618903727161162815826 - >>> print(cos(0.5)) - 0.87758256189 - >>> print(cos(0.5+0j)) - (0.87758256189+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num, sign = 0, 0, 1, 1, 1, 1 - while s != lasts: - lasts = s - i += 2 - fact *= i * (i-1) - num *= x * x - sign *= -1 - s += num / fact * sign - getcontext().prec -= 2 - return +s - - def sin(x): - """Return the sine of x as measured in radians. - - The Taylor series approximation works best for a small value of x. - For larger values, first compute x = x % (2 * pi). - - >>> print(sin(Decimal('0.5'))) - 0.4794255386042030002732879352 - >>> print(sin(0.5)) - 0.479425538604 - >>> print(sin(0.5+0j)) - (0.479425538604+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num, sign = 1, 0, x, 1, x, 1 - while s != lasts: - lasts = s - i += 2 - fact *= i * (i-1) - num *= x * x - sign *= -1 - s += num / fact * sign - getcontext().prec -= 2 - return +s - - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-faq: - -Decimal FAQ ------------ - -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to -minimize typing when using the interactive interpreter? - -A. Some users abbreviate the constructor to just a single letter: - - >>> D = decimal.Decimal - >>> D('1.23') + D('3.45') - Decimal('4.68') - -Q. In a fixed-point application with two decimal places, some inputs have many -places and need to be rounded. Others are not supposed to have excess digits -and need to be validated. What methods should be used? - -A. The :meth:`quantize` method rounds to a fixed number of decimal places. If -the :const:`Inexact` trap is set, it is also useful for validation: - - >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') - - >>> # Round to two places - >>> Decimal('3.214').quantize(TWOPLACES) - Decimal('3.21') - - >>> # Validate that a number does not exceed two places - >>> Decimal('3.21').quantize(TWOPLACES, context=Context(traps=[Inexact])) - Decimal('3.21') - - >>> Decimal('3.214').quantize(TWOPLACES, context=Context(traps=[Inexact])) - Traceback (most recent call last): - ... - Inexact: None - -Q. Once I have valid two place inputs, how do I maintain that invariant -throughout an application? - -A. Some operations like addition, subtraction, and multiplication by an integer -will automatically preserve fixed point. Others operations, like division and -non-integer multiplication, will change the number of decimal places and need to -be followed-up with a :meth:`quantize` step: - - >>> a = Decimal('102.72') # Initial fixed-point values - >>> b = Decimal('3.17') - >>> a + b # Addition preserves fixed-point - Decimal('105.89') - >>> a - b - Decimal('99.55') - >>> a * 42 # So does integer multiplication - Decimal('4314.24') - >>> (a * b).quantize(TWOPLACES) # Must quantize non-integer multiplication - Decimal('325.62') - >>> (b / a).quantize(TWOPLACES) # And quantize division - Decimal('0.03') - -In developing fixed-point applications, it is convenient to define functions -to handle the :meth:`quantize` step: - - >>> def mul(x, y, fp=TWOPLACES): - ... return (x * y).quantize(fp) - >>> def div(x, y, fp=TWOPLACES): - ... return (x / y).quantize(fp) - - >>> mul(a, b) # Automatically preserve fixed-point - Decimal('325.62') - >>> div(b, a) - Decimal('0.03') - -Q. There are many ways to express the same value. The numbers :const:`200`, -:const:`200.000`, :const:`2E2`, and :const:`.02E+4` all have the same value at -various precisions. Is there a way to transform them to a single recognizable -canonical value? - -A. The :meth:`normalize` method maps all equivalent values to a single -representative: - - >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) - >>> [v.normalize() for v in values] - [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] - -Q. Some decimal values always print with exponential notation. Is there a way -to get a non-exponential representation? - -A. For some values, exponential notation is the only way to express the number -of significant places in the coefficient. For example, expressing -:const:`5.0E+3` as :const:`5000` keeps the value constant but cannot show the -original's two-place significance. - -If an application does not care about tracking significance, it is easy to -remove the exponent and trailing zeroes, losing significance, but keeping the -value unchanged: - - >>> def remove_exponent(d): - ... return d.quantize(Decimal(1)) if d == d.to_integral() else d.normalize() - - >>> remove_exponent(Decimal('5E+3')) - Decimal('5000') - -Q. Is there a way to convert a regular float to a :class:`Decimal`? - -A. Yes, any binary floating point number can be exactly expressed as a -Decimal though an exact conversion may take more precision than intuition would -suggest: - -.. doctest:: - - >>> Decimal(math.pi) - Decimal('3.141592653589793115997963468544185161590576171875') - -Q. Within a complex calculation, how can I make sure that I haven't gotten a -spurious result because of insufficient precision or rounding anomalies. - -A. The decimal module makes it easy to test results. A best practice is to -re-run calculations using greater precision and with various rounding modes. -Widely differing results indicate insufficient precision, rounding mode issues, -ill-conditioned inputs, or a numerically unstable algorithm. - -Q. I noticed that context precision is applied to the results of operations but -not to the inputs. Is there anything to watch out for when mixing values of -different precisions? - -A. Yes. The principle is that all values are considered to be exact and so is -the arithmetic on those values. Only the results are rounded. The advantage -for inputs is that "what you type is what you get". A disadvantage is that the -results can look odd if you forget that the inputs haven't been rounded: - -.. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> Decimal('3.104') + Decimal('2.104') - Decimal('5.21') - >>> Decimal('3.104') + Decimal('0.000') + Decimal('2.104') - Decimal('5.20') - -The solution is either to increase precision or to force rounding of inputs -using the unary plus operation: - -.. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> +Decimal('1.23456789') # unary plus triggers rounding - Decimal('1.23') - -Alternatively, inputs can be rounded upon creation using the -:meth:`Context.create_decimal` method: - - >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') - Decimal('1.2345') diff --git a/third_party/python/Doc/library/development.rst b/third_party/python/Doc/library/development.rst deleted file mode 100644 index d2b5fa2aa..000000000 --- a/third_party/python/Doc/library/development.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _development: - -***************** -Development Tools -***************** - -The modules described in this chapter help you write software. For example, the -:mod:`pydoc` module takes a module and generates documentation based on the -module's contents. The :mod:`doctest` and :mod:`unittest` modules contains -frameworks for writing unit tests that automatically exercise code and verify -that the expected output is produced. :program:`2to3` can translate Python 2.x -source code into valid Python 3.x code. - -The list of modules described in this chapter is: - - -.. toctree:: - - typing.rst - pydoc.rst - doctest.rst - unittest.rst - unittest.mock.rst - unittest.mock-examples.rst - 2to3.rst - test.rst diff --git a/third_party/python/Doc/library/difflib.rst b/third_party/python/Doc/library/difflib.rst deleted file mode 100644 index f044cb2d6..000000000 --- a/third_party/python/Doc/library/difflib.rst +++ /dev/null @@ -1,746 +0,0 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== - -.. module:: difflib - :synopsis: Helpers for computing differences between objects. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. Markup by Fred L. Drake, Jr. - -**Source code:** :source:`Lib/difflib.py` - -.. testsetup:: - - import sys - from difflib import * - --------------- - -This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce difference -information in various formats, including HTML and context and unified -diffs. For comparing directories and files, see also, the :mod:`filecmp` module. - - -.. class:: SequenceMatcher - - This is a flexible class for comparing pairs of sequences of any type, so long - as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a - little fancier than, an algorithm published in the late 1980's by Ratcliff and - Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to - find the longest contiguous matching subsequence that contains no "junk" - elements; these "junk" elements are ones that are uninteresting in some - sense, such as blank lines or whitespace. (Handling junk is an - extension to the Ratcliff and Obershelp algorithm.) The same - idea is then applied recursively to the pieces of the sequences to the left and - to the right of the matching subsequence. This does not yield minimal edit - sequences, but does tend to yield matches that "look right" to people. - - **Timing:** The basic Ratcliff-Obershelp algorithm is cubic time in the worst - case and quadratic time in the expected case. :class:`SequenceMatcher` is - quadratic time for the worst case and has expected-case behavior dependent in a - complicated way on how many elements the sequences have in common; best case - time is linear. - - **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that - automatically treats certain sequence items as junk. The heuristic counts how many - times each individual item appears in the sequence. If an item's duplicates (after - the first one) account for more than 1% of the sequence and the sequence is at least - 200 items long, this item is marked as "popular" and is treated as junk for - the purpose of sequence matching. This heuristic can be turned off by setting - the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. - - .. versionadded:: 3.2 - The *autojunk* parameter. - - -.. class:: Differ - - This is a class for comparing sequences of lines of text, and producing - human-readable differences or deltas. Differ uses :class:`SequenceMatcher` - both to compare sequences of lines, and to compare sequences of characters - within similar (near-matching) lines. - - Each line of a :class:`Differ` delta begins with a two-letter code: - - +----------+-------------------------------------------+ - | Code | Meaning | - +==========+===========================================+ - | ``'- '`` | line unique to sequence 1 | - +----------+-------------------------------------------+ - | ``'+ '`` | line unique to sequence 2 | - +----------+-------------------------------------------+ - | ``' '`` | line common to both sequences | - +----------+-------------------------------------------+ - | ``'? '`` | line not present in either input sequence | - +----------+-------------------------------------------+ - - Lines beginning with '``?``' attempt to guide the eye to intraline differences, - and were not present in either input sequence. These lines can be confusing if - the sequences contain tab characters. - - -.. class:: HtmlDiff - - This class can be used to create an HTML table (or a complete HTML file - containing the table) showing a side by side, line by line comparison of text - with inter-line and intra-line change highlights. The table can be generated in - either full or contextual difference mode. - - The constructor for this class is: - - - .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK) - - Initializes instance of :class:`HtmlDiff`. - - *tabsize* is an optional keyword argument to specify tab stop spacing and - defaults to ``8``. - - *wrapcolumn* is an optional keyword to specify column number where lines are - broken and wrapped, defaults to ``None`` where lines are not wrapped. - - *linejunk* and *charjunk* are optional keyword arguments passed into :func:`ndiff` - (used by :class:`HtmlDiff` to generate the side by side HTML differences). See - :func:`ndiff` documentation for argument default values and descriptions. - - The following methods are public: - - .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, \ - numlines=5, *, charset='utf-8') - - Compares *fromlines* and *tolines* (lists of strings) and returns a string which - is a complete HTML file containing a table showing line by line differences with - inter-line and intra-line changes highlighted. - - *fromdesc* and *todesc* are optional keyword arguments to specify from/to file - column header strings (both default to an empty string). - - *context* and *numlines* are both optional keyword arguments. Set *context* to - ``True`` when contextual differences are to be shown, else the default is - ``False`` to show the full files. *numlines* defaults to ``5``. When *context* - is ``True`` *numlines* controls the number of context lines which surround the - difference highlights. When *context* is ``False`` *numlines* controls the - number of lines which are shown before a difference highlight when using the - "next" hyperlinks (setting to zero would cause the "next" hyperlinks to place - the next difference highlight at the top of the browser without any leading - context). - - .. versionchanged:: 3.5 - *charset* keyword-only argument was added. The default charset of - HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``. - - .. method:: make_table(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5) - - Compares *fromlines* and *tolines* (lists of strings) and returns a string which - is a complete HTML table showing line by line differences with inter-line and - intra-line changes highlighted. - - The arguments for this method are the same as those for the :meth:`make_file` - method. - - :file:`Tools/scripts/diff.py` is a command-line front-end to this class and - contains a good example of its use. - - -.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\\n') - - Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` - generating the delta lines) in context diff format. - - Context diffs are a compact way of showing just the lines that have changed plus - a few lines of context. The changes are shown in a before/after style. The - number of context lines is set by *n* which defaults to three. - - By default, the diff control lines (those with ``***`` or ``---``) are created - with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing - newlines. - - For inputs that do not have trailing newlines, set the *lineterm* argument to - ``""`` so that the output will be uniformly newline free. - - The context diff format normally has a header for filenames and modification - times. Any or all of these may be specified using strings for *fromfile*, - *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally - expressed in the ISO 8601 format. If not specified, the - strings default to blanks. - - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] - >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', tofile='after.py')) - *** before.py - --- after.py - *************** - *** 1,4 **** - ! bacon - ! eggs - ! ham - guido - --- 1,4 ---- - ! python - ! eggy - ! hamster - guido - - See :ref:`difflib-interface` for a more detailed example. - - -.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6) - - Return a list of the best "good enough" matches. *word* is a sequence for which - close matches are desired (typically a string), and *possibilities* is a list of - sequences against which to match *word* (typically a list of strings). - - Optional argument *n* (default ``3``) is the maximum number of close matches to - return; *n* must be greater than ``0``. - - Optional argument *cutoff* (default ``0.6``) is a float in the range [0, 1]. - Possibilities that don't score at least that similar to *word* are ignored. - - The best (no more than *n*) matches among the possibilities are returned in a - list, sorted by similarity score, most similar first. - - >>> get_close_matches('appel', ['ape', 'apple', 'peach', 'puppy']) - ['apple', 'ape'] - >>> import keyword - >>> get_close_matches('wheel', keyword.kwlist) - ['while'] - >>> get_close_matches('pineapple', keyword.kwlist) - [] - >>> get_close_matches('accept', keyword.kwlist) - ['except'] - - -.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK) - - Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style - delta (a :term:`generator` generating the delta lines). - - Optional keyword parameters *linejunk* and *charjunk* are filtering functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns - true if the string is junk, or false if not. The default is ``None``. There - is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) - -- however the underlying :class:`SequenceMatcher` class does a dynamic - analysis of which lines are so frequent as to constitute noise, and this - usually works better than using this function. - - *charjunk*: A function that accepts a character (a string of length 1), and - returns if the character is junk, or false if not. The default is module-level - function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a - blank or tab; it's a bad idea to include newline in this!). - - :file:`Tools/scripts/ndiff.py` is a command-line front-end to this function. - - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) - >>> print(''.join(diff), end="") - - one - ? ^ - + ore - ? ^ - - two - - three - ? - - + tree - + emu - - -.. function:: restore(sequence, which) - - Return one of the two sequences that generated a delta. - - Given a *sequence* produced by :meth:`Differ.compare` or :func:`ndiff`, extract - lines originating from file 1 or 2 (parameter *which*), stripping off line - prefixes. - - Example: - - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) - >>> diff = list(diff) # materialize the generated delta into a list - >>> print(''.join(restore(diff, 1)), end="") - one - two - three - >>> print(''.join(restore(diff, 2)), end="") - ore - tree - emu - - -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\\n') - - Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` - generating the delta lines) in unified diff format. - - Unified diffs are a compact way of showing just the lines that have changed plus - a few lines of context. The changes are shown in an inline style (instead of - separate before/after blocks). The number of context lines is set by *n* which - defaults to three. - - By default, the diff control lines (those with ``---``, ``+++``, or ``@@``) are - created with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing - newlines. - - For inputs that do not have trailing newlines, set the *lineterm* argument to - ``""`` so that the output will be uniformly newline free. - - The context diff format normally has a header for filenames and modification - times. Any or all of these may be specified using strings for *fromfile*, - *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally - expressed in the ISO 8601 format. If not specified, the - strings default to blanks. - - - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] - >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(unified_diff(s1, s2, fromfile='before.py', tofile='after.py')) - --- before.py - +++ after.py - @@ -1,4 +1,4 @@ - -bacon - -eggs - -ham - +python - +eggy - +hamster - guido - - See :ref:`difflib-interface` for a more detailed example. - -.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\\n') - - Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a - sequence of delta lines (also bytes) in the format returned by *dfunc*. - *dfunc* must be a callable, typically either :func:`unified_diff` or - :func:`context_diff`. - - Allows you to compare data with unknown or inconsistent encoding. All - inputs except *n* must be bytes objects, not str. Works by losslessly - converting all inputs (except *n*) to str, and calling ``dfunc(a, b, - fromfile, tofile, fromfiledate, tofiledate, n, lineterm)``. The output of - *dfunc* is then converted back to bytes, so the delta lines that you - receive have the same unknown/inconsistent encodings as *a* and *b*. - - .. versionadded:: 3.5 - -.. function:: IS_LINE_JUNK(line) - - Return true for ignorable lines. The line *line* is ignorable if *line* is - blank or contains a single ``'#'``, otherwise it is not ignorable. Used as a - default for parameter *linejunk* in :func:`ndiff` in older versions. - - -.. function:: IS_CHARACTER_JUNK(ch) - - Return true for ignorable characters. The character *ch* is ignorable if *ch* - is a space or tab, otherwise it is not ignorable. Used as a default for - parameter *charjunk* in :func:`ndiff`. - - -.. seealso:: - - `Pattern Matching: The Gestalt Approach `_ - Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. - - -.. _sequence-matcher: - -SequenceMatcher Objects ------------------------ - -The :class:`SequenceMatcher` class has this constructor: - - -.. class:: SequenceMatcher(isjunk=None, a='', b='', autojunk=True) - - Optional argument *isjunk* must be ``None`` (the default) or a one-argument - function that takes a sequence element and returns true if and only if the - element is "junk" and should be ignored. Passing ``None`` for *isjunk* is - equivalent to passing ``lambda x: 0``; in other words, no elements are ignored. - For example, pass:: - - lambda x: x in " \t" - - if you're comparing lines as sequences of characters, and don't want to synch up - on blanks or hard tabs. - - The optional arguments *a* and *b* are sequences to be compared; both default to - empty strings. The elements of both sequences must be :term:`hashable`. - - The optional argument *autojunk* can be used to disable the automatic junk - heuristic. - - .. versionadded:: 3.2 - The *autojunk* parameter. - - SequenceMatcher objects get three data attributes: *bjunk* is the - set of elements of *b* for which *isjunk* is ``True``; *bpopular* is the set of - non-junk elements considered popular by the heuristic (if it is not - disabled); *b2j* is a dict mapping the remaining elements of *b* to a list - of positions where they occur. All three are reset whenever *b* is reset - with :meth:`set_seqs` or :meth:`set_seq2`. - - .. versionadded:: 3.2 - The *bjunk* and *bpopular* attributes. - - :class:`SequenceMatcher` objects have the following methods: - - .. method:: set_seqs(a, b) - - Set the two sequences to be compared. - - :class:`SequenceMatcher` computes and caches detailed information about the - second sequence, so if you want to compare one sequence against many - sequences, use :meth:`set_seq2` to set the commonly used sequence once and - call :meth:`set_seq1` repeatedly, once for each of the other sequences. - - - .. method:: set_seq1(a) - - Set the first sequence to be compared. The second sequence to be compared - is not changed. - - - .. method:: set_seq2(b) - - Set the second sequence to be compared. The first sequence to be compared - is not changed. - - - .. method:: find_longest_match(alo, ahi, blo, bhi) - - Find longest matching block in ``a[alo:ahi]`` and ``b[blo:bhi]``. - - If *isjunk* was omitted or ``None``, :meth:`find_longest_match` returns - ``(i, j, k)`` such that ``a[i:i+k]`` is equal to ``b[j:j+k]``, where ``alo - <= i <= i+k <= ahi`` and ``blo <= j <= j+k <= bhi``. For all ``(i', j', - k')`` meeting those conditions, the additional conditions ``k >= k'``, ``i - <= i'``, and if ``i == i'``, ``j <= j'`` are also met. In other words, of - all maximal matching blocks, return one that starts earliest in *a*, and - of all those maximal matching blocks that start earliest in *a*, return - the one that starts earliest in *b*. - - >>> s = SequenceMatcher(None, " abcd", "abcd abcd") - >>> s.find_longest_match(0, 5, 0, 9) - Match(a=0, b=4, size=5) - - If *isjunk* was provided, first the longest matching block is determined - as above, but with the additional restriction that no junk element appears - in the block. Then that block is extended as far as possible by matching - (only) junk elements on both sides. So the resulting block never matches - on junk except as identical junk happens to be adjacent to an interesting - match. - - Here's the same example as before, but considering blanks to be junk. That - prevents ``' abcd'`` from matching the ``' abcd'`` at the tail end of the - second sequence directly. Instead only the ``'abcd'`` can match, and - matches the leftmost ``'abcd'`` in the second sequence: - - >>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd") - >>> s.find_longest_match(0, 5, 0, 9) - Match(a=1, b=0, size=4) - - If no blocks match, this returns ``(alo, blo, 0)``. - - This method returns a :term:`named tuple` ``Match(a, b, size)``. - - - .. method:: get_matching_blocks() - - Return list of triples describing non-overlapping matching subsequences. - Each triple is of the form ``(i, j, n)``, - and means that ``a[i:i+n] == b[j:j+n]``. The - triples are monotonically increasing in *i* and *j*. - - The last triple is a dummy, and has the value ``(len(a), len(b), 0)``. It - is the only triple with ``n == 0``. If ``(i, j, n)`` and ``(i', j', n')`` - are adjacent triples in the list, and the second is not the last triple in - the list, then ``i+n < i'`` or ``j+n < j'``; in other words, adjacent - triples always describe non-adjacent equal blocks. - - .. XXX Explain why a dummy is used! - - .. doctest:: - - >>> s = SequenceMatcher(None, "abxcd", "abcd") - >>> s.get_matching_blocks() - [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)] - - - .. method:: get_opcodes() - - Return list of 5-tuples describing how to turn *a* into *b*. Each tuple is - of the form ``(tag, i1, i2, j1, j2)``. The first tuple has ``i1 == j1 == - 0``, and remaining tuples have *i1* equal to the *i2* from the preceding - tuple, and, likewise, *j1* equal to the previous *j2*. - - The *tag* values are strings, with these meanings: - - +---------------+---------------------------------------------+ - | Value | Meaning | - +===============+=============================================+ - | ``'replace'`` | ``a[i1:i2]`` should be replaced by | - | | ``b[j1:j2]``. | - +---------------+---------------------------------------------+ - | ``'delete'`` | ``a[i1:i2]`` should be deleted. Note that | - | | ``j1 == j2`` in this case. | - +---------------+---------------------------------------------+ - | ``'insert'`` | ``b[j1:j2]`` should be inserted at | - | | ``a[i1:i1]``. Note that ``i1 == i2`` in | - | | this case. | - +---------------+---------------------------------------------+ - | ``'equal'`` | ``a[i1:i2] == b[j1:j2]`` (the sub-sequences | - | | are equal). | - +---------------+---------------------------------------------+ - - For example:: - - >>> a = "qabxcd" - >>> b = "abycdf" - >>> s = SequenceMatcher(None, a, b) - >>> for tag, i1, i2, j1, j2 in s.get_opcodes(): - ... print('{:7} a[{}:{}] --> b[{}:{}] {!r:>8} --> {!r}'.format( - ... tag, i1, i2, j1, j2, a[i1:i2], b[j1:j2])) - delete a[0:1] --> b[0:0] 'q' --> '' - equal a[1:3] --> b[0:2] 'ab' --> 'ab' - replace a[3:4] --> b[2:3] 'x' --> 'y' - equal a[4:6] --> b[3:5] 'cd' --> 'cd' - insert a[6:6] --> b[5:6] '' --> 'f' - - - .. method:: get_grouped_opcodes(n=3) - - Return a :term:`generator` of groups with up to *n* lines of context. - - Starting with the groups returned by :meth:`get_opcodes`, this method - splits out smaller change clusters and eliminates intervening ranges which - have no changes. - - The groups are returned in the same format as :meth:`get_opcodes`. - - - .. method:: ratio() - - Return a measure of the sequences' similarity as a float in the range [0, - 1]. - - Where T is the total number of elements in both sequences, and M is the - number of matches, this is 2.0\*M / T. Note that this is ``1.0`` if the - sequences are identical, and ``0.0`` if they have nothing in common. - - This is expensive to compute if :meth:`get_matching_blocks` or - :meth:`get_opcodes` hasn't already been called, in which case you may want - to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an - upper bound. - - - .. method:: quick_ratio() - - Return an upper bound on :meth:`ratio` relatively quickly. - - - .. method:: real_quick_ratio() - - Return an upper bound on :meth:`ratio` very quickly. - - -The three methods that return the ratio of matching to total characters can give -different results due to differing levels of approximation, although -:meth:`quick_ratio` and :meth:`real_quick_ratio` are always at least as large as -:meth:`ratio`: - - >>> s = SequenceMatcher(None, "abcd", "bcde") - >>> s.ratio() - 0.75 - >>> s.quick_ratio() - 0.75 - >>> s.real_quick_ratio() - 1.0 - - -.. _sequencematcher-examples: - -SequenceMatcher Examples ------------------------- - -This example compares two strings, considering blanks to be "junk": - - >>> s = SequenceMatcher(lambda x: x == " ", - ... "private Thread currentThread;", - ... "private volatile Thread currentThread;") - -:meth:`ratio` returns a float in [0, 1], measuring the similarity of the -sequences. As a rule of thumb, a :meth:`ratio` value over 0.6 means the -sequences are close matches: - - >>> print(round(s.ratio(), 3)) - 0.866 - -If you're only interested in where the sequences match, -:meth:`get_matching_blocks` is handy: - - >>> for block in s.get_matching_blocks(): - ... print("a[%d] and b[%d] match for %d elements" % block) - a[0] and b[0] match for 8 elements - a[8] and b[17] match for 21 elements - a[29] and b[38] match for 0 elements - -Note that the last tuple returned by :meth:`get_matching_blocks` is always a -dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last -tuple element (number of elements matched) is ``0``. - -If you want to know how to change the first sequence into the second, use -:meth:`get_opcodes`: - - >>> for opcode in s.get_opcodes(): - ... print("%6s a[%d:%d] b[%d:%d]" % opcode) - equal a[0:8] b[0:8] - insert a[8:8] b[8:17] - equal a[8:29] b[17:38] - -.. seealso:: - - * The :func:`get_close_matches` function in this module which shows how - simple code building on :class:`SequenceMatcher` can be used to do useful - work. - - * `Simple version control recipe - `_ for a small application - built with :class:`SequenceMatcher`. - - -.. _differ-objects: - -Differ Objects --------------- - -Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** -diffs. To the contrary, minimal diffs are often counter-intuitive, because they -synch up anywhere possible, sometimes accidental matches 100 pages apart. -Restricting synch points to contiguous matches preserves some notion of -locality, at the occasional cost of producing a longer diff. - -The :class:`Differ` class has this constructor: - - -.. class:: Differ(linejunk=None, charjunk=None) - - Optional keyword parameters *linejunk* and *charjunk* are for filter functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns true - if the string is junk. The default is ``None``, meaning that no line is - considered junk. - - *charjunk*: A function that accepts a single character argument (a string of - length 1), and returns true if the character is junk. The default is ``None``, - meaning that no character is considered junk. - - These junk-filtering functions speed up matching to find - differences and do not cause any differing lines or characters to - be ignored. Read the description of the - :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* - parameter for an explanation. - - :class:`Differ` objects are used (deltas generated) via a single method: - - - .. method:: Differ.compare(a, b) - - Compare two sequences of lines, and generate the delta (a sequence of lines). - - Each sequence must contain individual single-line strings ending with - newlines. Such sequences can be obtained from the - :meth:`~io.IOBase.readlines` method of file-like objects. The delta - generated also consists of newline-terminated strings, ready to be - printed as-is via the :meth:`~io.IOBase.writelines` method of a - file-like object. - - -.. _differ-examples: - -Differ Example --------------- - -This example compares two texts. First we set up the texts, sequences of -individual single-line strings ending with newlines (such sequences can also be -obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): - - >>> text1 = ''' 1. Beautiful is better than ugly. - ... 2. Explicit is better than implicit. - ... 3. Simple is better than complex. - ... 4. Complex is better than complicated. - ... '''.splitlines(keepends=True) - >>> len(text1) - 4 - >>> text1[0][-1] - '\n' - >>> text2 = ''' 1. Beautiful is better than ugly. - ... 3. Simple is better than complex. - ... 4. Complicated is better than complex. - ... 5. Flat is better than nested. - ... '''.splitlines(keepends=True) - -Next we instantiate a Differ object: - - >>> d = Differ() - -Note that when instantiating a :class:`Differ` object we may pass functions to -filter out line and character "junk." See the :meth:`Differ` constructor for -details. - -Finally, we compare the two: - - >>> result = list(d.compare(text1, text2)) - -``result`` is a list of strings, so let's pretty-print it: - - >>> from pprint import pprint - >>> pprint(result) - [' 1. Beautiful is better than ugly.\n', - '- 2. Explicit is better than implicit.\n', - '- 3. Simple is better than complex.\n', - '+ 3. Simple is better than complex.\n', - '? ++\n', - '- 4. Complex is better than complicated.\n', - '? ^ ---- ^\n', - '+ 4. Complicated is better than complex.\n', - '? ++++ ^ ^\n', - '+ 5. Flat is better than nested.\n'] - -As a single multi-line string it looks like this: - - >>> import sys - >>> sys.stdout.writelines(result) - 1. Beautiful is better than ugly. - - 2. Explicit is better than implicit. - - 3. Simple is better than complex. - + 3. Simple is better than complex. - ? ++ - - 4. Complex is better than complicated. - ? ^ ---- ^ - + 4. Complicated is better than complex. - ? ++++ ^ ^ - + 5. Flat is better than nested. - - -.. _difflib-interface: - -A command-line interface to difflib ------------------------------------ - -This example shows how to use difflib to create a ``diff``-like utility. -It is also contained in the Python source distribution, as -:file:`Tools/scripts/diff.py`. - -.. literalinclude:: ../../Tools/scripts/diff.py diff --git a/third_party/python/Doc/library/dis.rst b/third_party/python/Doc/library/dis.rst deleted file mode 100644 index d5757c2c6..000000000 --- a/third_party/python/Doc/library/dis.rst +++ /dev/null @@ -1,1223 +0,0 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== - -.. module:: dis - :synopsis: Disassembler for Python bytecode. - -**Source code:** :source:`Lib/dis.py` - --------------- - -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by -disassembling it. The CPython bytecode which this module takes as an input is -defined in the file :file:`Include/opcode.h` and used by the compiler and the -interpreter. - -.. impl-detail:: - - Bytecode is an implementation detail of the CPython interpreter. No - guarantees are made that bytecode will not be added, removed, or changed - between versions of Python. Use of this module should not be considered to - work across Python VMs or Python releases. - - .. versionchanged:: 3.6 - Use 2 bytes for each instruction. Previously the number of bytes varied - by instruction. - - -Example: Given the function :func:`myfunc`:: - - def myfunc(alist): - return len(alist) - -the following command can be used to display the disassembly of -:func:`myfunc`:: - - >>> dis.dis(myfunc) - 2 0 LOAD_GLOBAL 0 (len) - 2 LOAD_FAST 0 (alist) - 4 CALL_FUNCTION 1 - 6 RETURN_VALUE - -(The "2" is a line number). - -Bytecode analysis ------------------ - -.. versionadded:: 3.4 - -The bytecode analysis API allows pieces of Python code to be wrapped in a -:class:`Bytecode` object that provides easy access to details of the compiled -code. - -.. class:: Bytecode(x, *, first_line=None, current_offset=None) - - - Analyse the bytecode corresponding to a function, generator, method, string - of source code, or a code object (as returned by :func:`compile`). - - This is a convenience wrapper around many of the functions listed below, most - notably :func:`get_instructions`, as iterating over a :class:`Bytecode` - instance yields the bytecode operations as :class:`Instruction` instances. - - If *first_line* is not ``None``, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - If *current_offset* is not ``None``, it refers to an instruction offset in the - disassembled code. Setting this means :meth:`.dis` will display a "current - instruction" marker against the specified opcode. - - .. classmethod:: from_traceback(tb) - - Construct a :class:`Bytecode` instance from the given traceback, setting - *current_offset* to the instruction responsible for the exception. - - .. data:: codeobj - - The compiled code object. - - .. data:: first_line - - The first source line of the code object (if available) - - .. method:: dis() - - Return a formatted view of the bytecode operations (the same as printed by - :func:`dis.dis`, but returned as a multi-line string). - - .. method:: info() - - Return a formatted multi-line string with detailed information about the - code object, like :func:`code_info`. - -Example:: - - >>> bytecode = dis.Bytecode(myfunc) - >>> for instr in bytecode: - ... print(instr.opname) - ... - LOAD_GLOBAL - LOAD_FAST - CALL_FUNCTION - RETURN_VALUE - - -Analysis functions ------------------- - -The :mod:`dis` module also defines the following analysis functions that convert -the input directly to the desired output. They can be useful if only a single -operation is being performed, so the intermediate analysis object isn't useful: - -.. function:: code_info(x) - - Return a formatted multi-line string with detailed code object information - for the supplied function, generator, method, source code string or code object. - - Note that the exact contents of code info strings are highly implementation - dependent and they may change arbitrarily across Python VMs or Python - releases. - - .. versionadded:: 3.2 - - -.. function:: show_code(x, *, file=None) - - Print detailed code object information for the supplied function, method, - source code string or code object to *file* (or ``sys.stdout`` if *file* - is not specified). - - This is a convenient shorthand for ``print(code_info(x), file=file)``, - intended for interactive exploration at the interpreter prompt. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: dis(x=None, *, file=None) - - Disassemble the *x* object. *x* can denote either a module, a class, a - method, a function, a generator, a code object, a string of source code or - a byte sequence of raw bytecode. For a module, it disassembles all functions. - For a class, it disassembles all methods (including class and static methods). - For a code object or sequence of raw bytecode, it prints one line per bytecode - instruction. Strings are first compiled to code objects with the :func:`compile` - built-in function before being disassembled. If no object is provided, this - function disassembles the last traceback. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: distb(tb=None, *, file=None) - - Disassemble the top-of-stack function of a traceback, using the last - traceback if none was passed. The instruction causing the exception is - indicated. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: disassemble(code, lasti=-1, *, file=None) - disco(code, lasti=-1, *, file=None) - - Disassemble a code object, indicating the last instruction if *lasti* was - provided. The output is divided in the following columns: - - #. the line number, for the first instruction of each line - #. the current instruction, indicated as ``-->``, - #. a labelled instruction, indicated with ``>>``, - #. the address of the instruction, - #. the operation code name, - #. operation parameters, and - #. interpretation of the parameters in parentheses. - - The parameter interpretation recognizes local and global variable names, - constant values, branch targets, and compare operators. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: get_instructions(x, *, first_line=None) - - Return an iterator over the instructions in the supplied function, method, - source code string or code object. - - The iterator generates a series of :class:`Instruction` named tuples giving - the details of each operation in the supplied code. - - If *first_line* is not ``None``, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - .. versionadded:: 3.4 - - -.. function:: findlinestarts(code) - - This generator function uses the ``co_firstlineno`` and ``co_lnotab`` - attributes of the code object *code* to find the offsets which are starts of - lines in the source code. They are generated as ``(offset, lineno)`` pairs. - See :source:`Objects/lnotab_notes.txt` for the ``co_lnotab`` format and - how to decode it. - - .. versionchanged:: 3.6 - Line numbers can be decreasing. Before, they were always increasing. - - -.. function:: findlabels(code) - - Detect all offsets in the code object *code* which are jump targets, and - return a list of these offsets. - - -.. function:: stack_effect(opcode, [oparg]) - - Compute the stack effect of *opcode* with argument *oparg*. - - .. versionadded:: 3.4 - -.. _bytecodes: - -Python Bytecode Instructions ----------------------------- - -The :func:`get_instructions` function and :class:`Bytecode` class provide -details of bytecode instructions as :class:`Instruction` instances: - -.. class:: Instruction - - Details for a bytecode operation - - .. data:: opcode - - numeric code for operation, corresponding to the opcode values listed - below and the bytecode values in the :ref:`opcode_collections`. - - - .. data:: opname - - human readable name for operation - - - .. data:: arg - - numeric argument to operation (if any), otherwise ``None`` - - - .. data:: argval - - resolved arg value (if known), otherwise same as arg - - - .. data:: argrepr - - human readable description of operation argument - - - .. data:: offset - - start index of operation within bytecode sequence - - - .. data:: starts_line - - line started by this opcode (if any), otherwise ``None`` - - - .. data:: is_jump_target - - ``True`` if other code jumps to here, otherwise ``False`` - - .. versionadded:: 3.4 - - -The Python compiler currently generates the following bytecode instructions. - - -**General instructions** - -.. opcode:: NOP - - Do nothing code. Used as a placeholder by the bytecode optimizer. - - -.. opcode:: POP_TOP - - Removes the top-of-stack (TOS) item. - - -.. opcode:: ROT_TWO - - Swaps the two top-most stack items. - - -.. opcode:: ROT_THREE - - Lifts second and third stack item one position up, moves top down to position - three. - - -.. opcode:: DUP_TOP - - Duplicates the reference on top of the stack. - - .. versionadded:: 3.2 - - -.. opcode:: DUP_TOP_TWO - - Duplicates the two references on top of the stack, leaving them in the - same order. - - .. versionadded:: 3.2 - - -**Unary operations** - -Unary operations take the top of the stack, apply the operation, and push the -result back on the stack. - -.. opcode:: UNARY_POSITIVE - - Implements ``TOS = +TOS``. - - -.. opcode:: UNARY_NEGATIVE - - Implements ``TOS = -TOS``. - - -.. opcode:: UNARY_NOT - - Implements ``TOS = not TOS``. - - -.. opcode:: UNARY_INVERT - - Implements ``TOS = ~TOS``. - - -.. opcode:: GET_ITER - - Implements ``TOS = iter(TOS)``. - - -.. opcode:: GET_YIELD_FROM_ITER - - If ``TOS`` is a :term:`generator iterator` or :term:`coroutine` object - it is left as is. Otherwise, implements ``TOS = iter(TOS)``. - - .. versionadded:: 3.5 - - -**Binary operations** - -Binary operations remove the top of the stack (TOS) and the second top-most -stack item (TOS1) from the stack. They perform the operation, and put the -result back on the stack. - -.. opcode:: BINARY_POWER - - Implements ``TOS = TOS1 ** TOS``. - - -.. opcode:: BINARY_MULTIPLY - - Implements ``TOS = TOS1 * TOS``. - - -.. opcode:: BINARY_MATRIX_MULTIPLY - - Implements ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - -.. opcode:: BINARY_FLOOR_DIVIDE - - Implements ``TOS = TOS1 // TOS``. - - -.. opcode:: BINARY_TRUE_DIVIDE - - Implements ``TOS = TOS1 / TOS``. - - -.. opcode:: BINARY_MODULO - - Implements ``TOS = TOS1 % TOS``. - - -.. opcode:: BINARY_ADD - - Implements ``TOS = TOS1 + TOS``. - - -.. opcode:: BINARY_SUBTRACT - - Implements ``TOS = TOS1 - TOS``. - - -.. opcode:: BINARY_SUBSCR - - Implements ``TOS = TOS1[TOS]``. - - -.. opcode:: BINARY_LSHIFT - - Implements ``TOS = TOS1 << TOS``. - - -.. opcode:: BINARY_RSHIFT - - Implements ``TOS = TOS1 >> TOS``. - - -.. opcode:: BINARY_AND - - Implements ``TOS = TOS1 & TOS``. - - -.. opcode:: BINARY_XOR - - Implements ``TOS = TOS1 ^ TOS``. - - -.. opcode:: BINARY_OR - - Implements ``TOS = TOS1 | TOS``. - - -**In-place operations** - -In-place operations are like binary operations, in that they remove TOS and -TOS1, and push the result back on the stack, but the operation is done in-place -when TOS1 supports it, and the resulting TOS may be (but does not have to be) -the original TOS1. - -.. opcode:: INPLACE_POWER - - Implements in-place ``TOS = TOS1 ** TOS``. - - -.. opcode:: INPLACE_MULTIPLY - - Implements in-place ``TOS = TOS1 * TOS``. - - -.. opcode:: INPLACE_MATRIX_MULTIPLY - - Implements in-place ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - -.. opcode:: INPLACE_FLOOR_DIVIDE - - Implements in-place ``TOS = TOS1 // TOS``. - - -.. opcode:: INPLACE_TRUE_DIVIDE - - Implements in-place ``TOS = TOS1 / TOS``. - - -.. opcode:: INPLACE_MODULO - - Implements in-place ``TOS = TOS1 % TOS``. - - -.. opcode:: INPLACE_ADD - - Implements in-place ``TOS = TOS1 + TOS``. - - -.. opcode:: INPLACE_SUBTRACT - - Implements in-place ``TOS = TOS1 - TOS``. - - -.. opcode:: INPLACE_LSHIFT - - Implements in-place ``TOS = TOS1 << TOS``. - - -.. opcode:: INPLACE_RSHIFT - - Implements in-place ``TOS = TOS1 >> TOS``. - - -.. opcode:: INPLACE_AND - - Implements in-place ``TOS = TOS1 & TOS``. - - -.. opcode:: INPLACE_XOR - - Implements in-place ``TOS = TOS1 ^ TOS``. - - -.. opcode:: INPLACE_OR - - Implements in-place ``TOS = TOS1 | TOS``. - - -.. opcode:: STORE_SUBSCR - - Implements ``TOS1[TOS] = TOS2``. - - -.. opcode:: DELETE_SUBSCR - - Implements ``del TOS1[TOS]``. - - -**Coroutine opcodes** - -.. opcode:: GET_AWAITABLE - - Implements ``TOS = get_awaitable(TOS)``, where ``get_awaitable(o)`` - returns ``o`` if ``o`` is a coroutine object or a generator object with - the CO_ITERABLE_COROUTINE flag, or resolves - ``o.__await__``. - - .. versionadded:: 3.5 - - -.. opcode:: GET_AITER - - Implements ``TOS = get_awaitable(TOS.__aiter__())``. See ``GET_AWAITABLE`` - for details about ``get_awaitable`` - - .. versionadded:: 3.5 - - -.. opcode:: GET_ANEXT - - Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE`` - for details about ``get_awaitable`` - - .. versionadded:: 3.5 - - -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the - stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack. - - .. versionadded:: 3.5 - - -.. opcode:: SETUP_ASYNC_WITH - - Creates a new frame object. - - .. versionadded:: 3.5 - - - -**Miscellaneous opcodes** - -.. opcode:: PRINT_EXPR - - Implements the expression statement for the interactive mode. TOS is removed - from the stack and printed. In non-interactive mode, an expression statement - is terminated with :opcode:`POP_TOP`. - - -.. opcode:: BREAK_LOOP - - Terminates a loop due to a :keyword:`break` statement. - - -.. opcode:: CONTINUE_LOOP (target) - - Continues a loop due to a :keyword:`continue` statement. *target* is the - address to jump to (which should be a :opcode:`FOR_ITER` instruction). - - -.. opcode:: SET_ADD (i) - - Calls ``set.add(TOS1[-i], TOS)``. Used to implement set comprehensions. - - -.. opcode:: LIST_APPEND (i) - - Calls ``list.append(TOS[-i], TOS)``. Used to implement list comprehensions. - - -.. opcode:: MAP_ADD (i) - - Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict - comprehensions. - - .. versionadded:: 3.1 - -For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` -instructions, while the added value or key/value pair is popped off, the -container object remains on the stack so that it is available for further -iterations of the loop. - - -.. opcode:: RETURN_VALUE - - Returns with TOS to the caller of the function. - - -.. opcode:: YIELD_VALUE - - Pops TOS and yields it from a :term:`generator`. - - -.. opcode:: YIELD_FROM - - Pops TOS and delegates to it as a subiterator from a :term:`generator`. - - .. versionadded:: 3.3 - - -.. opcode:: SETUP_ANNOTATIONS - - Checks whether ``__annotations__`` is defined in ``locals()``, if not it is - set up to an empty ``dict``. This opcode is only emitted if a class - or module body contains :term:`variable annotations ` - statically. - - .. versionadded:: 3.6 - - -.. opcode:: IMPORT_STAR - - Loads all symbols not starting with ``'_'`` directly from the module TOS to - the local namespace. The module is popped after loading all names. This - opcode implements ``from module import *``. - - -.. opcode:: POP_BLOCK - - Removes one block from the block stack. Per frame, there is a stack of - blocks, denoting nested loops, try statements, and such. - - -.. opcode:: POP_EXCEPT - - Removes one block from the block stack. The popped block must be an exception - handler block, as implicitly created when entering an except handler. In - addition to popping extraneous values from the frame stack, the last three - popped values are used to restore the exception state. - - -.. opcode:: END_FINALLY - - Terminates a :keyword:`finally` clause. The interpreter recalls whether the - exception has to be re-raised, or whether the function returns, and continues - with the outer-next block. - - -.. opcode:: LOAD_BUILD_CLASS - - Pushes :func:`builtins.__build_class__` onto the stack. It is later called - by :opcode:`CALL_FUNCTION` to construct a class. - - -.. opcode:: SETUP_WITH (delta) - - This opcode performs several operations before a with block starts. First, - it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_CLEANUP`. Then, - :meth:`~object.__enter__` is called, and a finally block pointing to *delta* - is pushed. Finally, the result of calling the enter method is pushed onto - the stack. The next opcode will either ignore it (:opcode:`POP_TOP`), or - store it in (a) variable(s) (:opcode:`STORE_FAST`, :opcode:`STORE_NAME`, or - :opcode:`UNPACK_SEQUENCE`). - - .. versionadded:: 3.2 - - -.. opcode:: WITH_CLEANUP_START - - Cleans up the stack when a :keyword:`with` statement block exits. TOS is the - context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values - indicating how/why the finally clause was entered: - - * SECOND = ``None`` - * (SECOND, THIRD) = (``WHY_{RETURN,CONTINUE}``), retval - * SECOND = ``WHY_*``; no retval below it - * (SECOND, THIRD, FOURTH) = exc_info() - - In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise - ``TOS(None, None, None)``. Pushes SECOND and result of the call - to the stack. - - -.. opcode:: WITH_CLEANUP_FINISH - - Pops exception type and result of 'exit' function call from the stack. - - If the stack represents an exception, *and* the function call returns a - 'true' value, this information is "zapped" and replaced with a single - ``WHY_SILENCED`` to prevent :opcode:`END_FINALLY` from re-raising the - exception. (But non-local gotos will still be resumed.) - - .. XXX explain the WHY stuff! - - -All of the following opcodes use their arguments. - -.. opcode:: STORE_NAME (namei) - - Implements ``name = TOS``. *namei* is the index of *name* in the attribute - :attr:`co_names` of the code object. The compiler tries to use - :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. - - -.. opcode:: DELETE_NAME (namei) - - Implements ``del name``, where *namei* is the index into :attr:`co_names` - attribute of the code object. - - -.. opcode:: UNPACK_SEQUENCE (count) - - Unpacks TOS into *count* individual values, which are put onto the stack - right-to-left. - - -.. opcode:: UNPACK_EX (counts) - - Implements assignment with a starred target: Unpacks an iterable in TOS into - individual values, where the total number of values can be smaller than the - number of items in the iterable: one of the new values will be a list of all - leftover items. - - The low byte of *counts* is the number of values before the list value, the - high byte of *counts* the number of values after it. The resulting values - are put onto the stack right-to-left. - - -.. opcode:: STORE_ATTR (namei) - - Implements ``TOS.name = TOS1``, where *namei* is the index of name in - :attr:`co_names`. - - -.. opcode:: DELETE_ATTR (namei) - - Implements ``del TOS.name``, using *namei* as index into :attr:`co_names`. - - -.. opcode:: STORE_GLOBAL (namei) - - Works as :opcode:`STORE_NAME`, but stores the name as a global. - - -.. opcode:: DELETE_GLOBAL (namei) - - Works as :opcode:`DELETE_NAME`, but deletes a global name. - - -.. opcode:: LOAD_CONST (consti) - - Pushes ``co_consts[consti]`` onto the stack. - - -.. opcode:: LOAD_NAME (namei) - - Pushes the value associated with ``co_names[namei]`` onto the stack. - - -.. opcode:: BUILD_TUPLE (count) - - Creates a tuple consuming *count* items from the stack, and pushes the - resulting tuple onto the stack. - - -.. opcode:: BUILD_LIST (count) - - Works as :opcode:`BUILD_TUPLE`, but creates a list. - - -.. opcode:: BUILD_SET (count) - - Works as :opcode:`BUILD_TUPLE`, but creates a set. - - -.. opcode:: BUILD_MAP (count) - - Pushes a new dictionary object onto the stack. Pops ``2 * count`` items - so that the dictionary holds *count* entries: - ``{..., TOS3: TOS2, TOS1: TOS}``. - - .. versionchanged:: 3.5 - The dictionary is created from stack items instead of creating an - empty dictionary pre-sized to hold *count* items. - - -.. opcode:: BUILD_CONST_KEY_MAP (count) - - The version of :opcode:`BUILD_MAP` specialized for constant keys. *count* - values are consumed from the stack. The top element on the stack contains - a tuple of keys. - - .. versionadded:: 3.6 - - -.. opcode:: BUILD_STRING (count) - - Concatenates *count* strings from the stack and pushes the resulting string - onto the stack. - - .. versionadded:: 3.6 - - -.. opcode:: BUILD_TUPLE_UNPACK (count) - - Pops *count* iterables from the stack, joins them in a single tuple, - and pushes the result. Implements iterable unpacking in tuple - displays ``(*x, *y, *z)``. - - .. versionadded:: 3.5 - - -.. opcode:: BUILD_TUPLE_UNPACK_WITH_CALL (count) - - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, - but is used for ``f(*x, *y, *z)`` call syntax. The stack item at position - ``count + 1`` should be the corresponding callable ``f``. - - .. versionadded:: 3.6 - - -.. opcode:: BUILD_LIST_UNPACK (count) - - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a list - instead of tuple. Implements iterable unpacking in list - displays ``[*x, *y, *z]``. - - .. versionadded:: 3.5 - - -.. opcode:: BUILD_SET_UNPACK (count) - - This is similar to :opcode:`BUILD_TUPLE_UNPACK`, but pushes a set - instead of tuple. Implements iterable unpacking in set - displays ``{*x, *y, *z}``. - - .. versionadded:: 3.5 - - -.. opcode:: BUILD_MAP_UNPACK (count) - - Pops *count* mappings from the stack, merges them into a single dictionary, - and pushes the result. Implements dictionary unpacking in dictionary - displays ``{**x, **y, **z}``. - - .. versionadded:: 3.5 - - -.. opcode:: BUILD_MAP_UNPACK_WITH_CALL (count) - - This is similar to :opcode:`BUILD_MAP_UNPACK`, - but is used for ``f(**x, **y, **z)`` call syntax. The stack item at - position ``count + 2`` should be the corresponding callable ``f``. - - .. versionadded:: 3.5 - .. versionchanged:: 3.6 - The position of the callable is determined by adding 2 to the opcode - argument instead of encoding it in the second byte of the argument. - - -.. opcode:: LOAD_ATTR (namei) - - Replaces TOS with ``getattr(TOS, co_names[namei])``. - - -.. opcode:: COMPARE_OP (opname) - - Performs a Boolean operation. The operation name can be found in - ``cmp_op[opname]``. - - -.. opcode:: IMPORT_NAME (namei) - - Imports the module ``co_names[namei]``. TOS and TOS1 are popped and provide - the *fromlist* and *level* arguments of :func:`__import__`. The module - object is pushed onto the stack. The current namespace is not affected: for - a proper import statement, a subsequent :opcode:`STORE_FAST` instruction - modifies the namespace. - - -.. opcode:: IMPORT_FROM (namei) - - Loads the attribute ``co_names[namei]`` from the module found in TOS. The - resulting object is pushed onto the stack, to be subsequently stored by a - :opcode:`STORE_FAST` instruction. - - -.. opcode:: JUMP_FORWARD (delta) - - Increments bytecode counter by *delta*. - - -.. opcode:: POP_JUMP_IF_TRUE (target) - - If TOS is true, sets the bytecode counter to *target*. TOS is popped. - - .. versionadded:: 3.1 - - -.. opcode:: POP_JUMP_IF_FALSE (target) - - If TOS is false, sets the bytecode counter to *target*. TOS is popped. - - .. versionadded:: 3.1 - - -.. opcode:: JUMP_IF_TRUE_OR_POP (target) - - If TOS is true, sets the bytecode counter to *target* and leaves TOS on the - stack. Otherwise (TOS is false), TOS is popped. - - .. versionadded:: 3.1 - - -.. opcode:: JUMP_IF_FALSE_OR_POP (target) - - If TOS is false, sets the bytecode counter to *target* and leaves TOS on the - stack. Otherwise (TOS is true), TOS is popped. - - .. versionadded:: 3.1 - - -.. opcode:: JUMP_ABSOLUTE (target) - - Set bytecode counter to *target*. - - -.. opcode:: FOR_ITER (delta) - - TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If - this yields a new value, push it on the stack (leaving the iterator below - it). If the iterator indicates it is exhausted TOS is popped, and the byte - code counter is incremented by *delta*. - - -.. opcode:: LOAD_GLOBAL (namei) - - Loads the global named ``co_names[namei]`` onto the stack. - - -.. opcode:: SETUP_LOOP (delta) - - Pushes a block for a loop onto the block stack. The block spans from the - current instruction with a size of *delta* bytes. - - -.. opcode:: SETUP_EXCEPT (delta) - - Pushes a try block from a try-except clause onto the block stack. *delta* - points to the first except block. - - -.. opcode:: SETUP_FINALLY (delta) - - Pushes a try block from a try-except clause onto the block stack. *delta* - points to the finally block. - - -.. opcode:: LOAD_FAST (var_num) - - Pushes a reference to the local ``co_varnames[var_num]`` onto the stack. - - -.. opcode:: STORE_FAST (var_num) - - Stores TOS into the local ``co_varnames[var_num]``. - - -.. opcode:: DELETE_FAST (var_num) - - Deletes local ``co_varnames[var_num]``. - - -.. opcode:: STORE_ANNOTATION (namei) - - Stores TOS as ``locals()['__annotations__'][co_names[namei]] = TOS``. - - .. versionadded:: 3.6 - - -.. opcode:: LOAD_CLOSURE (i) - - Pushes a reference to the cell contained in slot *i* of the cell and free - variable storage. The name of the variable is ``co_cellvars[i]`` if *i* is - less than the length of *co_cellvars*. Otherwise it is ``co_freevars[i - - len(co_cellvars)]``. - - -.. opcode:: LOAD_DEREF (i) - - Loads the cell contained in slot *i* of the cell and free variable storage. - Pushes a reference to the object the cell contains on the stack. - - -.. opcode:: LOAD_CLASSDEREF (i) - - Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before - consulting the cell. This is used for loading free variables in class - bodies. - - .. versionadded:: 3.4 - - -.. opcode:: STORE_DEREF (i) - - Stores TOS into the cell contained in slot *i* of the cell and free variable - storage. - - -.. opcode:: DELETE_DEREF (i) - - Empties the cell contained in slot *i* of the cell and free variable storage. - Used by the :keyword:`del` statement. - - .. versionadded:: 3.2 - - -.. opcode:: RAISE_VARARGS (argc) - - Raises an exception. *argc* indicates the number of arguments to the raise - statement, ranging from 0 to 3. The handler will find the traceback as TOS2, - the parameter as TOS1, and the exception as TOS. - - -.. opcode:: CALL_FUNCTION (argc) - - Calls a callable object with positional arguments. - *argc* indicates the number of positional arguments. - The top of the stack contains positional arguments, with the right-most - argument on top. Below the arguments is a callable object to call. - ``CALL_FUNCTION`` pops all arguments and the callable object off the stack, - calls the callable object with those arguments, and pushes the return value - returned by the callable object. - - .. versionchanged:: 3.6 - This opcode is used only for calls with positional arguments. - - -.. opcode:: CALL_FUNCTION_KW (argc) - - Calls a callable object with positional (if any) and keyword arguments. - *argc* indicates the total number of positional and keyword arguments. - The top element on the stack contains a tuple of keyword argument names. - Below that are keyword arguments in the order corresponding to the tuple. - Below that are positional arguments, with the right-most parameter on - top. Below the arguments is a callable object to call. - ``CALL_FUNCTION_KW`` pops all arguments and the callable object off the stack, - calls the callable object with those arguments, and pushes the return value - returned by the callable object. - - .. versionchanged:: 3.6 - Keyword arguments are packed in a tuple instead of a dictionary, - *argc* indicates the total number of arguments. - - -.. opcode:: CALL_FUNCTION_EX (flags) - - Calls a callable object with variable set of positional and keyword - arguments. If the lowest bit of *flags* is set, the top of the stack - contains a mapping object containing additional keyword arguments. - Below that is an iterable object containing positional arguments and - a callable object to call. :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and - :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` can be used for merging multiple - mapping objects and iterables containing arguments. - Before the callable is called, the mapping object and iterable object - are each "unpacked" and their contents passed in as keyword and - positional arguments respectively. - ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, - calls the callable object with those arguments, and pushes the return value - returned by the callable object. - - .. versionadded:: 3.6 - - -.. opcode:: MAKE_FUNCTION (argc) - - Pushes a new function object on the stack. From bottom to top, the consumed - stack must consist of values if the argument carries a specified flag value - - * ``0x01`` a tuple of default values for positional-only and - positional-or-keyword parameters in positional order - * ``0x02`` a dictionary of keyword-only parameters' default values - * ``0x04`` an annotation dictionary - * ``0x08`` a tuple containing cells for free variables, making a closure - * the code associated with the function (at TOS1) - * the :term:`qualified name` of the function (at TOS) - - -.. opcode:: BUILD_SLICE (argc) - - .. index:: builtin: slice - - Pushes a slice object on the stack. *argc* must be 2 or 3. If it is 2, - ``slice(TOS1, TOS)`` is pushed; if it is 3, ``slice(TOS2, TOS1, TOS)`` is - pushed. See the :func:`slice` built-in function for more information. - - -.. opcode:: EXTENDED_ARG (ext) - - Prefixes any opcode which has an argument too big to fit into the default two - bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. - - -.. opcode:: FORMAT_VALUE (flags) - - Used for implementing formatted literal strings (f-strings). Pops - an optional *fmt_spec* from the stack, then a required *value*. - *flags* is interpreted as follows: - - * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. - * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before - formatting it. - * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before - formatting it. - * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before - formatting it. - * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use - it, else use an empty *fmt_spec*. - - Formatting is performed using :c:func:`PyObject_Format`. The - result is pushed on the stack. - - .. versionadded:: 3.6 - - -.. opcode:: HAVE_ARGUMENT - - This is not really an opcode. It identifies the dividing line between - opcodes which don't use their argument and those that do - (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). - - .. versionchanged:: 3.6 - Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` - ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. - - -.. _opcode_collections: - -Opcode collections ------------------- - -These collections are provided for automatic introspection of bytecode -instructions: - -.. data:: opname - - Sequence of operation names, indexable using the bytecode. - - -.. data:: opmap - - Dictionary mapping operation names to bytecodes. - - -.. data:: cmp_op - - Sequence of all compare operation names. - - -.. data:: hasconst - - Sequence of bytecodes that access a constant. - - -.. data:: hasfree - - Sequence of bytecodes that access a free variable (note that 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes). - - -.. data:: hasname - - Sequence of bytecodes that access an attribute by name. - - -.. data:: hasjrel - - Sequence of bytecodes that have a relative jump target. - - -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. - - -.. data:: haslocal - - Sequence of bytecodes that access a local variable. - - -.. data:: hascompare - - Sequence of bytecodes of Boolean operations. diff --git a/third_party/python/Doc/library/distribution.rst b/third_party/python/Doc/library/distribution.rst deleted file mode 100644 index 8d4befe41..000000000 --- a/third_party/python/Doc/library/distribution.rst +++ /dev/null @@ -1,15 +0,0 @@ -*********************************** -Software Packaging and Distribution -*********************************** - -These libraries help you with publishing and installing Python software. -While these modules are designed to work in conjunction with the -`Python Package Index `__, they can also be used -with a local index server, or without any index server at all. - -.. toctree:: - - distutils.rst - ensurepip.rst - venv.rst - zipapp.rst diff --git a/third_party/python/Doc/library/distutils.rst b/third_party/python/Doc/library/distutils.rst deleted file mode 100644 index 62abc85ac..000000000 --- a/third_party/python/Doc/library/distutils.rst +++ /dev/null @@ -1,44 +0,0 @@ -:mod:`distutils` --- Building and installing Python modules -=========================================================== - -.. module:: distutils - :synopsis: Support for building and installing Python modules into an - existing Python installation. - -.. sectionauthor:: Fred L. Drake, Jr. - --------------- - -The :mod:`distutils` package provides support for building and installing -additional modules into a Python installation. The new modules may be either -100%-pure Python, or may be extension modules written in C, or may be -collections of Python packages which include modules coded in both Python and C. - -Most Python users will *not* want to use this module directly, but instead -use the cross-version tools maintained by the Python Packaging Authority. In -particular, -`setuptools `__ is an -enhanced alternative to :mod:`distutils` that provides: - -* support for declaring project dependencies -* additional mechanisms for configuring which files to include in source - releases (including plugins for integration with version control systems) -* the ability to declare project "entry points", which can be used as the - basis for application plugin systems -* the ability to automatically generate Windows command line executables at - installation time rather than needing to prebuild them -* consistent behaviour across all supported Python versions - -The recommended `pip `__ installer runs all -``setup.py`` scripts with ``setuptools``, even if the script itself only -imports ``distutils``. Refer to the -`Python Packaging User Guide `_ for more -information. - -For the benefits of packaging tool authors and users seeking a deeper -understanding of the details of the current packaging and distribution -system, the legacy :mod:`distutils` based user documentation and API -reference remain available: - -* :ref:`install-index` -* :ref:`distutils-index` diff --git a/third_party/python/Doc/library/doctest.rst b/third_party/python/Doc/library/doctest.rst deleted file mode 100644 index a138e6874..000000000 --- a/third_party/python/Doc/library/doctest.rst +++ /dev/null @@ -1,1864 +0,0 @@ -:keepdoctest: - -:mod:`doctest` --- Test interactive Python examples -=================================================== - -.. module:: doctest - :synopsis: Test pieces of code within docstrings. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Edward Loper - -**Source code:** :source:`Lib/doctest.py` - --------------- - -The :mod:`doctest` module searches for pieces of text that look like interactive -Python sessions, and then executes those sessions to verify that they work -exactly as shown. There are several common ways to use doctest: - -* To check that a module's docstrings are up-to-date by verifying that all - interactive examples still work as documented. - -* To perform regression testing by verifying that interactive examples from a - test file or a test object work as expected. - -* To write tutorial documentation for a package, liberally illustrated with - input-output examples. Depending on whether the examples or the expository text - are emphasized, this has the flavor of "literate testing" or "executable - documentation". - -Here's a complete but small example module:: - - """ - This is the "example" module. - - The example module supplies one function, factorial(). For example, - - >>> factorial(5) - 120 - """ - - def factorial(n): - """Return the factorial of n, an exact integer >= 0. - - >>> [factorial(n) for n in range(6)] - [1, 1, 2, 6, 24, 120] - >>> factorial(30) - 265252859812191058636308480000000 - >>> factorial(-1) - Traceback (most recent call last): - ... - ValueError: n must be >= 0 - - Factorials of floats are OK, but the float must be an exact integer: - >>> factorial(30.1) - Traceback (most recent call last): - ... - ValueError: n must be exact integer - >>> factorial(30.0) - 265252859812191058636308480000000 - - It must also not be ridiculously large: - >>> factorial(1e100) - Traceback (most recent call last): - ... - OverflowError: n too large - """ - - import math - if not n >= 0: - raise ValueError("n must be >= 0") - if math.floor(n) != n: - raise ValueError("n must be exact integer") - if n+1 == n: # catch a value like 1e300 - raise OverflowError("n too large") - result = 1 - factor = 2 - while factor <= n: - result *= factor - factor += 1 - return result - - - if __name__ == "__main__": - import doctest - doctest.testmod() - -If you run :file:`example.py` directly from the command line, :mod:`doctest` -works its magic: - -.. code-block:: shell-session - - $ python example.py - $ - -There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what -it's trying, and prints a summary at the end: - -.. code-block:: shell-session - - $ python example.py -v - Trying: - factorial(5) - Expecting: - 120 - ok - Trying: - [factorial(n) for n in range(6)] - Expecting: - [1, 1, 2, 6, 24, 120] - ok - -And so on, eventually ending with: - -.. code-block:: none - - Trying: - factorial(1e100) - Expecting: - Traceback (most recent call last): - ... - OverflowError: n too large - ok - 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. - Test passed. - $ - -That's all you need to know to start making productive use of :mod:`doctest`! -Jump in. The following sections provide full details. Note that there are many -examples of doctests in the standard Python test suite and libraries. -Especially useful examples can be found in the standard test file -:file:`Lib/test/test_doctest.py`. - - -.. _doctest-simple-testmod: - -Simple Usage: Checking Examples in Docstrings ---------------------------------------------- - -The simplest way to start using doctest (but not necessarily the way you'll -continue to do it) is to end each module :mod:`M` with:: - - if __name__ == "__main__": - import doctest - doctest.testmod() - -:mod:`doctest` then examines docstrings in module :mod:`M`. - -Running the module as a script causes the examples in the docstrings to get -executed and verified:: - - python M.py - -This won't display anything unless an example fails, in which case the failing -example(s) and the cause(s) of the failure(s) are printed to stdout, and the -final line of output is ``***Test Failed*** N failures.``, where *N* is the -number of examples that failed. - -Run it with the ``-v`` switch instead:: - - python M.py -v - -and a detailed report of all examples tried is printed to standard output, along -with assorted summaries at the end. - -You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or -prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not -has no effect). - -There is also a command line shortcut for running :func:`testmod`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the module name(s) on the command line:: - - python -m doctest -v example.py - -This will import :file:`example.py` as a standalone module and run -:func:`testmod` on it. Note that this may not work correctly if the file is -part of a package and imports other submodules from that package. - -For more information on :func:`testmod`, see section :ref:`doctest-basic-api`. - - -.. _doctest-simple-testfile: - -Simple Usage: Checking Examples in a Text File ----------------------------------------------- - -Another simple application of doctest is testing interactive examples in a text -file. This can be done with the :func:`testfile` function:: - - import doctest - doctest.testfile("example.txt") - -That short script executes and verifies any interactive Python examples -contained in the file :file:`example.txt`. The file content is treated as if it -were a single giant docstring; the file doesn't need to contain a Python -program! For example, perhaps :file:`example.txt` contains this: - -.. code-block:: none - - The ``example`` module - ====================== - - Using ``factorial`` - ------------------- - - This is an example text file in reStructuredText format. First import - ``factorial`` from the ``example`` module: - - >>> from example import factorial - - Now use it: - - >>> factorial(6) - 120 - -Running ``doctest.testfile("example.txt")`` then finds the error in this -documentation:: - - File "./example.txt", line 14, in example.txt - Failed example: - factorial(6) - Expected: - 120 - Got: - 720 - -As with :func:`testmod`, :func:`testfile` won't display anything unless an -example fails. If an example does fail, then the failing example(s) and the -cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. - -By default, :func:`testfile` looks for files in the calling module's directory. -See section :ref:`doctest-basic-api` for a description of the optional arguments -that can be used to tell it to look for files in other locations. - -Like :func:`testmod`, :func:`testfile`'s verbosity can be set with the -``-v`` command-line switch or with the optional keyword argument -*verbose*. - -There is also a command line shortcut for running :func:`testfile`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the file name(s) on the command line:: - - python -m doctest -v example.txt - -Because the file name does not end with :file:`.py`, :mod:`doctest` infers that -it must be run with :func:`testfile`, not :func:`testmod`. - -For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. - - -.. _doctest-how-it-works: - -How It Works ------------- - -This section examines in detail how doctest works: which docstrings it looks at, -how it finds interactive examples, what execution context it uses, how it -handles exceptions, and how option flags can be used to control its behavior. -This is the information that you need to know to write doctest examples; for -information about actually running doctest on these examples, see the following -sections. - - -.. _doctest-which-docstrings: - -Which Docstrings Are Examined? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The module docstring, and all function, class and method docstrings are -searched. Objects imported into the module are not searched. - -In addition, if ``M.__test__`` exists and "is true", it must be a dict, and each -entry maps a (string) name to a function object, class object, or string. -Function and class object docstrings found from ``M.__test__`` are searched, and -strings are treated as if they were docstrings. In output, a key ``K`` in -``M.__test__`` appears with name :: - - .__test__.K - -Any classes found are recursively searched similarly, to test docstrings in -their contained methods and nested classes. - -.. impl-detail:: - Prior to version 3.4, extension modules written in C were not fully - searched by doctest. - - -.. _doctest-finding-examples: - -How are Docstring Examples Recognized? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In most cases a copy-and-paste of an interactive console session works fine, -but doctest isn't trying to do an exact emulation of any specific Python shell. - -:: - - >>> # comments are ignored - >>> x = 12 - >>> x - 12 - >>> if x == 13: - ... print("yes") - ... else: - ... print("no") - ... print("NO") - ... print("NO!!!") - ... - no - NO - NO!!! - >>> - -.. index:: - single: >>>; interpreter prompt - single: ...; interpreter prompt - -Any expected output must immediately follow the final ``'>>> '`` or ``'... '`` -line containing the code, and the expected output (if any) extends to the next -``'>>> '`` or all-whitespace line. - -The fine print: - -* Expected output cannot contain an all-whitespace line, since such a line is - taken to signal the end of expected output. If expected output does contain a - blank line, put ```` in your doctest example each place a blank line - is expected. - -* All hard tab characters are expanded to spaces, using 8-column tab stops. - Tabs in output generated by the tested code are not modified. Because any - hard tabs in the sample output *are* expanded, this means that if the code - output includes hard tabs, the only way the doctest can pass is if the - :const:`NORMALIZE_WHITESPACE` option or :ref:`directive ` - is in effect. - Alternatively, the test can be rewritten to capture the output and compare it - to an expected value as part of the test. This handling of tabs in the - source was arrived at through trial and error, and has proven to be the least - error prone way of handling them. It is possible to use a different - algorithm for handling tabs by writing a custom :class:`DocTestParser` class. - -* Output to stdout is captured, but not output to stderr (exception tracebacks - are captured via a different means). - -* If you continue a line via backslashing in an interactive session, or for any - other reason use a backslash, you should use a raw docstring, which will - preserve your backslashes exactly as you type them:: - - >>> def f(x): - ... r'''Backslashes in a raw docstring: m\n''' - >>> print(f.__doc__) - Backslashes in a raw docstring: m\n - - Otherwise, the backslash will be interpreted as part of the string. For example, - the ``\n`` above would be interpreted as a newline character. Alternatively, you - can double each backslash in the doctest version (and not use a raw string):: - - >>> def f(x): - ... '''Backslashes in a raw docstring: m\\n''' - >>> print(f.__doc__) - Backslashes in a raw docstring: m\n - -* The starting column doesn't matter:: - - >>> assert "Easy!" - >>> import math - >>> math.floor(1.9) - 1 - - and as many leading whitespace characters are stripped from the expected output - as appeared in the initial ``'>>> '`` line that started the example. - - -.. _doctest-execution-context: - -What's the Execution Context? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, each time :mod:`doctest` finds a docstring to test, it uses a -*shallow copy* of :mod:`M`'s globals, so that running tests doesn't change the -module's real globals, and so that one test in :mod:`M` can't leave behind -crumbs that accidentally allow another test to work. This means examples can -freely use any names defined at top-level in :mod:`M`, and names defined earlier -in the docstring being run. Examples cannot see names defined in other -docstrings. - -You can force use of your own dict as the execution context by passing -``globs=your_dict`` to :func:`testmod` or :func:`testfile` instead. - - -.. _doctest-exceptions: - -What About Exceptions? -^^^^^^^^^^^^^^^^^^^^^^ - -No problem, provided that the traceback is the only output produced by the -example: just paste in the traceback. [#]_ Since tracebacks contain details -that are likely to change rapidly (for example, exact file paths and line -numbers), this is one case where doctest works hard to be flexible in what it -accepts. - -Simple example:: - - >>> [1, 2, 3].remove(42) - Traceback (most recent call last): - File "", line 1, in - ValueError: list.remove(x): x not in list - -That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): -x not in list`` detail as shown. - -The expected output for an exception must start with a traceback header, which -may be either of the following two lines, indented the same as the first line of -the example:: - - Traceback (most recent call last): - Traceback (innermost last): - -The traceback header is followed by an optional traceback stack, whose contents -are ignored by doctest. The traceback stack is typically omitted, or copied -verbatim from an interactive session. - -The traceback stack is followed by the most interesting part: the line(s) -containing the exception type and detail. This is usually the last line of a -traceback, but can extend across multiple lines if the exception has a -multi-line detail:: - - >>> raise ValueError('multi\n line\ndetail') - Traceback (most recent call last): - File "", line 1, in - ValueError: multi - line - detail - -The last three lines (starting with :exc:`ValueError`) are compared against the -exception's type and detail, and the rest are ignored. - -Best practice is to omit the traceback stack, unless it adds significant -documentation value to the example. So the last example is probably better as:: - - >>> raise ValueError('multi\n line\ndetail') - Traceback (most recent call last): - ... - ValueError: multi - line - detail - -Note that tracebacks are treated very specially. In particular, in the -rewritten example, the use of ``...`` is independent of doctest's -:const:`ELLIPSIS` option. The ellipsis in that example could be left out, or -could just as well be three (or three hundred) commas or digits, or an indented -transcript of a Monty Python skit. - -Some details you should read once, but won't need to remember: - -* Doctest can't guess whether your expected output came from an exception - traceback or from ordinary printing. So, e.g., an example that expects - ``ValueError: 42 is prime`` will pass whether :exc:`ValueError` is actually - raised or if the example merely prints that traceback text. In practice, - ordinary output rarely begins with a traceback header line, so this doesn't - create real problems. - -* Each line of the traceback stack (if present) must be indented further than - the first line of the example, *or* start with a non-alphanumeric character. - The first line following the traceback header indented the same and starting - with an alphanumeric is taken to be the start of the exception detail. Of - course this does the right thing for genuine tracebacks. - -* When the :const:`IGNORE_EXCEPTION_DETAIL` doctest option is specified, - everything following the leftmost colon and any module information in the - exception name is ignored. - -* The interactive shell omits the traceback header line for some - :exc:`SyntaxError`\ s. But doctest uses the traceback header line to - distinguish exceptions from non-exceptions. So in the rare case where you need - to test a :exc:`SyntaxError` that omits the traceback header, you will need to - manually add the traceback header line to your test example. - -.. index:: single: ^ (caret); marker - -* For some :exc:`SyntaxError`\ s, Python displays the character position of the - syntax error, using a ``^`` marker:: - - >>> 1 1 - File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax - - Since the lines showing the position of the error come before the exception type - and detail, they are not checked by doctest. For example, the following test - would pass, even though it puts the ``^`` marker in the wrong location:: - - >>> 1 1 - Traceback (most recent call last): - File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax - - -.. _option-flags-and-directives: -.. _doctest-options: - -Option Flags -^^^^^^^^^^^^ - -A number of option flags control various aspects of doctest's behavior. -Symbolic names for the flags are supplied as module constants, which can be -:ref:`bitwise ORed ` together and passed to various functions. -The names can also be used in :ref:`doctest directives `, -and may be passed to the doctest command line interface via the ``-o`` option. - -.. versionadded:: 3.4 - The ``-o`` command line option. - -The first group of options define test semantics, controlling aspects of how -doctest decides whether actual output matches an example's expected output: - - -.. data:: DONT_ACCEPT_TRUE_FOR_1 - - By default, if an expected output block contains just ``1``, an actual output - block containing just ``1`` or just ``True`` is considered to be a match, and - similarly for ``0`` versus ``False``. When :const:`DONT_ACCEPT_TRUE_FOR_1` is - specified, neither substitution is allowed. The default behavior caters to that - Python changed the return type of many functions from integer to boolean; - doctests expecting "little integer" output still work in these cases. This - option will probably go away, but not for several years. - - -.. index:: single: -.. data:: DONT_ACCEPT_BLANKLINE - - By default, if an expected output block contains a line containing only the - string ````, then that line will match a blank line in the actual - output. Because a genuinely blank line delimits the expected output, this is - the only way to communicate that a blank line is expected. When - :const:`DONT_ACCEPT_BLANKLINE` is specified, this substitution is not allowed. - - -.. data:: NORMALIZE_WHITESPACE - - When specified, all sequences of whitespace (blanks and newlines) are treated as - equal. Any sequence of whitespace within the expected output will match any - sequence of whitespace within the actual output. By default, whitespace must - match exactly. :const:`NORMALIZE_WHITESPACE` is especially useful when a line of - expected output is very long, and you want to wrap it across multiple lines in - your source. - - -.. index:: single: ...; in doctests -.. data:: ELLIPSIS - - When specified, an ellipsis marker (``...``) in the expected output can match - any substring in the actual output. This includes substrings that span line - boundaries, and empty substrings, so it's best to keep usage of this simple. - Complicated uses can lead to the same kinds of "oops, it matched too much!" - surprises that ``.*`` is prone to in regular expressions. - - -.. data:: IGNORE_EXCEPTION_DETAIL - - When specified, an example that expects an exception passes if an exception of - the expected type is raised, even if the exception detail does not match. For - example, an example expecting ``ValueError: 42`` will pass if the actual - exception raised is ``ValueError: 3*14``, but will fail, e.g., if - :exc:`TypeError` is raised. - - It will also ignore the module name used in Python 3 doctest reports. Hence - both of these variations will work with the flag specified, regardless of - whether the test is run under Python 2.7 or Python 3.2 (or later versions):: - - >>> raise CustomError('message') - Traceback (most recent call last): - CustomError: message - - >>> raise CustomError('message') - Traceback (most recent call last): - my_module.CustomError: message - - Note that :const:`ELLIPSIS` can also be used to ignore the - details of the exception message, but such a test may still fail based - on whether or not the module details are printed as part of the - exception name. Using :const:`IGNORE_EXCEPTION_DETAIL` and the details - from Python 2.3 is also the only clear way to write a doctest that doesn't - care about the exception detail yet continues to pass under Python 2.3 or - earlier (those releases do not support :ref:`doctest directives - ` and ignore them as irrelevant comments). For example:: - - >>> (1, 2)[3] = 'moo' - Traceback (most recent call last): - File "", line 1, in - TypeError: object doesn't support item assignment - - passes under Python 2.3 and later Python versions with the flag specified, - even though the detail - changed in Python 2.4 to say "does not" instead of "doesn't". - - .. versionchanged:: 3.2 - :const:`IGNORE_EXCEPTION_DETAIL` now also ignores any information relating - to the module containing the exception under test. - - -.. data:: SKIP - - When specified, do not run the example at all. This can be useful in contexts - where doctest examples serve as both documentation and test cases, and an - example should be included for documentation purposes, but should not be - checked. E.g., the example's output might be random; or the example might - depend on resources which would be unavailable to the test driver. - - The SKIP flag can also be used for temporarily "commenting out" examples. - - -.. data:: COMPARISON_FLAGS - - A bitmask or'ing together all the comparison flags above. - -The second group of options controls how test failures are reported: - - -.. data:: REPORT_UDIFF - - When specified, failures that involve multi-line expected and actual outputs are - displayed using a unified diff. - - -.. data:: REPORT_CDIFF - - When specified, failures that involve multi-line expected and actual outputs - will be displayed using a context diff. - - -.. data:: REPORT_NDIFF - - When specified, differences are computed by ``difflib.Differ``, using the same - algorithm as the popular :file:`ndiff.py` utility. This is the only method that - marks differences within lines as well as across lines. For example, if a line - of expected output contains digit ``1`` where actual output contains letter - ``l``, a line is inserted with a caret marking the mismatching column positions. - - -.. data:: REPORT_ONLY_FIRST_FAILURE - - When specified, display the first failing example in each doctest, but suppress - output for all remaining examples. This will prevent doctest from reporting - correct examples that break because of earlier failures; but it might also hide - incorrect examples that fail independently of the first failure. When - :const:`REPORT_ONLY_FIRST_FAILURE` is specified, the remaining examples are - still run, and still count towards the total number of failures reported; only - the output is suppressed. - - -.. data:: FAIL_FAST - - When specified, exit after the first failing example and don't attempt to run - the remaining examples. Thus, the number of failures reported will be at most - 1. This flag may be useful during debugging, since examples after the first - failure won't even produce debugging output. - - The doctest command line accepts the option ``-f`` as a shorthand for ``-o - FAIL_FAST``. - - .. versionadded:: 3.4 - - -.. data:: REPORTING_FLAGS - - A bitmask or'ing together all the reporting flags above. - - -There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: - - -.. function:: register_optionflag(name) - - Create a new option flag with a given name, and return the new flag's integer - value. :func:`register_optionflag` can be used when subclassing - :class:`OutputChecker` or :class:`DocTestRunner` to create new options that are - supported by your subclasses. :func:`register_optionflag` should always be - called using the following idiom:: - - MY_FLAG = register_optionflag('MY_FLAG') - - -.. index:: - single: # (hash); in doctests - single: + (plus); in doctests - single: - (minus); in doctests -.. _doctest-directives: - -Directives -^^^^^^^^^^ - -Doctest directives may be used to modify the :ref:`option flags -` for an individual example. Doctest directives are -special Python comments following an example's source code: - -.. productionlist:: doctest - directive: "#" "doctest:" `directive_options` - directive_options: `directive_option` ("," `directive_option`)\* - directive_option: `on_or_off` `directive_option_name` - on_or_off: "+" \| "-" - directive_option_name: "DONT_ACCEPT_BLANKLINE" \| "NORMALIZE_WHITESPACE" \| ... - -Whitespace is not allowed between the ``+`` or ``-`` and the directive option -name. The directive option name can be any of the option flag names explained -above. - -An example's doctest directives modify doctest's behavior for that single -example. Use ``+`` to enable the named behavior, or ``-`` to disable it. - -For example, this test passes:: - - >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] - -Without the directive it would fail, both because the actual output doesn't have -two blanks before the single-digit list elements, and because the actual output -is on a single line. This test also passes, and also requires a directive to do -so:: - - >>> print(list(range(20))) # doctest: +ELLIPSIS - [0, 1, ..., 18, 19] - -Multiple directives can be used on a single physical line, separated by -commas:: - - >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - [0, 1, ..., 18, 19] - -If multiple directive comments are used for a single example, then they are -combined:: - - >>> print(list(range(20))) # doctest: +ELLIPSIS - ... # doctest: +NORMALIZE_WHITESPACE - [0, 1, ..., 18, 19] - -As the previous example shows, you can add ``...`` lines to your example -containing only directives. This can be useful when an example is too long for -a directive to comfortably fit on the same line:: - - >>> print(list(range(5)) + list(range(10, 20)) + list(range(30, 40))) - ... # doctest: +ELLIPSIS - [0, ..., 4, 10, ..., 19, 30, ..., 39] - -Note that since all options are disabled by default, and directives apply only -to the example they appear in, enabling options (via ``+`` in a directive) is -usually the only meaningful choice. However, option flags can also be passed to -functions that run doctests, establishing different defaults. In such cases, -disabling an option via ``-`` in a directive can be useful. - - -.. _doctest-warnings: - -Warnings -^^^^^^^^ - -:mod:`doctest` is serious about requiring exact matches in expected output. If -even a single character doesn't match, the test fails. This will probably -surprise you a few times, as you learn exactly what Python does and doesn't -guarantee about output. For example, when printing a dict, Python doesn't -guarantee that the key-value pairs will be printed in any particular order, so a -test like :: - - >>> foo() - {"Hermione": "hippogryph", "Harry": "broomstick"} - -is vulnerable! One workaround is to do :: - - >>> foo() == {"Hermione": "hippogryph", "Harry": "broomstick"} - True - -instead. Another is to do :: - - >>> d = sorted(foo().items()) - >>> d - [('Harry', 'broomstick'), ('Hermione', 'hippogryph')] - -There are others, but you get the idea. - -Another bad idea is to print things that embed an object address, like :: - - >>> id(1.0) # certain to fail some of the time - 7948648 - >>> class C: pass - >>> C() # the default repr() for instances embeds an address - <__main__.C instance at 0x00AC18F0> - -The :const:`ELLIPSIS` directive gives a nice approach for the last example:: - - >>> C() #doctest: +ELLIPSIS - <__main__.C instance at 0x...> - -Floating-point numbers are also subject to small output variations across -platforms, because Python defers to the platform C library for float formatting, -and C libraries vary widely in quality here. :: - - >>> 1./7 # risky - 0.14285714285714285 - >>> print(1./7) # safer - 0.142857142857 - >>> print(round(1./7, 6)) # much safer - 0.142857 - -Numbers of the form ``I/2.**J`` are safe across all platforms, and I often -contrive doctest examples to produce numbers of that form:: - - >>> 3./4 # utterly safe - 0.75 - -Simple fractions are also easier for people to understand, and that makes for -better documentation. - - -.. _doctest-basic-api: - -Basic API ---------- - -The functions :func:`testmod` and :func:`testfile` provide a simple interface to -doctest that should be sufficient for most basic uses. For a less formal -introduction to these two functions, see sections :ref:`doctest-simple-testmod` -and :ref:`doctest-simple-testfile`. - - -.. function:: testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, parser=DocTestParser(), encoding=None) - - All arguments except *filename* are optional, and should be specified in keyword - form. - - Test examples in the file named *filename*. Return ``(failure_count, - test_count)``. - - Optional argument *module_relative* specifies how the filename should be - interpreted: - - * If *module_relative* is ``True`` (the default), then *filename* specifies an - OS-independent module-relative path. By default, this path is relative to the - calling module's directory; but if the *package* argument is specified, then it - is relative to that package. To ensure OS-independence, *filename* should use - ``/`` characters to separate path segments, and may not be an absolute path - (i.e., it may not begin with ``/``). - - * If *module_relative* is ``False``, then *filename* specifies an OS-specific - path. The path may be absolute or relative; relative paths are resolved with - respect to the current working directory. - - Optional argument *name* gives the name of the test; by default, or if ``None``, - ``os.path.basename(filename)`` is used. - - Optional argument *package* is a Python package or the name of a Python package - whose directory should be used as the base directory for a module-relative - filename. If no package is specified, then the calling module's directory is - used as the base directory for module-relative filenames. It is an error to - specify *package* if *module_relative* is ``False``. - - Optional argument *globs* gives a dict to be used as the globals when executing - examples. A new shallow copy of this dict is created for the doctest, so its - examples start with a clean slate. By default, or if ``None``, a new empty dict - is used. - - Optional argument *extraglobs* gives a dict merged into the globals used to - execute examples. This works like :meth:`dict.update`: if *globs* and - *extraglobs* have a common key, the associated value in *extraglobs* appears in - the combined dict. By default, or if ``None``, no extra globals are used. This - is an advanced feature that allows parameterization of doctests. For example, a - doctest can be written for a base class, using a generic name for the class, - then reused to test any number of subclasses by passing an *extraglobs* dict - mapping the generic name to the subclass to be tested. - - Optional argument *verbose* prints lots of stuff if true, and prints only - failures if false; by default, or if ``None``, it's true if and only if ``'-v'`` - is in ``sys.argv``. - - Optional argument *report* prints a summary at the end when true, else prints - nothing at the end. In verbose mode, the summary is detailed, else the summary - is very brief (in fact, empty if all tests passed). - - Optional argument *optionflags* (default value 0) takes the - :ref:`bitwise OR ` of option flags. - See section :ref:`doctest-options`. - - Optional argument *raise_on_error* defaults to false. If true, an exception is - raised upon the first failure or unexpected exception in an example. This - allows failures to be post-mortem debugged. Default behavior is to continue - running examples. - - Optional argument *parser* specifies a :class:`DocTestParser` (or subclass) that - should be used to extract tests from the files. It defaults to a normal parser - (i.e., ``DocTestParser()``). - - Optional argument *encoding* specifies an encoding that should be used to - convert the file to unicode. - - -.. function:: testmod(m=None, name=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, exclude_empty=False) - - All arguments are optional, and all except for *m* should be specified in - keyword form. - - Test examples in docstrings in functions and classes reachable from module *m* - (or module :mod:`__main__` if *m* is not supplied or is ``None``), starting with - ``m.__doc__``. - - Also test examples reachable from dict ``m.__test__``, if it exists and is not - ``None``. ``m.__test__`` maps names (strings) to functions, classes and - strings; function and class docstrings are searched for examples; strings are - searched directly, as if they were docstrings. - - Only docstrings attached to objects belonging to module *m* are searched. - - Return ``(failure_count, test_count)``. - - Optional argument *name* gives the name of the module; by default, or if - ``None``, ``m.__name__`` is used. - - Optional argument *exclude_empty* defaults to false. If true, objects for which - no doctests are found are excluded from consideration. The default is a backward - compatibility hack, so that code still using :meth:`doctest.master.summarize` in - conjunction with :func:`testmod` continues to get output for objects with no - tests. The *exclude_empty* argument to the newer :class:`DocTestFinder` - constructor defaults to true. - - Optional arguments *extraglobs*, *verbose*, *report*, *optionflags*, - *raise_on_error*, and *globs* are the same as for function :func:`testfile` - above, except that *globs* defaults to ``m.__dict__``. - - -.. function:: run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0) - - Test examples associated with object *f*; for example, *f* may be a string, - a module, a function, or a class object. - - A shallow copy of dictionary argument *globs* is used for the execution context. - - Optional argument *name* is used in failure messages, and defaults to - ``"NoName"``. - - If optional argument *verbose* is true, output is generated even if there are no - failures. By default, output is generated only in case of an example failure. - - Optional argument *compileflags* gives the set of flags that should be used by - the Python compiler when running the examples. By default, or if ``None``, - flags are deduced corresponding to the set of future features found in *globs*. - - Optional argument *optionflags* works as for function :func:`testfile` above. - - -.. _doctest-unittest-api: - -Unittest API ------------- - -As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can -be used to create :mod:`unittest` test suites from modules and text files -containing doctests. To integrate with :mod:`unittest` test discovery, include -a :func:`load_tests` function in your test module:: - - import unittest - import doctest - import my_module_with_doctests - - def load_tests(loader, tests, ignore): - tests.addTests(doctest.DocTestSuite(my_module_with_doctests)) - return tests - -There are two main functions for creating :class:`unittest.TestSuite` instances -from text files and modules with doctests: - - -.. function:: DocFileSuite(*paths, module_relative=True, package=None, setUp=None, tearDown=None, globs=None, optionflags=0, parser=DocTestParser(), encoding=None) - - Convert doctest tests from one or more text files to a - :class:`unittest.TestSuite`. - - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`failureException` - exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. - - Pass one or more paths (as strings) to text files to be examined. - - Options may be provided as keyword arguments: - - Optional argument *module_relative* specifies how the filenames in *paths* - should be interpreted: - - * If *module_relative* is ``True`` (the default), then each filename in - *paths* specifies an OS-independent module-relative path. By default, this - path is relative to the calling module's directory; but if the *package* - argument is specified, then it is relative to that package. To ensure - OS-independence, each filename should use ``/`` characters to separate path - segments, and may not be an absolute path (i.e., it may not begin with - ``/``). - - * If *module_relative* is ``False``, then each filename in *paths* specifies - an OS-specific path. The path may be absolute or relative; relative paths - are resolved with respect to the current working directory. - - Optional argument *package* is a Python package or the name of a Python - package whose directory should be used as the base directory for - module-relative filenames in *paths*. If no package is specified, then the - calling module's directory is used as the base directory for module-relative - filenames. It is an error to specify *package* if *module_relative* is - ``False``. - - Optional argument *setUp* specifies a set-up function for the test suite. - This is called before running the tests in each file. The *setUp* function - will be passed a :class:`DocTest` object. The setUp function can access the - test globals as the *globs* attribute of the test passed. - - Optional argument *tearDown* specifies a tear-down function for the test - suite. This is called after running the tests in each file. The *tearDown* - function will be passed a :class:`DocTest` object. The setUp function can - access the test globals as the *globs* attribute of the test passed. - - Optional argument *globs* is a dictionary containing the initial global - variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. - - Optional argument *optionflags* specifies the default doctest options for the - tests, created by or-ing together individual option flags. See section - :ref:`doctest-options`. See function :func:`set_unittest_reportflags` below - for a better way to set reporting options. - - Optional argument *parser* specifies a :class:`DocTestParser` (or subclass) - that should be used to extract tests from the files. It defaults to a normal - parser (i.e., ``DocTestParser()``). - - Optional argument *encoding* specifies an encoding that should be used to - convert the file to unicode. - - The global ``__file__`` is added to the globals provided to doctests loaded - from a text file using :func:`DocFileSuite`. - - -.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, checker=None) - - Convert doctest tests for a module to a :class:`unittest.TestSuite`. - - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. If any of the doctests fail, then the - synthesized unit test fails, and a :exc:`failureException` exception is raised - showing the name of the file containing the test and a (sometimes approximate) - line number. - - Optional argument *module* provides the module to be tested. It can be a module - object or a (possibly dotted) module name. If not specified, the module calling - this function is used. - - Optional argument *globs* is a dictionary containing the initial global - variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. - - Optional argument *extraglobs* specifies an extra set of global variables, which - is merged into *globs*. By default, no extra globals are used. - - Optional argument *test_finder* is the :class:`DocTestFinder` object (or a - drop-in replacement) that is used to extract doctests from the module. - - Optional arguments *setUp*, *tearDown*, and *optionflags* are the same as for - function :func:`DocFileSuite` above. - - This function uses the same search technique as :func:`testmod`. - - .. versionchanged:: 3.5 - :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* - contains no docstrings instead of raising :exc:`ValueError`. - - -Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out -of :class:`doctest.DocTestCase` instances, and :class:`DocTestCase` is a -subclass of :class:`unittest.TestCase`. :class:`DocTestCase` isn't documented -here (it's an internal detail), but studying its code can answer questions about -the exact details of :mod:`unittest` integration. - -Similarly, :func:`DocFileSuite` creates a :class:`unittest.TestSuite` out of -:class:`doctest.DocFileCase` instances, and :class:`DocFileCase` is a subclass -of :class:`DocTestCase`. - -So both ways of creating a :class:`unittest.TestSuite` run instances of -:class:`DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`doctest` options in -use directly, by passing option flags to :mod:`doctest` functions. However, if -you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls -when and how tests get run. The framework author typically wants to control -:mod:`doctest` reporting options (perhaps, e.g., specified by command line -options), but there's no way to pass options through :mod:`unittest` to -:mod:`doctest` test runners. - -For this reason, :mod:`doctest` also supports a notion of :mod:`doctest` -reporting flags specific to :mod:`unittest` support, via this function: - - -.. function:: set_unittest_reportflags(flags) - - Set the :mod:`doctest` reporting flags to use. - - Argument *flags* takes the :ref:`bitwise OR ` of option flags. See - section :ref:`doctest-options`. Only "reporting flags" can be used. - - This is a module-global setting, and affects all future doctests run by module - :mod:`unittest`: the :meth:`runTest` method of :class:`DocTestCase` looks at - the option flags specified for the test case when the :class:`DocTestCase` - instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`doctest`'s :mod:`unittest` reporting flags are - :ref:`bitwise ORed ` into the option flags, and the option flags - so augmented are passed to the :class:`DocTestRunner` instance created to - run the doctest. If any reporting flags were specified when the - :class:`DocTestCase` instance was constructed, :mod:`doctest`'s - :mod:`unittest` reporting flags are ignored. - - The value of the :mod:`unittest` reporting flags in effect before the function - was called is returned by the function. - - -.. _doctest-advanced-api: - -Advanced API ------------- - -The basic API is a simple wrapper that's intended to make doctest easy to use. -It is fairly flexible, and should meet most users' needs; however, if you -require more fine-grained control over testing, or wish to extend doctest's -capabilities, then you should use the advanced API. - -The advanced API revolves around two container classes, which are used to store -the interactive examples extracted from doctest cases: - -* :class:`Example`: A single Python :term:`statement`, paired with its expected - output. - -* :class:`DocTest`: A collection of :class:`Example`\ s, typically extracted - from a single docstring or text file. - -Additional processing classes are defined to find, parse, and run, and check -doctest examples: - -* :class:`DocTestFinder`: Finds all docstrings in a given module, and uses a - :class:`DocTestParser` to create a :class:`DocTest` from every docstring that - contains interactive examples. - -* :class:`DocTestParser`: Creates a :class:`DocTest` object from a string (such - as an object's docstring). - -* :class:`DocTestRunner`: Executes the examples in a :class:`DocTest`, and uses - an :class:`OutputChecker` to verify their output. - -* :class:`OutputChecker`: Compares the actual output from a doctest example with - the expected output, and decides whether they match. - -The relationships among these processing classes are summarized in the following -diagram:: - - list of: - +------+ +---------+ - |module| --DocTestFinder-> | DocTest | --DocTestRunner-> results - +------+ | ^ +---------+ | ^ (printed) - | | | Example | | | - v | | ... | v | - DocTestParser | Example | OutputChecker - +---------+ - - -.. _doctest-doctest: - -DocTest Objects -^^^^^^^^^^^^^^^ - - -.. class:: DocTest(examples, globs, name, filename, lineno, docstring) - - A collection of doctest examples that should be run in a single namespace. The - constructor arguments are used to initialize the attributes of the same names. - - - :class:`DocTest` defines the following attributes. They are initialized by - the constructor, and should not be modified directly. - - - .. attribute:: examples - - A list of :class:`Example` objects encoding the individual interactive Python - examples that should be run by this test. - - - .. attribute:: globs - - The namespace (aka globals) that the examples should be run in. This is a - dictionary mapping names to values. Any changes to the namespace made by the - examples (such as binding new variables) will be reflected in :attr:`globs` - after the test is run. - - - .. attribute:: name - - A string name identifying the :class:`DocTest`. Typically, this is the name - of the object or file that the test was extracted from. - - - .. attribute:: filename - - The name of the file that this :class:`DocTest` was extracted from; or - ``None`` if the filename is unknown, or if the :class:`DocTest` was not - extracted from a file. - - - .. attribute:: lineno - - The line number within :attr:`filename` where this :class:`DocTest` begins, or - ``None`` if the line number is unavailable. This line number is zero-based - with respect to the beginning of the file. - - - .. attribute:: docstring - - The string that the test was extracted from, or ``None`` if the string is - unavailable, or if the test was not extracted from a string. - - -.. _doctest-example: - -Example Objects -^^^^^^^^^^^^^^^ - - -.. class:: Example(source, want, exc_msg=None, lineno=0, indent=0, options=None) - - A single interactive example, consisting of a Python statement and its expected - output. The constructor arguments are used to initialize the attributes of - the same names. - - - :class:`Example` defines the following attributes. They are initialized by - the constructor, and should not be modified directly. - - - .. attribute:: source - - A string containing the example's source code. This source code consists of a - single Python statement, and always ends with a newline; the constructor adds - a newline when necessary. - - - .. attribute:: want - - The expected output from running the example's source code (either from - stdout, or a traceback in case of exception). :attr:`want` ends with a - newline unless no output is expected, in which case it's an empty string. The - constructor adds a newline when necessary. - - - .. attribute:: exc_msg - - The exception message generated by the example, if the example is expected to - generate an exception; or ``None`` if it is not expected to generate an - exception. This exception message is compared against the return value of - :func:`traceback.format_exception_only`. :attr:`exc_msg` ends with a newline - unless it's ``None``. The constructor adds a newline if needed. - - - .. attribute:: lineno - - The line number within the string containing this example where the example - begins. This line number is zero-based with respect to the beginning of the - containing string. - - - .. attribute:: indent - - The example's indentation in the containing string, i.e., the number of space - characters that precede the example's first prompt. - - - .. attribute:: options - - A dictionary mapping from option flags to ``True`` or ``False``, which is used - to override default options for this example. Any option flags not contained - in this dictionary are left at their default value (as specified by the - :class:`DocTestRunner`'s :attr:`optionflags`). By default, no options are set. - - -.. _doctest-doctestfinder: - -DocTestFinder objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestFinder(verbose=False, parser=DocTestParser(), recurse=True, exclude_empty=True) - - A processing class used to extract the :class:`DocTest`\ s that are relevant to - a given object, from its docstring and the docstrings of its contained objects. - :class:`DocTest`\ s can be extracted from modules, classes, functions, - methods, staticmethods, classmethods, and properties. - - The optional argument *verbose* can be used to display the objects searched by - the finder. It defaults to ``False`` (no output). - - The optional argument *parser* specifies the :class:`DocTestParser` object (or a - drop-in replacement) that is used to extract doctests from docstrings. - - If the optional argument *recurse* is false, then :meth:`DocTestFinder.find` - will only examine the given object, and not any contained objects. - - If the optional argument *exclude_empty* is false, then - :meth:`DocTestFinder.find` will include tests for objects with empty docstrings. - - - :class:`DocTestFinder` defines the following method: - - - .. method:: find(obj[, name][, module][, globs][, extraglobs]) - - Return a list of the :class:`DocTest`\ s that are defined by *obj*'s - docstring, or by any of its contained objects' docstrings. - - The optional argument *name* specifies the object's name; this name will be - used to construct names for the returned :class:`DocTest`\ s. If *name* is - not specified, then ``obj.__name__`` is used. - - The optional parameter *module* is the module that contains the given object. - If the module is not specified or is ``None``, then the test finder will attempt - to automatically determine the correct module. The object's module is used: - - * As a default namespace, if *globs* is not specified. - - * To prevent the DocTestFinder from extracting DocTests from objects that are - imported from other modules. (Contained objects with modules other than - *module* are ignored.) - - * To find the name of the file containing the object. - - * To help find the line number of the object within its file. - - If *module* is ``False``, no attempt to find the module will be made. This is - obscure, of use mostly in testing doctest itself: if *module* is ``False``, or - is ``None`` but cannot be found automatically, then all objects are considered - to belong to the (non-existent) module, so all contained objects will - (recursively) be searched for doctests. - - The globals for each :class:`DocTest` is formed by combining *globs* and - *extraglobs* (bindings in *extraglobs* override bindings in *globs*). A new - shallow copy of the globals dictionary is created for each :class:`DocTest`. - If *globs* is not specified, then it defaults to the module's *__dict__*, if - specified, or ``{}`` otherwise. If *extraglobs* is not specified, then it - defaults to ``{}``. - - -.. _doctest-doctestparser: - -DocTestParser objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestParser() - - A processing class used to extract interactive examples from a string, and use - them to create a :class:`DocTest` object. - - - :class:`DocTestParser` defines the following methods: - - - .. method:: get_doctest(string, globs, name, filename, lineno) - - Extract all doctest examples from the given string, and collect them into a - :class:`DocTest` object. - - *globs*, *name*, *filename*, and *lineno* are attributes for the new - :class:`DocTest` object. See the documentation for :class:`DocTest` for more - information. - - - .. method:: get_examples(string, name='') - - Extract all doctest examples from the given string, and return them as a list - of :class:`Example` objects. Line numbers are 0-based. The optional argument - *name* is a name identifying this string, and is only used for error messages. - - - .. method:: parse(string, name='') - - Divide the given string into examples and intervening text, and return them as - a list of alternating :class:`Example`\ s and strings. Line numbers for the - :class:`Example`\ s are 0-based. The optional argument *name* is a name - identifying this string, and is only used for error messages. - - -.. _doctest-doctestrunner: - -DocTestRunner objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestRunner(checker=None, verbose=None, optionflags=0) - - A processing class used to execute and verify the interactive examples in a - :class:`DocTest`. - - The comparison between expected outputs and actual outputs is done by an - :class:`OutputChecker`. This comparison may be customized with a number of - option flags; see section :ref:`doctest-options` for more information. If the - option flags are insufficient, then the comparison may also be customized by - passing a subclass of :class:`OutputChecker` to the constructor. - - The test runner's display output can be controlled in two ways. First, an output - function can be passed to :meth:`TestRunner.run`; this function will be called - with strings that should be displayed. It defaults to ``sys.stdout.write``. If - capturing the output is not sufficient, then the display output can be also - customized by subclassing DocTestRunner, and overriding the methods - :meth:`report_start`, :meth:`report_success`, - :meth:`report_unexpected_exception`, and :meth:`report_failure`. - - The optional keyword argument *checker* specifies the :class:`OutputChecker` - object (or drop-in replacement) that should be used to compare the expected - outputs to the actual outputs of doctest examples. - - The optional keyword argument *verbose* controls the :class:`DocTestRunner`'s - verbosity. If *verbose* is ``True``, then information is printed about each - example, as it is run. If *verbose* is ``False``, then only failures are - printed. If *verbose* is unspecified, or ``None``, then verbose output is used - iff the command-line switch ``-v`` is used. - - The optional keyword argument *optionflags* can be used to control how the test - runner compares expected output to actual output, and how it displays failures. - For more information, see section :ref:`doctest-options`. - - - :class:`DocTestParser` defines the following methods: - - - .. method:: report_start(out, test, example) - - Report that the test runner is about to process the given example. This method - is provided to allow subclasses of :class:`DocTestRunner` to customize their - output; it should not be called directly. - - *example* is the example about to be processed. *test* is the test - *containing example*. *out* is the output function that was passed to - :meth:`DocTestRunner.run`. - - - .. method:: report_success(out, test, example, got) - - Report that the given example ran successfully. This method is provided to - allow subclasses of :class:`DocTestRunner` to customize their output; it - should not be called directly. - - *example* is the example about to be processed. *got* is the actual output - from the example. *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: report_failure(out, test, example, got) - - Report that the given example failed. This method is provided to allow - subclasses of :class:`DocTestRunner` to customize their output; it should not - be called directly. - - *example* is the example about to be processed. *got* is the actual output - from the example. *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: report_unexpected_exception(out, test, example, exc_info) - - Report that the given example raised an unexpected exception. This method is - provided to allow subclasses of :class:`DocTestRunner` to customize their - output; it should not be called directly. - - *example* is the example about to be processed. *exc_info* is a tuple - containing information about the unexpected exception (as returned by - :func:`sys.exc_info`). *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: run(test, compileflags=None, out=None, clear_globs=True) - - Run the examples in *test* (a :class:`DocTest` object), and display the - results using the writer function *out*. - - The examples are run in the namespace ``test.globs``. If *clear_globs* is - true (the default), then this namespace will be cleared after the test runs, - to help with garbage collection. If you would like to examine the namespace - after the test completes, then use *clear_globs=False*. - - *compileflags* gives the set of flags that should be used by the Python - compiler when running the examples. If not specified, then it will default to - the set of future-import flags that apply to *globs*. - - The output of each example is checked using the :class:`DocTestRunner`'s - output checker, and the results are formatted by the - :meth:`DocTestRunner.report_\*` methods. - - - .. method:: summarize(verbose=None) - - Print a summary of all the test cases that have been run by this DocTestRunner, - and return a :term:`named tuple` ``TestResults(failed, attempted)``. - - The optional *verbose* argument controls how detailed the summary is. If the - verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is - used. - -.. _doctest-outputchecker: - -OutputChecker objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: OutputChecker() - - A class used to check the whether the actual output from a doctest example - matches the expected output. :class:`OutputChecker` defines two methods: - :meth:`check_output`, which compares a given pair of outputs, and returns true - if they match; and :meth:`output_difference`, which returns a string describing - the differences between two outputs. - - - :class:`OutputChecker` defines the following methods: - - .. method:: check_output(want, got, optionflags) - - Return ``True`` iff the actual output from an example (*got*) matches the - expected output (*want*). These strings are always considered to match if - they are identical; but depending on what option flags the test runner is - using, several non-exact match types are also possible. See section - :ref:`doctest-options` for more information about option flags. - - - .. method:: output_difference(example, got, optionflags) - - Return a string describing the differences between the expected output for a - given example (*example*) and the actual output (*got*). *optionflags* is the - set of option flags used to compare *want* and *got*. - - -.. _doctest-debugging: - -Debugging ---------- - -Doctest provides several mechanisms for debugging doctest examples: - -* Several functions convert doctests to executable Python programs, which can be - run under the Python debugger, :mod:`pdb`. - -* The :class:`DebugRunner` class is a subclass of :class:`DocTestRunner` that - raises an exception for the first failing example, containing information about - that example. This information can be used to perform post-mortem debugging on - the example. - -* The :mod:`unittest` cases generated by :func:`DocTestSuite` support the - :meth:`debug` method defined by :class:`unittest.TestCase`. - -* You can add a call to :func:`pdb.set_trace` in a doctest example, and you'll - drop into the Python debugger when that line is executed. Then you can inspect - current values of variables, and so on. For example, suppose :file:`a.py` - contains just this module docstring:: - - """ - >>> def f(x): - ... g(x*2) - >>> def g(x): - ... print(x+3) - ... import pdb; pdb.set_trace() - >>> f(3) - 9 - """ - - Then an interactive Python session may look like this:: - - >>> import a, doctest - >>> doctest.testmod(a) - --Return-- - > (3)g()->None - -> import pdb; pdb.set_trace() - (Pdb) list - 1 def g(x): - 2 print(x+3) - 3 -> import pdb; pdb.set_trace() - [EOF] - (Pdb) p x - 6 - (Pdb) step - --Return-- - > (2)f()->None - -> g(x*2) - (Pdb) list - 1 def f(x): - 2 -> g(x*2) - [EOF] - (Pdb) p x - 3 - (Pdb) step - --Return-- - > (1)?()->None - -> f(3) - (Pdb) cont - (0, 3) - >>> - - -Functions that convert doctests to Python code, and possibly run the synthesized -code under the debugger: - - -.. function:: script_from_examples(s) - - Convert text with examples to a script. - - Argument *s* is a string containing doctest examples. The string is converted - to a Python script, where doctest examples in *s* are converted to regular code, - and everything else is converted to Python comments. The generated script is - returned as a string. For example, :: - - import doctest - print(doctest.script_from_examples(r""" - Set x and y to 1 and 2. - >>> x, y = 1, 2 - - Print their sum: - >>> print(x+y) - 3 - """)) - - displays:: - - # Set x and y to 1 and 2. - x, y = 1, 2 - # - # Print their sum: - print(x+y) - # Expected: - ## 3 - - This function is used internally by other functions (see below), but can also be - useful when you want to transform an interactive Python session into a Python - script. - - -.. function:: testsource(module, name) - - Convert the doctest for an object to a script. - - Argument *module* is a module object, or dotted name of a module, containing the - object whose doctests are of interest. Argument *name* is the name (within the - module) of the object with the doctests of interest. The result is a string, - containing the object's docstring converted to a Python script, as described for - :func:`script_from_examples` above. For example, if module :file:`a.py` - contains a top-level function :func:`f`, then :: - - import a, doctest - print(doctest.testsource(a, "a.f")) - - prints a script version of function :func:`f`'s docstring, with doctests - converted to code, and the rest placed in comments. - - -.. function:: debug(module, name, pm=False) - - Debug the doctests for an object. - - The *module* and *name* arguments are the same as for function - :func:`testsource` above. The synthesized Python script for the named object's - docstring is written to a temporary file, and then that file is run under the - control of the Python debugger, :mod:`pdb`. - - A shallow copy of ``module.__dict__`` is used for both local and global - execution context. - - Optional argument *pm* controls whether post-mortem debugging is used. If *pm* - has a true value, the script file is run directly, and the debugger gets - involved only if the script terminates via raising an unhandled exception. If - it does, then post-mortem debugging is invoked, via :func:`pdb.post_mortem`, - passing the traceback object from the unhandled exception. If *pm* is not - specified, or is false, the script is run under the debugger from the start, via - passing an appropriate :func:`exec` call to :func:`pdb.run`. - - -.. function:: debug_src(src, pm=False, globs=None) - - Debug the doctests in a string. - - This is like function :func:`debug` above, except that a string containing - doctest examples is specified directly, via the *src* argument. - - Optional argument *pm* has the same meaning as in function :func:`debug` above. - - Optional argument *globs* gives a dictionary to use as both local and global - execution context. If not specified, or ``None``, an empty dictionary is used. - If specified, a shallow copy of the dictionary is used. - - -The :class:`DebugRunner` class, and the special exceptions it may raise, are of -most interest to testing framework authors, and will only be sketched here. See -the source code, and especially :class:`DebugRunner`'s docstring (which is a -doctest!) for more details: - - -.. class:: DebugRunner(checker=None, verbose=None, optionflags=0) - - A subclass of :class:`DocTestRunner` that raises an exception as soon as a - failure is encountered. If an unexpected exception occurs, an - :exc:`UnexpectedException` exception is raised, containing the test, the - example, and the original exception. If the output doesn't match, then a - :exc:`DocTestFailure` exception is raised, containing the test, the example, and - the actual output. - - For information about the constructor parameters and methods, see the - documentation for :class:`DocTestRunner` in section :ref:`doctest-advanced-api`. - -There are two exceptions that may be raised by :class:`DebugRunner` instances: - - -.. exception:: DocTestFailure(test, example, got) - - An exception raised by :class:`DocTestRunner` to signal that a doctest example's - actual output did not match its expected output. The constructor arguments are - used to initialize the attributes of the same names. - -:exc:`DocTestFailure` defines the following attributes: - - -.. attribute:: DocTestFailure.test - - The :class:`DocTest` object that was being run when the example failed. - - -.. attribute:: DocTestFailure.example - - The :class:`Example` that failed. - - -.. attribute:: DocTestFailure.got - - The example's actual output. - - -.. exception:: UnexpectedException(test, example, exc_info) - - An exception raised by :class:`DocTestRunner` to signal that a doctest - example raised an unexpected exception. The constructor arguments are used - to initialize the attributes of the same names. - -:exc:`UnexpectedException` defines the following attributes: - - -.. attribute:: UnexpectedException.test - - The :class:`DocTest` object that was being run when the example failed. - - -.. attribute:: UnexpectedException.example - - The :class:`Example` that failed. - - -.. attribute:: UnexpectedException.exc_info - - A tuple containing information about the unexpected exception, as returned by - :func:`sys.exc_info`. - - -.. _doctest-soapbox: - -Soapbox -------- - -As mentioned in the introduction, :mod:`doctest` has grown to have three primary -uses: - -#. Checking examples in docstrings. - -#. Regression testing. - -#. Executable documentation / literate testing. - -These uses have different requirements, and it is important to distinguish them. -In particular, filling your docstrings with obscure test cases makes for bad -documentation. - -When writing a docstring, choose docstring examples with care. There's an art to -this that needs to be learned---it may not be natural at first. Examples should -add genuine value to the documentation. A good example can often be worth many -words. If done with care, the examples will be invaluable for your users, and -will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` -examples stops working after a "harmless" change. - -Doctest also makes an excellent tool for regression testing, especially if you -don't skimp on explanatory text. By interleaving prose and examples, it becomes -much easier to keep track of what's actually being tested, and why. When a test -fails, good prose can make it much easier to figure out what the problem is, and -how it should be fixed. It's true that you could write extensive comments in -code-based testing, but few programmers do. Many have found that using doctest -approaches instead leads to much clearer tests. Perhaps this is simply because -doctest makes writing prose a little easier than writing code, while writing -comments in code is a little harder. I think it goes deeper than just that: -the natural attitude when writing a doctest-based test is that you want to -explain the fine points of your software, and illustrate them with examples. -This in turn naturally leads to test files that start with the simplest -features, and logically progress to complications and edge cases. A coherent -narrative is the result, instead of a collection of isolated functions that test -isolated bits of functionality seemingly at random. It's a different attitude, -and produces different results, blurring the distinction between testing and -explaining. - -Regression testing is best confined to dedicated objects or files. There are -several options for organizing tests: - -* Write text files containing test cases as interactive examples, and test the - files using :func:`testfile` or :func:`DocFileSuite`. This is recommended, - although is easiest to do for new projects, designed from the start to use - doctest. - -* Define functions named ``_regrtest_topic`` that consist of single docstrings, - containing test cases for the named topics. These functions can be included in - the same file as the module, or separated out into a separate test file. - -* Define a ``__test__`` dictionary mapping from regression test topics to - docstrings containing test cases. - -When you have placed your tests in a module, the module can itself be the test -runner. When a test fails, you can arrange for your test runner to re-run only -the failing doctest while you debug the problem. Here is a minimal example of -such a test runner:: - - if __name__ == '__main__': - import doctest - flags = doctest.REPORT_NDIFF|doctest.FAIL_FAST - if len(sys.argv) > 1: - name = sys.argv[1] - if name in globals(): - obj = globals()[name] - else: - obj = __test__[name] - doctest.run_docstring_examples(obj, globals(), name=name, - optionflags=flags) - else: - fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) - - -.. rubric:: Footnotes - -.. [#] Examples containing both expected output and an exception are not supported. - Trying to guess where one ends and the other begins is too error-prone, and that - also makes for a confusing test. diff --git a/third_party/python/Doc/library/dummy_threading.rst b/third_party/python/Doc/library/dummy_threading.rst deleted file mode 100644 index 30a3ebba6..000000000 --- a/third_party/python/Doc/library/dummy_threading.rst +++ /dev/null @@ -1,25 +0,0 @@ -:mod:`dummy_threading` --- Drop-in replacement for the :mod:`threading` module -============================================================================== - -.. module:: dummy_threading - :synopsis: Drop-in replacement for the threading module. - -**Source code:** :source:`Lib/dummy_threading.py` - --------------- - -This module provides a duplicate interface to the :mod:`threading` module. It -is meant to be imported when the :mod:`_thread` module is not provided on a -platform. - -Suggested usage is:: - - try: - import threading - except ImportError: - import dummy_threading as threading - -Be careful to not use this module where deadlock might occur from a thread being -created that blocks waiting for another thread to be created. This often occurs -with blocking I/O. - diff --git a/third_party/python/Doc/library/email.charset.rst b/third_party/python/Doc/library/email.charset.rst deleted file mode 100644 index 053463f97..000000000 --- a/third_party/python/Doc/library/email.charset.rst +++ /dev/null @@ -1,247 +0,0 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- - -.. module:: email.charset - :synopsis: Character Sets - -**Source code:** :source:`Lib/email/charset.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the new -API only the aliases table is used. - -The remaining text in this section is the original documentation of the module. - -This module provides a class :class:`Charset` for representing character sets -and character set conversions in email messages, as well as a character set -registry and several convenience methods for manipulating this registry. -Instances of :class:`Charset` are used in several other modules within the -:mod:`email` package. - -Import this class from the :mod:`email.charset` module. - - -.. class:: Charset(input_charset=DEFAULT_CHARSET) - - Map character sets to their email properties. - - This class provides information about the requirements imposed on email for a - specific character set. It also provides convenience routines for converting - between character sets, given the availability of the applicable codecs. Given - a character set, it will do its best to provide information on how to use that - character set in an email message in an RFC-compliant way. - - Certain character sets must be encoded with quoted-printable or base64 when used - in email headers or bodies. Certain character sets must be converted outright, - and are not allowed in email. - - Optional *input_charset* is as described below; it is always coerced to lower - case. After being alias normalized it is also used as a lookup into the - registry of character sets to find out the header encoding, body encoding, and - output conversion codec to be used for the character set. For example, if - *input_charset* is ``iso-8859-1``, then headers and bodies will be encoded using - quoted-printable and no output conversion codec is necessary. If - *input_charset* is ``euc-jp``, then headers will be encoded with base64, bodies - will not be encoded, but output text will be converted from the ``euc-jp`` - character set to the ``iso-2022-jp`` character set. - - :class:`Charset` instances have the following data attributes: - - .. attribute:: input_charset - - The initial character set specified. Common aliases are converted to - their *official* email names (e.g. ``latin_1`` is converted to - ``iso-8859-1``). Defaults to 7-bit ``us-ascii``. - - - .. attribute:: header_encoding - - If the character set must be encoded before it can be used in an email - header, this attribute will be set to ``Charset.QP`` (for - quoted-printable), ``Charset.BASE64`` (for base64 encoding), or - ``Charset.SHORTEST`` for the shortest of QP or BASE64 encoding. Otherwise, - it will be ``None``. - - - .. attribute:: body_encoding - - Same as *header_encoding*, but describes the encoding for the mail - message's body, which indeed may be different than the header encoding. - ``Charset.SHORTEST`` is not allowed for *body_encoding*. - - - .. attribute:: output_charset - - Some character sets must be converted before they can be used in email - headers or bodies. If the *input_charset* is one of them, this attribute - will contain the name of the character set output will be converted to. - Otherwise, it will be ``None``. - - - .. attribute:: input_codec - - The name of the Python codec used to convert the *input_charset* to - Unicode. If no conversion codec is necessary, this attribute will be - ``None``. - - - .. attribute:: output_codec - - The name of the Python codec used to convert Unicode to the - *output_charset*. If no conversion codec is necessary, this attribute - will have the same value as the *input_codec*. - - - :class:`Charset` instances also have the following methods: - - .. method:: get_body_encoding() - - Return the content transfer encoding used for body encoding. - - This is either the string ``quoted-printable`` or ``base64`` depending on - the encoding used, or it is a function, in which case you should call the - function with a single argument, the Message object being encoded. The - function should then set the :mailheader:`Content-Transfer-Encoding` - header itself to whatever is appropriate. - - Returns the string ``quoted-printable`` if *body_encoding* is ``QP``, - returns the string ``base64`` if *body_encoding* is ``BASE64``, and - returns the string ``7bit`` otherwise. - - - .. XXX to_splittable and from_splittable are not there anymore! - - .. to_splittable(s) - - Convert a possibly multibyte string to a safely splittable format. *s* is - the string to split. - - Uses the *input_codec* to try and convert the string to Unicode, so it can - be safely split on character boundaries (even for multibyte characters). - - Returns the string as-is if it isn't known how to convert *s* to Unicode - with the *input_charset*. - - Characters that could not be converted to Unicode will be replaced with - the Unicode replacement character ``'U+FFFD'``. - - - .. from_splittable(ustr[, to_output]) - - Convert a splittable string back into an encoded string. *ustr* is a - Unicode string to "unsplit". - - This method uses the proper codec to try and convert the string from - Unicode back into an encoded format. Return the string as-is if it is not - Unicode, or if it could not be converted from Unicode. - - Characters that could not be converted from Unicode will be replaced with - an appropriate character (usually ``'?'``). - - If *to_output* is ``True`` (the default), uses *output_codec* to convert - to an encoded format. If *to_output* is ``False``, it uses *input_codec*. - - - .. method:: get_output_charset() - - Return the output character set. - - This is the *output_charset* attribute if that is not ``None``, otherwise - it is *input_charset*. - - - .. method:: header_encode(string) - - Header-encode the string *string*. - - The type of encoding (base64 or quoted-printable) will be based on the - *header_encoding* attribute. - - - .. method:: header_encode_lines(string, maxlengths) - - Header-encode a *string* by converting it first to bytes. - - This is similar to :meth:`header_encode` except that the string is fit - into maximum line lengths as given by the argument *maxlengths*, which - must be an iterator: each element returned from this iterator will provide - the next maximum line length. - - - .. method:: body_encode(string) - - Body-encode the string *string*. - - The type of encoding (base64 or quoted-printable) will be based on the - *body_encoding* attribute. - - The :class:`Charset` class also provides a number of methods to support - standard operations and built-in functions. - - - .. method:: __str__() - - Returns *input_charset* as a string coerced to lower - case. :meth:`__repr__` is an alias for :meth:`__str__`. - - - .. method:: __eq__(other) - - This method allows you to compare two :class:`Charset` instances for - equality. - - - .. method:: __ne__(other) - - This method allows you to compare two :class:`Charset` instances for - inequality. - -The :mod:`email.charset` module also provides the following functions for adding -new entries to the global character set, alias, and codec registries: - - -.. function:: add_charset(charset, header_enc=None, body_enc=None, output_charset=None) - - Add character properties to the global registry. - - *charset* is the input character set, and must be the canonical name of a - character set. - - Optional *header_enc* and *body_enc* is either ``Charset.QP`` for - quoted-printable, ``Charset.BASE64`` for base64 encoding, - ``Charset.SHORTEST`` for the shortest of quoted-printable or base64 encoding, - or ``None`` for no encoding. ``SHORTEST`` is only valid for - *header_enc*. The default is ``None`` for no encoding. - - Optional *output_charset* is the character set that the output should be in. - Conversions will proceed from input charset, to Unicode, to the output charset - when the method :meth:`Charset.convert` is called. The default is to output in - the same character set as the input. - - Both *input_charset* and *output_charset* must have Unicode codec entries in the - module's character set-to-codec mapping; use :func:`add_codec` to add codecs the - module does not know about. See the :mod:`codecs` module's documentation for - more information. - - The global character set registry is kept in the module global dictionary - ``CHARSETS``. - - -.. function:: add_alias(alias, canonical) - - Add a character set alias. *alias* is the alias name, e.g. ``latin-1``. - *canonical* is the character set's canonical name, e.g. ``iso-8859-1``. - - The global charset alias registry is kept in the module global dictionary - ``ALIASES``. - - -.. function:: add_codec(charset, codecname) - - Add a codec that map characters in the given character set to and from Unicode. - - *charset* is the canonical name of a character set. *codecname* is the name of a - Python codec, as appropriate for the second argument to the :class:`str`'s - :meth:`~str.encode` method. - diff --git a/third_party/python/Doc/library/email.compat32-message.rst b/third_party/python/Doc/library/email.compat32-message.rst deleted file mode 100644 index 8d1c2f5da..000000000 --- a/third_party/python/Doc/library/email.compat32-message.rst +++ /dev/null @@ -1,755 +0,0 @@ -.. _compat32_message: - -:mod:`email.message.Message`: Representing an email message using the :data:`~email.policy.compat32` API --------------------------------------------------------------------------------------------------------- - -.. module:: email.message - :synopsis: The base class representing email messages in a fashion - backward compatible with python3.2 - :noindex: - - -The :class:`Message` class is very similar to the -:class:`~email.message.EmailMessage` class, without the methods added by that -class, and with the default behavior of certain other methods being slightly -different. We also document here some methods that, while supported by the -:class:`~email.message.EmailMessage` class, are not recommended unless you are -dealing with legacy code. - -The philosophy and structure of the two classes is otherwise the same. - -This document describes the behavior under the default (for :class:`Message`) -policy :attr:`~email.policy.Compat32`. If you are going to use another policy, -you should be using the :class:`~email.message.EmailMessage` class instead. - -An email message consists of *headers* and a *payload*. Headers must be -:rfc:`5233` style names and values, where the field name and value are -separated by a colon. The colon is not part of either the field name or the -field value. The payload may be a simple text message, or a binary object, or -a structured sequence of sub-messages each with their own set of headers and -their own payload. The latter type of payload is indicated by the message -having a MIME type such as :mimetype:`multipart/\*` or -:mimetype:`message/rfc822`. - -The conceptual model provided by a :class:`Message` object is that of an -ordered dictionary of headers with additional methods for accessing both -specialized information from the headers, for accessing the payload, for -generating a serialized version of the message, and for recursively walking -over the object tree. Note that duplicate headers are supported but special -methods must be used to access them. - -The :class:`Message` pseudo-dictionary is indexed by the header names, which -must be ASCII values. The values of the dictionary are strings that are -supposed to contain only ASCII characters; there is some special handling for -non-ASCII input, but it doesn't always produce the correct results. Headers -are stored and returned in case-preserving form, but field names are matched -case-insensitively. There may also be a single envelope header, also known as -the *Unix-From* header or the ``From_`` header. The *payload* is either a -string or bytes, in the case of simple message objects, or a list of -:class:`Message` objects, for MIME container documents (e.g. -:mimetype:`multipart/\*` and :mimetype:`message/rfc822`). - -Here are the methods of the :class:`Message` class: - - -.. class:: Message(policy=compat32) - - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update and serialize the representation - of the message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. - - .. versionchanged:: 3.3 The *policy* keyword argument was added. - - - .. method:: as_string(unixfrom=False, maxheaderlen=0, policy=None) - - Return the entire message flattened as a string. When optional *unixfrom* - is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compatibility reasons, - *maxheaderlen* defaults to ``0``, so if you want a different value you - must override it explicitly (the value specified for *max_line_length* in - the policy will be ignored by this method). The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the ``Generator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not always - format the message the way you want. For example, by default it does - not do the mangling of lines that begin with ``From`` that is - required by the unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.Generator` instance and use its - :meth:`~email.generator.Generator.flatten` method directly. For example:: - - from io import StringIO - from email.generator import Generator - fp = StringIO() - g = Generator(fp, mangle_from_=True, maxheaderlen=60) - g.flatten(msg) - text = fp.getvalue() - - If the message object contains binary data that is not encoded according - to RFC standards, the non-compliant data will be replaced by unicode - "unknown character" code points. (See also :meth:`.as_bytes` and - :class:`~email.generator.BytesGenerator`.) - - .. versionchanged:: 3.4 the *policy* keyword argument was added. - - - .. method:: __str__() - - Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a - string containing the formatted message. - - - .. method:: as_bytes(unixfrom=False, policy=None) - - Return the entire message flattened as a bytes object. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the - ``BytesGenerator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not always - format the message the way you want. For example, by default it does - not do the mangling of lines that begin with ``From`` that is - required by the unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.BytesGenerator` instance and use its - :meth:`~email.generator.BytesGenerator.flatten` method directly. - For example:: - - from io import BytesIO - from email.generator import BytesGenerator - fp = BytesIO() - g = BytesGenerator(fp, mangle_from_=True, maxheaderlen=60) - g.flatten(msg) - text = fp.getvalue() - - .. versionadded:: 3.4 - - - .. method:: __bytes__() - - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a - bytes object containing the formatted message. - - .. versionadded:: 3.4 - - - .. method:: is_multipart() - - Return ``True`` if the message's payload is a list of - sub-\ :class:`Message` objects, otherwise return ``False``. When - :meth:`is_multipart` returns ``False``, the payload should be a string - object (which might be a CTE encoded binary payload). (Note that - :meth:`is_multipart` returning ``True`` does not necessarily mean that - "msg.get_content_maintype() == 'multipart'" will return the ``True``. - For example, ``is_multipart`` will return ``True`` when the - :class:`Message` is of type ``message/rfc822``.) - - - .. method:: set_unixfrom(unixfrom) - - Set the message's envelope header to *unixfrom*, which should be a string. - - - .. method:: get_unixfrom() - - Return the message's envelope header. Defaults to ``None`` if the - envelope header was never set. - - - .. method:: attach(payload) - - Add the given *payload* to the current payload, which must be ``None`` or - a list of :class:`Message` objects before the call. After the call, the - payload will always be a list of :class:`Message` objects. If you want to - set the payload to a scalar object (e.g. a string), use - :meth:`set_payload` instead. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.set_content` and the - related ``make`` and ``add`` methods. - - - .. method:: get_payload(i=None, decode=False) - - Return the current payload, which will be a list of - :class:`Message` objects when :meth:`is_multipart` is ``True``, or a - string when :meth:`is_multipart` is ``False``. If the payload is a list - and you mutate the list object, you modify the message's payload in place. - - With optional argument *i*, :meth:`get_payload` will return the *i*-th - element of the payload, counting from zero, if :meth:`is_multipart` is - ``True``. An :exc:`IndexError` will be raised if *i* is less than 0 or - greater than or equal to the number of items in the payload. If the - payload is a string (i.e. :meth:`is_multipart` is ``False``) and *i* is - given, a :exc:`TypeError` is raised. - - Optional *decode* is a flag indicating whether the payload should be - decoded or not, according to the :mailheader:`Content-Transfer-Encoding` - header. When ``True`` and the message is not a multipart, the payload will - be decoded if this header's value is ``quoted-printable`` or ``base64``. - If some other encoding is used, or :mailheader:`Content-Transfer-Encoding` - header is missing, the payload is - returned as-is (undecoded). In all cases the returned value is binary - data. If the message is a multipart and the *decode* flag is ``True``, - then ``None`` is returned. If the payload is base64 and it was not - perfectly formed (missing padding, characters outside the base64 - alphabet), then an appropriate defect will be added to the message's - defect property (:class:`~email.errors.InvalidBase64PaddingDefect` or - :class:`~email.errors.InvalidBase64CharactersDefect`, respectively). - - When *decode* is ``False`` (the default) the body is returned as a string - without decoding the :mailheader:`Content-Transfer-Encoding`. However, - for a :mailheader:`Content-Transfer-Encoding` of 8bit, an attempt is made - to decode the original bytes using the ``charset`` specified by the - :mailheader:`Content-Type` header, using the ``replace`` error handler. - If no ``charset`` is specified, or if the ``charset`` given is not - recognized by the email package, the body is decoded using the default - ASCII charset. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.get_content` and - :meth:`~email.message.EmailMessage.iter_parts`. - - - .. method:: set_payload(payload, charset=None) - - Set the entire message object's payload to *payload*. It is the client's - responsibility to ensure the payload invariants. Optional *charset* sets - the message's default character set; see :meth:`set_charset` for details. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.set_content`. - - - .. method:: set_charset(charset) - - Set the character set of the payload to *charset*, which can either be a - :class:`~email.charset.Charset` instance (see :mod:`email.charset`), a - string naming a character set, or ``None``. If it is a string, it will - be converted to a :class:`~email.charset.Charset` instance. If *charset* - is ``None``, the ``charset`` parameter will be removed from the - :mailheader:`Content-Type` header (the message will not be otherwise - modified). Anything else will generate a :exc:`TypeError`. - - If there is no existing :mailheader:`MIME-Version` header one will be - added. If there is no existing :mailheader:`Content-Type` header, one - will be added with a value of :mimetype:`text/plain`. Whether the - :mailheader:`Content-Type` header already exists or not, its ``charset`` - parameter will be set to *charset.output_charset*. If - *charset.input_charset* and *charset.output_charset* differ, the payload - will be re-encoded to the *output_charset*. If there is no existing - :mailheader:`Content-Transfer-Encoding` header, then the payload will be - transfer-encoded, if needed, using the specified - :class:`~email.charset.Charset`, and a header with the appropriate value - will be added. If a :mailheader:`Content-Transfer-Encoding` header - already exists, the payload is assumed to already be correctly encoded - using that :mailheader:`Content-Transfer-Encoding` and is not modified. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. - - - .. method:: get_charset() - - Return the :class:`~email.charset.Charset` instance associated with the - message's payload. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns - ``None``. - - - The following methods implement a mapping-like interface for accessing the - message's :rfc:`2822` headers. Note that there are some semantic differences - between these methods and a normal mapping (i.e. dictionary) interface. For - example, in a dictionary there are no duplicate keys, but here there may be - duplicate message headers. Also, in dictionaries there is no guaranteed - order to the keys returned by :meth:`keys`, but in a :class:`Message` object, - headers are always returned in the order they appeared in the original - message, or were added to the message later. Any header deleted and then - re-added are always appended to the end of the header list. - - These semantic differences are intentional and are biased toward maximal - convenience. - - Note that in all cases, any envelope header present in the message is not - included in the mapping interface. - - In a model generated from bytes, any header values that (in contravention of - the RFCs) contain non-ASCII bytes will, when retrieved through this - interface, be represented as :class:`~email.header.Header` objects with - a charset of `unknown-8bit`. - - - .. method:: __len__() - - Return the total number of headers, including duplicates. - - - .. method:: __contains__(name) - - Return true if the message object has a field named *name*. Matching is - done case-insensitively and *name* should not include the trailing colon. - Used for the ``in`` operator, e.g.:: - - if 'message-id' in myMessage: - print('Message-ID:', myMessage['message-id']) - - - .. method:: __getitem__(name) - - Return the value of the named header field. *name* should not include the - colon field separator. If the header is missing, ``None`` is returned; a - :exc:`KeyError` is never raised. - - Note that if the named field appears more than once in the message's - headers, exactly which of those field values will be returned is - undefined. Use the :meth:`get_all` method to get the values of all the - extant named headers. - - - .. method:: __setitem__(name, val) - - Add a header to the message with field name *name* and value *val*. The - field is appended to the end of the message's existing fields. - - Note that this does *not* overwrite or delete any existing header with the same - name. If you want to ensure that the new header is the only one present in the - message with field name *name*, delete the field first, e.g.:: - - del msg['subject'] - msg['subject'] = 'Python roolz!' - - - .. method:: __delitem__(name) - - Delete all occurrences of the field with name *name* from the message's - headers. No exception is raised if the named field isn't present in the - headers. - - - .. method:: keys() - - Return a list of all the message's header field names. - - - .. method:: values() - - Return a list of all the message's field values. - - - .. method:: items() - - Return a list of 2-tuples containing all the message's field headers and - values. - - - .. method:: get(name, failobj=None) - - Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the - named header is missing (defaults to ``None``). - - Here are some additional useful methods: - - - .. method:: get_all(name, failobj=None) - - Return a list of all the values for the field named *name*. If there are - no such named headers in the message, *failobj* is returned (defaults to - ``None``). - - - .. method:: add_header(_name, _value, **_params) - - Extended header setting. This method is similar to :meth:`__setitem__` - except that additional header parameters can be provided as keyword - arguments. *_name* is the header field to add and *_value* is the - *primary* value for the header. - - For each item in the keyword argument dictionary *_params*, the key is - taken as the parameter name, with underscores converted to dashes (since - dashes are illegal in Python identifiers). Normally, the parameter will - be added as ``key="value"`` unless the value is ``None``, in which case - only the key will be added. If the value contains non-ASCII characters, - it can be specified as a three tuple in the format - ``(CHARSET, LANGUAGE, VALUE)``, where ``CHARSET`` is a string naming the - charset to be used to encode the value, ``LANGUAGE`` can usually be set - to ``None`` or the empty string (see :rfc:`2231` for other possibilities), - and ``VALUE`` is the string value containing non-ASCII code points. If - a three tuple is not passed and the value contains non-ASCII characters, - it is automatically encoded in :rfc:`2231` format using a ``CHARSET`` - of ``utf-8`` and a ``LANGUAGE`` of ``None``. - - Here's an example:: - - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - - This will add a header that looks like :: - - Content-Disposition: attachment; filename="bud.gif" - - An example with non-ASCII characters:: - - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fußballer.ppt')) - - Which produces :: - - Content-Disposition: attachment; filename*="iso-8859-1''Fu%DFballer.ppt" - - - .. method:: replace_header(_name, _value) - - Replace a header. Replace the first header found in the message that - matches *_name*, retaining header order and field name case. If no - matching header was found, a :exc:`KeyError` is raised. - - - .. method:: get_content_type() - - Return the message's content type. The returned string is coerced to - lower case of the form :mimetype:`maintype/subtype`. If there was no - :mailheader:`Content-Type` header in the message the default type as given - by :meth:`get_default_type` will be returned. Since according to - :rfc:`2045`, messages always have a default type, :meth:`get_content_type` - will always return a value. - - :rfc:`2045` defines a message's default type to be :mimetype:`text/plain` - unless it appears inside a :mimetype:`multipart/digest` container, in - which case it would be :mimetype:`message/rfc822`. If the - :mailheader:`Content-Type` header has an invalid type specification, - :rfc:`2045` mandates that the default type be :mimetype:`text/plain`. - - - .. method:: get_content_maintype() - - Return the message's main content type. This is the :mimetype:`maintype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_content_subtype() - - Return the message's sub-content type. This is the :mimetype:`subtype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_default_type() - - Return the default content type. Most messages have a default content - type of :mimetype:`text/plain`, except for messages that are subparts of - :mimetype:`multipart/digest` containers. Such subparts have a default - content type of :mimetype:`message/rfc822`. - - - .. method:: set_default_type(ctype) - - Set the default content type. *ctype* should either be - :mimetype:`text/plain` or :mimetype:`message/rfc822`, although this is not - enforced. The default content type is not stored in the - :mailheader:`Content-Type` header. - - - .. method:: get_params(failobj=None, header='content-type', unquote=True) - - Return the message's :mailheader:`Content-Type` parameters, as a list. - The elements of the returned list are 2-tuples of key/value pairs, as - split on the ``'='`` sign. The left hand side of the ``'='`` is the key, - while the right hand side is the value. If there is no ``'='`` sign in - the parameter the value is the empty string, otherwise the value is as - described in :meth:`get_param` and is unquoted if optional *unquote* is - ``True`` (the default). - - Optional *failobj* is the object to return if there is no - :mailheader:`Content-Type` header. Optional *header* is the header to - search instead of :mailheader:`Content-Type`. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *params* property of the individual header objects - returned by the header access methods. - - - .. method:: get_param(param, failobj=None, header='content-type', unquote=True) - - Return the value of the :mailheader:`Content-Type` header's parameter - *param* as a string. If the message has no :mailheader:`Content-Type` - header or if there is no such parameter, then *failobj* is returned - (defaults to ``None``). - - Optional *header* if given, specifies the message header to use instead of - :mailheader:`Content-Type`. - - Parameter keys are always compared case insensitively. The return value - can either be a string, or a 3-tuple if the parameter was :rfc:`2231` - encoded. When it's a 3-tuple, the elements of the value are of the form - ``(CHARSET, LANGUAGE, VALUE)``. Note that both ``CHARSET`` and - ``LANGUAGE`` can be ``None``, in which case you should consider ``VALUE`` - to be encoded in the ``us-ascii`` charset. You can usually ignore - ``LANGUAGE``. - - If your application doesn't care whether the parameter was encoded as in - :rfc:`2231`, you can collapse the parameter value by calling - :func:`email.utils.collapse_rfc2231_value`, passing in the return value - from :meth:`get_param`. This will return a suitably decoded Unicode - string when the value is a tuple, or the original string unquoted if it - isn't. For example:: - - rawparam = msg.get_param('foo') - param = email.utils.collapse_rfc2231_value(rawparam) - - In any case, the parameter value (either the returned string, or the - ``VALUE`` item in the 3-tuple) is always unquoted, unless *unquote* is set - to ``False``. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *params* property of the individual header objects - returned by the header access methods. - - - .. method:: set_param(param, value, header='Content-Type', requote=True, \ - charset=None, language='', replace=False) - - Set a parameter in the :mailheader:`Content-Type` header. If the - parameter already exists in the header, its value will be replaced with - *value*. If the :mailheader:`Content-Type` header as not yet been defined - for this message, it will be set to :mimetype:`text/plain` and the new - parameter value will be appended as per :rfc:`2045`. - - Optional *header* specifies an alternative header to - :mailheader:`Content-Type`, and all parameters will be quoted as necessary - unless optional *requote* is ``False`` (the default is ``True``). - - If optional *charset* is specified, the parameter will be encoded - according to :rfc:`2231`. Optional *language* specifies the RFC 2231 - language, defaulting to the empty string. Both *charset* and *language* - should be strings. - - If *replace* is ``False`` (the default) the header is moved to the - end of the list of headers. If *replace* is ``True``, the header - will be updated in place. - - .. versionchanged:: 3.4 ``replace`` keyword was added. - - - .. method:: del_param(param, header='content-type', requote=True) - - Remove the given parameter completely from the :mailheader:`Content-Type` - header. The header will be re-written in place without the parameter or - its value. All values will be quoted as necessary unless *requote* is - ``False`` (the default is ``True``). Optional *header* specifies an - alternative to :mailheader:`Content-Type`. - - - .. method:: set_type(type, header='Content-Type', requote=True) - - Set the main type and subtype for the :mailheader:`Content-Type` - header. *type* must be a string in the form :mimetype:`maintype/subtype`, - otherwise a :exc:`ValueError` is raised. - - This method replaces the :mailheader:`Content-Type` header, keeping all - the parameters in place. If *requote* is ``False``, this leaves the - existing header's quoting as is, otherwise the parameters will be quoted - (the default). - - An alternative header can be specified in the *header* argument. When the - :mailheader:`Content-Type` header is set a :mailheader:`MIME-Version` - header is also added. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the ``make_`` and ``add_`` methods. - - - .. method:: get_filename(failobj=None) - - Return the value of the ``filename`` parameter of the - :mailheader:`Content-Disposition` header of the message. If the header - does not have a ``filename`` parameter, this method falls back to looking - for the ``name`` parameter on the :mailheader:`Content-Type` header. If - neither is found, or the header is missing, then *failobj* is returned. - The returned string will always be unquoted as per - :func:`email.utils.unquote`. - - - .. method:: get_boundary(failobj=None) - - Return the value of the ``boundary`` parameter of the - :mailheader:`Content-Type` header of the message, or *failobj* if either - the header is missing, or has no ``boundary`` parameter. The returned - string will always be unquoted as per :func:`email.utils.unquote`. - - - .. method:: set_boundary(boundary) - - Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to - *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`~email.errors.HeaderParseError` is raised if the - message object has no :mailheader:`Content-Type` header. - - Note that using this method is subtly different than deleting the old - :mailheader:`Content-Type` header and adding a new one with the new - boundary via :meth:`add_header`, because :meth:`set_boundary` preserves - the order of the :mailheader:`Content-Type` header in the list of - headers. However, it does *not* preserve any continuation lines which may - have been present in the original :mailheader:`Content-Type` header. - - - .. method:: get_content_charset(failobj=None) - - Return the ``charset`` parameter of the :mailheader:`Content-Type` header, - coerced to lower case. If there is no :mailheader:`Content-Type` header, or if - that header has no ``charset`` parameter, *failobj* is returned. - - Note that this method differs from :meth:`get_charset` which returns the - :class:`~email.charset.Charset` instance for the default encoding of the message body. - - - .. method:: get_charsets(failobj=None) - - Return a list containing the character set names in the message. If the - message is a :mimetype:`multipart`, then the list will contain one element - for each subpart in the payload, otherwise, it will be a list of length 1. - - Each item in the list will be a string which is the value of the - ``charset`` parameter in the :mailheader:`Content-Type` header for the - represented subpart. However, if the subpart has no - :mailheader:`Content-Type` header, no ``charset`` parameter, or is not of - the :mimetype:`text` main MIME type, then that item in the returned list - will be *failobj*. - - - .. method:: get_content_disposition() - - Return the lowercased value (without parameters) of the message's - :mailheader:`Content-Disposition` header if it has one, or ``None``. The - possible values for this method are *inline*, *attachment* or ``None`` - if the message follows :rfc:`2183`. - - .. versionadded:: 3.5 - - .. method:: walk() - - The :meth:`walk` method is an all-purpose generator which can be used to - iterate over all the parts and subparts of a message object tree, in - depth-first traversal order. You will typically use :meth:`walk` as the - iterator in a ``for`` loop; each iteration returns the next subpart. - - Here's an example that prints the MIME type of every part of a multipart - message structure: - - .. testsetup:: - - >>> from email import message_from_binary_file - >>> with open('Lib/test/test_email/data/msg_16.txt', 'rb') as f: - ... msg = message_from_binary_file(f) - >>> from email.iterators import _structure - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_type()) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - ``walk`` iterates over the subparts of any part where - :meth:`is_multipart` returns ``True``, even though - ``msg.get_content_maintype() == 'multipart'`` may return ``False``. We - can see this in our example by making use of the ``_structure`` debug - helper function: - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_maintype() == 'multipart'), - ... part.is_multipart()) - True True - False False - False True - False False - False False - False True - False False - >>> _structure(msg) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - Here the ``message`` parts are not ``multiparts``, but they do contain - subparts. ``is_multipart()`` returns ``True`` and ``walk`` descends - into the subparts. - - - :class:`Message` objects can also optionally contain two instance attributes, - which can be used when generating the plain text of a MIME message. - - - .. attribute:: preamble - - The format of a MIME document allows for some text between the blank line - following the headers, and the first multipart boundary string. Normally, - this text is never visible in a MIME-aware mail reader because it falls - outside the standard MIME armor. However, when viewing the raw text of - the message, or when viewing the message in a non-MIME aware reader, this - text can become visible. - - The *preamble* attribute contains this leading extra-armor text for MIME - documents. When the :class:`~email.parser.Parser` discovers some text - after the headers but before the first boundary string, it assigns this - text to the message's *preamble* attribute. When the - :class:`~email.generator.Generator` is writing out the plain text - representation of a MIME message, and it finds the - message has a *preamble* attribute, it will write this text in the area - between the headers and the first boundary. See :mod:`email.parser` and - :mod:`email.generator` for details. - - Note that if the message object has no preamble, the *preamble* attribute - will be ``None``. - - - .. attribute:: epilogue - - The *epilogue* attribute acts the same way as the *preamble* attribute, - except that it contains text that appears between the last boundary and - the end of the message. - - You do not need to set the epilogue to the empty string in order for the - :class:`~email.generator.Generator` to print a newline at the end of the - file. - - - .. attribute:: defects - - The *defects* attribute contains a list of all the problems found when - parsing this message. See :mod:`email.errors` for a detailed description - of the possible parsing defects. diff --git a/third_party/python/Doc/library/email.contentmanager.rst b/third_party/python/Doc/library/email.contentmanager.rst deleted file mode 100644 index e09c7c0e4..000000000 --- a/third_party/python/Doc/library/email.contentmanager.rst +++ /dev/null @@ -1,198 +0,0 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- - -.. module:: email.contentmanager - :synopsis: Storing and Retrieving Content from MIME Parts - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/contentmanager.py` - ------------- - -.. versionadded:: 3.6 [1]_ - - -.. class:: ContentManager() - - Base class for content managers. Provides the standard registry mechanisms - to register converters between MIME content and other representations, as - well as the ``get_content`` and ``set_content`` dispatch methods. - - - .. method:: get_content(msg, *args, **kw) - - Look up a handler function based on the ``mimetype`` of *msg* (see next - paragraph), call it, passing through all arguments, and return the result - of the call. The expectation is that the handler will extract the - payload from *msg* and return an object that encodes information about - the extracted data. - - To find the handler, look for the following keys in the registry, - stopping with the first one found: - - * the string representing the full MIME type (``maintype/subtype``) - * the string representing the ``maintype`` - * the empty string - - If none of these keys produce a handler, raise a :exc:`KeyError` for the - full MIME type. - - - .. method:: set_content(msg, obj, *args, **kw) - - If the ``maintype`` is ``multipart``, raise a :exc:`TypeError`; otherwise - look up a handler function based on the type of *obj* (see next - paragraph), call :meth:`~email.message.EmailMessage.clear_content` on the - *msg*, and call the handler function, passing through all arguments. The - expectation is that the handler will transform and store *obj* into - *msg*, possibly making other changes to *msg* as well, such as adding - various MIME headers to encode information needed to interpret the stored - data. - - To find the handler, obtain the type of *obj* (``typ = type(obj)``), and - look for the following keys in the registry, stopping with the first one - found: - - * the type itself (``typ``) - * the type's fully qualified name (``typ.__module__ + '.' + - typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). - - If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key - yields a handler, check for a handler for the key ``None``. If there is - no handler for ``None``, raise a :exc:`KeyError` for the fully - qualified name of the type. - - Also add a :mailheader:`MIME-Version` header if one is not present (see - also :class:`.MIMEPart`). - - - .. method:: add_get_handler(key, handler) - - Record the function *handler* as the handler for *key*. For the possible - values of *key*, see :meth:`get_content`. - - - .. method:: add_set_handler(typekey, handler) - - Record *handler* as the function to call when an object of a type - matching *typekey* is passed to :meth:`set_content`. For the possible - values of *typekey*, see :meth:`set_content`. - - -Content Manager Instances -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Currently the email package provides only one concrete content manager, -:data:`raw_data_manager`, although more may be added in the future. -:data:`raw_data_manager` is the -:attr:`~email.policy.EmailPolicy.content_manager` provided by -:attr:`~email.policy.EmailPolicy` and its derivatives. - - -.. data:: raw_data_manager - - This content manager provides only a minimum interface beyond that provided - by :class:`~email.message.Message` itself: it deals only with text, raw - byte strings, and :class:`~email.message.Message` objects. Nevertheless, it - provides significant advantages compared to the base API: ``get_content`` on - a text part will return a unicode string without the application needing to - manually decode it, ``set_content`` provides a rich set of options for - controlling the headers added to a part and controlling the content transfer - encoding, and it enables the use of the various ``add_`` methods, thereby - simplifying the creation of multipart messages. - - .. method:: get_content(msg, errors='replace') - - Return the payload of the part as either a string (for ``text`` parts), an - :class:`~email.message.EmailMessage` object (for ``message/rfc822`` - parts), or a ``bytes`` object (for all other non-multipart types). Raise - a :exc:`KeyError` if called on a ``multipart``. If the part is a - ``text`` part and *errors* is specified, use it as the error handler when - decoding the payload to unicode. The default error handler is - ``replace``. - - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ - cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'bytes'>, maintype, subtype, cte="base64", \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'EmailMessage'>, cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - - Add headers and payload to *msg*: - - Add a :mailheader:`Content-Type` header with a ``maintype/subtype`` - value. - - * For ``str``, set the MIME ``maintype`` to ``text``, and set the - subtype to *subtype* if it is specified, or ``plain`` if it is not. - * For ``bytes``, use the specified *maintype* and *subtype*, or - raise a :exc:`TypeError` if they are not specified. - * For :class:`~email.message.EmailMessage` objects, set the maintype - to ``message``, and set the subtype to *subtype* if it is - specified or ``rfc822`` if it is not. If *subtype* is - ``partial``, raise an error (``bytes`` objects must be used to - construct ``message/partial`` parts). - - If *charset* is provided (which is valid only for ``str``), encode the - string to bytes using the specified character set. The default is - ``utf-8``. If the specified *charset* is a known alias for a standard - MIME charset name, use the standard charset instead. - - If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Encoding` header to - that value. Possible values for *cte* are ``quoted-printable``, - ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be - encoded in the specified encoding (for example, specifying a *cte* of - ``7bit`` for an input that contains non-ASCII values), raise a - :exc:`ValueError`. - - * For ``str`` objects, if *cte* is not set use heuristics to - determine the most compact encoding. - * For :class:`~email.message.EmailMessage`, per :rfc:`2046`, raise - an error if a *cte* of ``quoted-printable`` or ``base64`` is - requested for *subtype* ``rfc822``, and for any *cte* other than - ``7bit`` for *subtype* ``external-body``. For - ``message/rfc822``, use ``8bit`` if *cte* is not specified. For - all other values of *subtype*, use ``7bit``. - - .. note:: A *cte* of ``binary`` does not actually work correctly yet. - The ``EmailMessage`` object as modified by ``set_content`` is - correct, but :class:`~email.generator.BytesGenerator` does not - serialize it correctly. - - If *disposition* is set, use it as the value of the - :mailheader:`Content-Disposition` header. If not specified, and - *filename* is specified, add the header with the value ``attachment``. - If *disposition* is not specified and *filename* is also not specified, - do not add the header. The only valid values for *disposition* are - ``attachment`` and ``inline``. - - If *filename* is specified, use it as the value of the ``filename`` - parameter of the :mailheader:`Content-Disposition` header. - - If *cid* is specified, add a :mailheader:`Content-ID` header with - *cid* as its value. - - If *params* is specified, iterate its ``items`` method and use the - resulting ``(key, value)`` pairs to set additional parameters on the - :mailheader:`Content-Type` header. - - If *headers* is specified and is a list of strings of the form - ``headername: headervalue`` or a list of ``header`` objects - (distinguished from strings by having a ``name`` attribute), add the - headers to *msg*. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.4 as a :term:`provisional module ` diff --git a/third_party/python/Doc/library/email.encoders.rst b/third_party/python/Doc/library/email.encoders.rst deleted file mode 100644 index e24ac7b90..000000000 --- a/third_party/python/Doc/library/email.encoders.rst +++ /dev/null @@ -1,70 +0,0 @@ -:mod:`email.encoders`: Encoders -------------------------------- - -.. module:: email.encoders - :synopsis: Encoders for email message payloads. - -**Source code:** :source:`Lib/email/encoders.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the -new API the functionality is provided by the *cte* parameter of -the :meth:`~email.message.EmailMessage.set_content` method. - -The remaining text in this section is the original documentation of the module. - -When creating :class:`~email.message.Message` objects from scratch, you often -need to encode the payloads for transport through compliant mail servers. This -is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type messages -containing binary data. - -The :mod:`email` package provides some convenient encodings in its -:mod:`encoders` module. These encoders are actually used by the -:class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` -class constructors to provide default encodings. All encoder functions take -exactly one argument, the message object to encode. They usually extract the -payload, encode it, and reset the payload to this newly encoded value. They -should also set the :mailheader:`Content-Transfer-Encoding` header as appropriate. - -Note that these functions are not meaningful for a multipart message. They -must be applied to individual subparts instead, and will raise a -:exc:`TypeError` if passed a message whose type is multipart. - -Here are the encoding functions provided: - - -.. function:: encode_quopri(msg) - - Encodes the payload into quoted-printable form and sets the - :mailheader:`Content-Transfer-Encoding` header to ``quoted-printable`` [#]_. - This is a good encoding to use when most of your payload is normal printable - data, but contains a few unprintable characters. - - -.. function:: encode_base64(msg) - - Encodes the payload into base64 form and sets the - :mailheader:`Content-Transfer-Encoding` header to ``base64``. This is a good - encoding to use when most of your payload is unprintable data since it is a more - compact form than quoted-printable. The drawback of base64 encoding is that it - renders the text non-human readable. - - -.. function:: encode_7or8bit(msg) - - This doesn't actually modify the message's payload, but it does set the - :mailheader:`Content-Transfer-Encoding` header to either ``7bit`` or ``8bit`` as - appropriate, based on the payload data. - - -.. function:: encode_noop(msg) - - This does nothing; it doesn't even set the - :mailheader:`Content-Transfer-Encoding` header. - -.. rubric:: Footnotes - -.. [#] Note that encoding with :meth:`encode_quopri` also encodes all tabs and space - characters in the data. - diff --git a/third_party/python/Doc/library/email.errors.rst b/third_party/python/Doc/library/email.errors.rst deleted file mode 100644 index 511ad1635..000000000 --- a/third_party/python/Doc/library/email.errors.rst +++ /dev/null @@ -1,114 +0,0 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- - -.. module:: email.errors - :synopsis: The exception classes used by the email package. - -**Source code:** :source:`Lib/email/errors.py` - --------------- - -The following exception classes are defined in the :mod:`email.errors` module: - - -.. exception:: MessageError() - - This is the base class for all exceptions that the :mod:`email` package can - raise. It is derived from the standard :exc:`Exception` class and defines no - additional methods. - - -.. exception:: MessageParseError() - - This is the base class for exceptions raised by the - :class:`~email.parser.Parser` class. It is derived from - :exc:`MessageError`. This class is also used internally by the parser used - by :mod:`~email.headerregistry`. - - -.. exception:: HeaderParseError() - - Raised under some error conditions when parsing the :rfc:`5322` headers of a - message, this class is derived from :exc:`MessageParseError`. The - :meth:`~email.message.EmailMessage.set_boundary` method will raise this - error if the content type is unknown when the method is called. - :class:`~email.header.Header` may raise this error for certain base64 - decoding errors, and when an attempt is made to create a header that appears - to contain an embedded header (that is, there is what is supposed to be a - continuation line that has no leading whitespace and looks like a header). - - -.. exception:: BoundaryError() - - Deprecated and no longer used. - - -.. exception:: MultipartConversionError() - - Raised when a payload is added to a :class:`~email.message.Message` object - using :meth:`add_payload`, but the payload is already a scalar and the - message's :mailheader:`Content-Type` main type is not either - :mimetype:`multipart` or missing. :exc:`MultipartConversionError` multiply - inherits from :exc:`MessageError` and the built-in :exc:`TypeError`. - - Since :meth:`Message.add_payload` is deprecated, this exception is rarely - raised in practice. However the exception may also be raised if the - :meth:`~email.message.Message.attach` - method is called on an instance of a class derived from - :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. - :class:`~email.mime.image.MIMEImage`). - - -Here is the list of the defects that the :class:`~email.parser.FeedParser` -can find while parsing messages. Note that the defects are added to the message -where the problem was found, so for example, if a message nested inside a -:mimetype:`multipart/alternative` had a malformed header, that nested message -object would have a defect, but the containing messages would not. - -All defect classes are subclassed from :class:`email.errors.MessageDefect`. - -* :class:`NoBoundaryInMultipartDefect` -- A message claimed to be a multipart, - but had no :mimetype:`boundary` parameter. - -* :class:`StartBoundaryNotFoundDefect` -- The start boundary claimed in the - :mailheader:`Content-Type` header was never found. - -* :class:`CloseBoundaryNotFoundDefect` -- A start boundary was found, but - no corresponding close boundary was ever found. - - .. versionadded:: 3.3 - -* :class:`FirstHeaderLineIsContinuationDefect` -- The message had a continuation - line as its first header line. - -* :class:`MisplacedEnvelopeHeaderDefect` - A "Unix From" header was found in the - middle of a header block. - -* :class:`MissingHeaderBodySeparatorDefect` - A line was found while parsing - headers that had no leading white space but contained no ':'. Parsing - continues assuming that the line represents the first line of the body. - - .. versionadded:: 3.3 - -* :class:`MalformedHeaderDefect` -- A header was found that was missing a colon, - or was otherwise malformed. - - .. deprecated:: 3.3 - This defect has not been used for several Python versions. - -* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a - :mimetype:`multipart`, but no subparts were found. Note that when a message - has this defect, its :meth:`~email.message.Message.is_multipart` method may - return false even though its content type claims to be :mimetype:`multipart`. - -* :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - encoded bytes, the padding was not correct. Enough padding is added to - perform the decode, but the resulting decoded bytes may be invalid. - -* :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - encoded bytes, characters outside the base64 alphabet were encountered. - The characters are ignored, but the resulting decoded bytes may be invalid. - -* :class:`InvalidBase64LengthDefect` -- When decoding a block of base64 encoded - bytes, the number of non-padding base64 characters was invalid (1 more than - a multiple of 4). The encoded block was kept as-is. diff --git a/third_party/python/Doc/library/email.examples.rst b/third_party/python/Doc/library/email.examples.rst deleted file mode 100644 index fc9646228..000000000 --- a/third_party/python/Doc/library/email.examples.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. _email-examples: - -:mod:`email`: Examples ----------------------- - -Here are a few examples of how to use the :mod:`email` package to read, write, -and send simple email messages, as well as more complex MIME messages. - -First, let's see how to create and send a simple text message (both the -text content and the addresses may contain unicode characters): - -.. literalinclude:: ../includes/email-simple.py - - -Parsing :rfc:`822` headers can easily be done by the using the classes -from the :mod:`~email.parser` module: - -.. literalinclude:: ../includes/email-headers.py - - -Here's an example of how to send a MIME message containing a bunch of family -pictures that may be residing in a directory: - -.. literalinclude:: ../includes/email-mime.py - - -Here's an example of how to send the entire contents of a directory as an email -message: [1]_ - -.. literalinclude:: ../includes/email-dir.py - - -Here's an example of how to unpack a MIME message like the one -above, into a directory of files: - -.. literalinclude:: ../includes/email-unpack.py - - -Here's an example of how to create an HTML message with an alternative plain -text version. To make things a bit more interesting, we include a related -image in the html part, and we save a copy of what we are going to send to -disk, as well as sending it. - -.. literalinclude:: ../includes/email-alternative.py - - -If we were sent the message from the last example, here is one way we could -process it: - -.. literalinclude:: ../includes/email-read-alternative.py - -Up to the prompt, the output from the above is: - -.. code-block:: none - - To: Penelope Pussycat , Fabrette Pussycat - From: Pepé Le Pew - Subject: Ayons asperges pour le déjeuner - - Salut! - - Cela ressemble à un excellent recipie[1] déjeuner. - - -.. rubric:: Footnotes - -.. [1] Thanks to Matthew Dixon Cowles for the original inspiration and examples. diff --git a/third_party/python/Doc/library/email.generator.rst b/third_party/python/Doc/library/email.generator.rst deleted file mode 100644 index b4ed3b41f..000000000 --- a/third_party/python/Doc/library/email.generator.rst +++ /dev/null @@ -1,279 +0,0 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- - -.. module:: email.generator - :synopsis: Generate flat text email messages from a message structure. - -**Source code:** :source:`Lib/email/generator.py` - --------------- - -One of the most common tasks is to generate the flat (serialized) version of -the email message represented by a message object structure. You will need to -do this if you want to send your message via :meth:`smtplib.SMTP.sendmail` or -the :mod:`nntplib` module, or print the message on the console. Taking a -message object structure and producing a serialized representation is the job -of the generator classes. - -As with the :mod:`email.parser` module, you aren't limited to the functionality -of the bundled generator; you could write one from scratch yourself. However -the bundled generator knows how to generate most email in a standards-compliant -way, should handle MIME and non-MIME email messages just fine, and is designed -so that the bytes-oriented parsing and generation operations are inverses, -assuming the same non-transforming :mod:`~email.policy` is used for both. That -is, parsing the serialized byte stream via the -:class:`~email.parser.BytesParser` class and then regenerating the serialized -byte stream using :class:`BytesGenerator` should produce output identical to -the input [#]_. (On the other hand, using the generator on an -:class:`~email.message.EmailMessage` constructed by program may result in -changes to the :class:`~email.message.EmailMessage` object as defaults are -filled in.) - -The :class:`Generator` class can be used to flatten a message into a text (as -opposed to binary) serialized representation, but since Unicode cannot -represent binary data directly, the message is of necessity transformed into -something that contains only ASCII characters, using the standard email RFC -Content Transfer Encoding techniques for encoding email messages for transport -over channels that are not "8 bit clean". - - -.. class:: BytesGenerator(outfp, mangle_from_=None, maxheaderlen=None, *, \ - policy=None) - - Return a :class:`BytesGenerator` object that will write any message provided - to the :meth:`flatten` method, or any surrogateescape encoded text provided - to the :meth:`write` method, to the :term:`file-like object` *outfp*. - *outfp* must support a ``write`` method that accepts binary data. - - If optional *mangle_from_* is ``True``, put a ``>`` character in front of - any line in the body that starts with the exact string ``"From "``, that is - ``From`` followed by a space at the beginning of a line. *mangle_from_* - defaults to the value of the :attr:`~email.policy.Policy.mangle_from_` - setting of the *policy* (which is ``True`` for the - :data:`~email.policy.compat32` policy and ``False`` for all others). - *mangle_from_* is intended for use when messages are stored in unix mbox - format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). - - If *maxheaderlen* is not ``None``, refold any header lines that are longer - than *maxheaderlen*, or if ``0``, do not rewrap any headers. If - *manheaderlen* is ``None`` (the default), wrap headers and other message - lines according to the *policy* settings. - - If *policy* is specified, use that policy to control message generation. If - *policy* is ``None`` (the default), use the policy associated with the - :class:`~email.message.Message` or :class:`~email.message.EmailMessage` - object passed to ``flatten`` to control the message generation. See - :mod:`email.policy` for details on what *policy* controls. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 Added the *policy* keyword. - - .. versionchanged:: 3.6 The default behavior of the *mangle_from_* - and *maxheaderlen* parameters is to follow the policy. - - - .. method:: flatten(msg, unixfrom=False, linesep=None) - - Print the textual representation of the message object structure rooted - at *msg* to the output file specified when the :class:`BytesGenerator` - instance was created. - - If the :mod:`~email.policy` option :attr:`~email.policy.Policy.cte_type` - is ``8bit`` (the default), copy any headers in the original parsed - message that have not been modified to the output with any bytes with the - high bit set reproduced as in the original, and preserve the non-ASCII - :mailheader:`Content-Transfer-Encoding` of any body parts that have them. - If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as - needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. - That is, transform parts with non-ASCII - :mailheader:`Content-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible - :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII - bytes in headers using the MIME ``unknown-8bit`` character set, thus - rendering them RFC-compliant. - - .. XXX: There should be an option that just does the RFC - compliance transformation on headers but leaves CTE 8bit parts alone. - - If *unixfrom* is ``True``, print the envelope header delimiter used by - the Unix mailbox format (see :mod:`mailbox`) before the first of the - :rfc:`5322` headers of the root message object. If the root object has - no envelope header, craft a standard one. The default is ``False``. - Note that for subparts, no envelope header is ever printed. - - If *linesep* is not ``None``, use it as the separator character between - all the lines of the flattened message. If *linesep* is ``None`` (the - default), use the value specified in the *policy*. - - .. XXX: flatten should take a *policy* keyword. - - - .. method:: clone(fp) - - Return an independent clone of this :class:`BytesGenerator` instance with - the exact same option settings, and *fp* as the new *outfp*. - - - .. method:: write(s) - - Encode *s* using the ``ASCII`` codec and the ``surrogateescape`` error - handler, and pass it to the *write* method of the *outfp* passed to the - :class:`BytesGenerator`'s constructor. - - -As a convenience, :class:`~email.message.EmailMessage` provides the methods -:meth:`~email.message.EmailMessage.as_bytes` and ``bytes(aMessage)`` (a.k.a. -:meth:`~email.message.EmailMessage.__bytes__`), which simplify the generation of -a serialized binary representation of a message object. For more detail, see -:mod:`email.message`. - - -Because strings cannot represent binary data, the :class:`Generator` class must -convert any binary data in any message it flattens to an ASCII compatible -format, by converting them to an ASCII compatible -:mailheader:`Content-Transfer_Encoding`. Using the terminology of the email -RFCs, you can think of this as :class:`Generator` serializing to an I/O stream -that is not "8 bit clean". In other words, most applications will want -to be using :class:`BytesGenerator`, and not :class:`Generator`. - -.. class:: Generator(outfp, mangle_from_=None, maxheaderlen=None, *, \ - policy=None) - - Return a :class:`Generator` object that will write any message provided - to the :meth:`flatten` method, or any text provided to the :meth:`write` - method, to the :term:`file-like object` *outfp*. *outfp* must support a - ``write`` method that accepts string data. - - If optional *mangle_from_* is ``True``, put a ``>`` character in front of - any line in the body that starts with the exact string ``"From "``, that is - ``From`` followed by a space at the beginning of a line. *mangle_from_* - defaults to the value of the :attr:`~email.policy.Policy.mangle_from_` - setting of the *policy* (which is ``True`` for the - :data:`~email.policy.compat32` policy and ``False`` for all others). - *mangle_from_* is intended for use when messages are stored in unix mbox - format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). - - If *maxheaderlen* is not ``None``, refold any header lines that are longer - than *maxheaderlen*, or if ``0``, do not rewrap any headers. If - *manheaderlen* is ``None`` (the default), wrap headers and other message - lines according to the *policy* settings. - - If *policy* is specified, use that policy to control message generation. If - *policy* is ``None`` (the default), use the policy associated with the - :class:`~email.message.Message` or :class:`~email.message.EmailMessage` - object passed to ``flatten`` to control the message generation. See - :mod:`email.policy` for details on what *policy* controls. - - .. versionchanged:: 3.3 Added the *policy* keyword. - - .. versionchanged:: 3.6 The default behavior of the *mangle_from_* - and *maxheaderlen* parameters is to follow the policy. - - - .. method:: flatten(msg, unixfrom=False, linesep=None) - - Print the textual representation of the message object structure rooted - at *msg* to the output file specified when the :class:`Generator` - instance was created. - - If the :mod:`~email.policy` option :attr:`~email.policy.Policy.cte_type` - is ``8bit``, generate the message as if the option were set to ``7bit``. - (This is required because strings cannot represent non-ASCII bytes.) - Convert any bytes with the high bit set as needed using an - ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, - transform parts with non-ASCII :mailheader:`Cotnent-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible - :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII - bytes in headers using the MIME ``unknown-8bit`` character set, thus - rendering them RFC-compliant. - - If *unixfrom* is ``True``, print the envelope header delimiter used by - the Unix mailbox format (see :mod:`mailbox`) before the first of the - :rfc:`5322` headers of the root message object. If the root object has - no envelope header, craft a standard one. The default is ``False``. - Note that for subparts, no envelope header is ever printed. - - If *linesep* is not ``None``, use it as the separator character between - all the lines of the flattened message. If *linesep* is ``None`` (the - default), use the value specified in the *policy*. - - .. XXX: flatten should take a *policy* keyword. - - .. versionchanged:: 3.2 - Added support for re-encoding ``8bit`` message bodies, and the - *linesep* argument. - - - .. method:: clone(fp) - - Return an independent clone of this :class:`Generator` instance with the - exact same options, and *fp* as the new *outfp*. - - - .. method:: write(s) - - Write *s* to the *write* method of the *outfp* passed to the - :class:`Generator`'s constructor. This provides just enough file-like - API for :class:`Generator` instances to be used in the :func:`print` - function. - - -As a convenience, :class:`~email.message.EmailMessage` provides the methods -:meth:`~email.message.EmailMessage.as_string` and ``str(aMessage)`` (a.k.a. -:meth:`~email.message.EmailMessage.__str__`), which simplify the generation of -a formatted string representation of a message object. For more detail, see -:mod:`email.message`. - - -The :mod:`email.generator` module also provides a derived class, -:class:`DecodedGenerator`, which is like the :class:`Generator` base class, -except that non-\ :mimetype:`text` parts are not serialized, but are instead -represented in the output stream by a string derived from a template filled -in with information about the part. - -.. class:: DecodedGenerator(outfp, mangle_from_=None, maxheaderlen=None, \ - fmt=None, *, policy=None) - - Act like :class:`Generator`, except that for any subpart of the message - passed to :meth:`Generator.flatten`, if the subpart is of main type - :mimetype:`text`, print the decoded payload of the subpart, and if the main - type is not :mimetype:`text`, instead of printing it fill in the string - *fmt* using information from the part and print the resulting - filled-in string. - - To fill in *fmt*, execute ``fmt % part_info``, where ``part_info`` - is a dictionary composed of the following keys and values: - - * ``type`` -- Full MIME type of the non-\ :mimetype:`text` part - - * ``maintype`` -- Main MIME type of the non-\ :mimetype:`text` part - - * ``subtype`` -- Sub-MIME type of the non-\ :mimetype:`text` part - - * ``filename`` -- Filename of the non-\ :mimetype:`text` part - - * ``description`` -- Description associated with the non-\ :mimetype:`text` part - - * ``encoding`` -- Content transfer encoding of the non-\ :mimetype:`text` part - - If *fmt* is ``None``, use the following default *fmt*: - - "[Non-text (%(type)s) part of message omitted, filename %(filename)s]" - - Optional *_mangle_from_* and *maxheaderlen* are as with the - :class:`Generator` base class. - - -.. rubric:: Footnotes - -.. [#] This statement assumes that you use the appropriate setting for - ``unixfrom``, and that there are no :mod:`policy` settings calling for - automatic adjustments (for example, - :attr:`~email.policy.Policy.refold_source` must be ``none``, which is - *not* the default). It is also not 100% true, since if the message - does not conform to the RFC standards occasionally information about the - exact original text is lost during parsing error recovery. It is a goal - to fix these latter edge cases when possible. diff --git a/third_party/python/Doc/library/email.header.rst b/third_party/python/Doc/library/email.header.rst deleted file mode 100644 index 07152c224..000000000 --- a/third_party/python/Doc/library/email.header.rst +++ /dev/null @@ -1,205 +0,0 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- - -.. module:: email.header - :synopsis: Representing non-ASCII headers - -**Source code:** :source:`Lib/email/header.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the current API -encoding and decoding of headers is handled transparently by the -dictionary-like API of the :class:`~email.message.EmailMessage` class. In -addition to uses in legacy code, this module can be useful in applications that -need to completely control the character sets used when encoding headers. - -The remaining text in this section is the original documentation of the module. - -:rfc:`2822` is the base standard that describes the format of email messages. -It derives from the older :rfc:`822` standard which came into widespread use at -a time when most email was composed of ASCII characters only. :rfc:`2822` is a -specification written assuming email contains only 7-bit ASCII characters. - -Of course, as email has been deployed worldwide, it has become -internationalized, such that language specific character sets can now be used in -email messages. The base standard still requires email messages to be -transferred using only 7-bit ASCII characters, so a slew of RFCs have been -written describing how to encode email containing non-ASCII characters into -:rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, -:rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. - -If you want to include non-ASCII characters in your email headers, say in the -:mailheader:`Subject` or :mailheader:`To` fields, you should use the -:class:`Header` class and assign the field in the :class:`~email.message.Message` -object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. -For example:: - - >>> from email.message import Message - >>> from email.header import Header - >>> msg = Message() - >>> h = Header('p\xf6stal', 'iso-8859-1') - >>> msg['Subject'] = h - >>> msg.as_string() - 'Subject: =?iso-8859-1?q?p=F6stal?=\n\n' - - - -Notice here how we wanted the :mailheader:`Subject` field to contain a non-ASCII -character? We did this by creating a :class:`Header` instance and passing in -the character set that the byte string was encoded in. When the subsequent -:class:`~email.message.Message` instance was flattened, the :mailheader:`Subject` -field was properly :rfc:`2047` encoded. MIME-aware mail readers would show this -header using the embedded ISO-8859-1 character. - -Here is the :class:`Header` class description: - - -.. class:: Header(s=None, charset=None, maxlinelen=None, header_name=None, continuation_ws=' ', errors='strict') - - Create a MIME-compliant header that can contain strings in different character - sets. - - Optional *s* is the initial header value. If ``None`` (the default), the - initial header value is not set. You can later append to the header with - :meth:`append` method calls. *s* may be an instance of :class:`bytes` or - :class:`str`, but see the :meth:`append` documentation for semantics. - - Optional *charset* serves two purposes: it has the same meaning as the *charset* - argument to the :meth:`append` method. It also sets the default character set - for all subsequent :meth:`append` calls that omit the *charset* argument. If - *charset* is not provided in the constructor (the default), the ``us-ascii`` - character set is used both as *s*'s initial charset and as the default for - subsequent :meth:`append` calls. - - The maximum line length can be specified explicitly via *maxlinelen*. For - splitting the first line to a shorter value (to account for the field header - which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value - for *header_name* is ``None``, meaning it is not taken into account for the - first line of a long, split header. - - Optional *continuation_ws* must be :rfc:`2822`\ -compliant folding - whitespace, and is usually either a space or a hard tab character. This - character will be prepended to continuation lines. *continuation_ws* - defaults to a single space character. - - Optional *errors* is passed straight through to the :meth:`append` method. - - - .. method:: append(s, charset=None, errors='strict') - - Append the string *s* to the MIME header. - - Optional *charset*, if given, should be a :class:`~email.charset.Charset` - instance (see :mod:`email.charset`) or the name of a character set, which - will be converted to a :class:`~email.charset.Charset` instance. A value - of ``None`` (the default) means that the *charset* given in the constructor - is used. - - *s* may be an instance of :class:`bytes` or :class:`str`. If it is an - instance of :class:`bytes`, then *charset* is the encoding of that byte - string, and a :exc:`UnicodeError` will be raised if the string cannot be - decoded with that character set. - - If *s* is an instance of :class:`str`, then *charset* is a hint specifying - the character set of the characters in the string. - - In either case, when producing an :rfc:`2822`\ -compliant header using - :rfc:`2047` rules, the string will be encoded using the output codec of - the charset. If the string cannot be encoded using the output codec, a - UnicodeError will be raised. - - Optional *errors* is passed as the errors argument to the decode call - if *s* is a byte string. - - - .. method:: encode(splitchars=';, \\t', maxlinelen=None, linesep='\\n') - - Encode a message header into an RFC-compliant format, possibly wrapping - long lines and encapsulating non-ASCII parts in base64 or quoted-printable - encodings. - - Optional *splitchars* is a string containing characters which should be - given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of :RFC:`2822`\'s 'higher level - syntactic breaks': split points preceded by a splitchar are preferred - during line splitting, with the characters preferred in the order in - which they appear in the string. Space and tab may be included in the - string to indicate whether preference should be given to one over the - other as a split point when other split chars do not appear in the line - being split. Splitchars does not affect :RFC:`2047` encoded lines. - - *maxlinelen*, if given, overrides the instance's value for the maximum - line length. - - *linesep* specifies the characters used to separate the lines of the - folded header. It defaults to the most useful value for Python - application code (``\n``), but ``\r\n`` can be specified in order - to produce headers with RFC-compliant line separators. - - .. versionchanged:: 3.2 - Added the *linesep* argument. - - - The :class:`Header` class also provides a number of methods to support - standard operators and built-in functions. - - .. method:: __str__() - - Returns an approximation of the :class:`Header` as a string, using an - unlimited line length. All pieces are converted to unicode using the - specified encoding and joined together appropriately. Any pieces with a - charset of ``'unknown-8bit'`` are decoded as ASCII using the ``'replace'`` - error handler. - - .. versionchanged:: 3.2 - Added handling for the ``'unknown-8bit'`` charset. - - - .. method:: __eq__(other) - - This method allows you to compare two :class:`Header` instances for - equality. - - - .. method:: __ne__(other) - - This method allows you to compare two :class:`Header` instances for - inequality. - -The :mod:`email.header` module also provides the following convenient functions. - - -.. function:: decode_header(header) - - Decode a message header value without converting the character set. The header - value is in *header*. - - This function returns a list of ``(decoded_string, charset)`` pairs containing - each of the decoded parts of the header. *charset* is ``None`` for non-encoded - parts of the header, otherwise a lower case string containing the name of the - character set specified in the encoded string. - - Here's an example:: - - >>> from email.header import decode_header - >>> decode_header('=?iso-8859-1?q?p=F6stal?=') - [(b'p\xf6stal', 'iso-8859-1')] - - -.. function:: make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' ') - - Create a :class:`Header` instance from a sequence of pairs as returned by - :func:`decode_header`. - - :func:`decode_header` takes a header value string and returns a sequence of - pairs of the format ``(decoded_string, charset)`` where *charset* is the name of - the character set. - - This function takes one of those sequence of pairs and returns a - :class:`Header` instance. Optional *maxlinelen*, *header_name*, and - *continuation_ws* are as in the :class:`Header` constructor. - diff --git a/third_party/python/Doc/library/email.headerregistry.rst b/third_party/python/Doc/library/email.headerregistry.rst deleted file mode 100644 index ce283c6b5..000000000 --- a/third_party/python/Doc/library/email.headerregistry.rst +++ /dev/null @@ -1,455 +0,0 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- - -.. module:: email.headerregistry - :synopsis: Automatic Parsing of headers based on the field name - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/headerregistry.py` - --------------- - -.. versionadded:: 3.6 [1]_ - -Headers are represented by customized subclasses of :class:`str`. The -particular class used to represent a given header is determined by the -:attr:`~email.policy.EmailPolicy.header_factory` of the :mod:`~email.policy` in -effect when the headers are created. This section documents the particular -``header_factory`` implemented by the email package for handling :RFC:`5322` -compliant email messages, which not only provides customized header objects for -various header types, but also provides an extension mechanism for applications -to add their own custom header types. - -When using any of the policy objects derived from -:data:`~email.policy.EmailPolicy`, all headers are produced by -:class:`.HeaderRegistry` and have :class:`.BaseHeader` as their last base -class. Each header class has an additional base class that is determined by -the type of the header. For example, many headers have the class -:class:`.UnstructuredHeader` as their other base class. The specialized second -class for a header is determined by the name of the header, using a lookup -table stored in the :class:`.HeaderRegistry`. All of this is managed -transparently for the typical application program, but interfaces are provided -for modifying the default behavior for use by more complex applications. - -The sections below first document the header base classes and their attributes, -followed by the API for modifying the behavior of :class:`.HeaderRegistry`, and -finally the support classes used to represent the data parsed from structured -headers. - - -.. class:: BaseHeader(name, value) - - *name* and *value* are passed to ``BaseHeader`` from the - :attr:`~email.policy.EmailPolicy.header_factory` call. The string value of - any header object is the *value* fully decoded to unicode. - - This base class defines the following read-only properties: - - - .. attribute:: name - - The name of the header (the portion of the field before the ':'). This - is exactly the value passed in the - :attr:`~email.policy.EmailPolicy.header_factory` call for *name*; that - is, case is preserved. - - - .. attribute:: defects - - A tuple of :exc:`~email.errors.HeaderDefect` instances reporting any - RFC compliance problems found during parsing. The email package tries to - be complete about detecting compliance issues. See the :mod:`~email.errors` - module for a discussion of the types of defects that may be reported. - - - .. attribute:: max_count - - The maximum number of headers of this type that can have the same - ``name``. A value of ``None`` means unlimited. The ``BaseHeader`` value - for this attribute is ``None``; it is expected that specialized header - classes will override this value as needed. - - ``BaseHeader`` also provides the following method, which is called by the - email library code and should not in general be called by application - programs: - - .. method:: fold(*, policy) - - Return a string containing :attr:`~email.policy.Policy.linesep` - characters as required to correctly fold the header according to - *policy*. A :attr:`~email.policy.Policy.cte_type` of ``8bit`` will be - treated as if it were ``7bit``, since headers may not contain arbitrary - binary data. If :attr:`~email.policy.EmailPolicy.utf8` is ``False``, - non-ASCII data will be :rfc:`2047` encoded. - - - ``BaseHeader`` by itself cannot be used to create a header object. It - defines a protocol that each specialized header cooperates with in order to - produce the header object. Specifically, ``BaseHeader`` requires that - the specialized class provide a :func:`classmethod` named ``parse``. This - method is called as follows:: - - parse(string, kwds) - - ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. - ``defects`` is an empty list. The parse method should append any detected - defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may - contain content-transfer-encoded parts, but should correctly handle all valid - unicode characters as well so that it can parse un-encoded header values. - - ``BaseHeader``'s ``__new__`` then creates the header instance, and calls its - ``init`` method. The specialized class only needs to provide an ``init`` - method if it wishes to set additional attributes beyond those provided by - ``BaseHeader`` itself. Such an ``init`` method should look like this:: - - def init(self, *args, **kw): - self._myattr = kw.pop('myattr') - super().init(*args, **kw) - - That is, anything extra that the specialized class puts in to the ``kwds`` - dictionary should be removed and handled, and the remaining contents of - ``kw`` (and ``args``) passed to the ``BaseHeader`` ``init`` method. - - -.. class:: UnstructuredHeader - - An "unstructured" header is the default type of header in :rfc:`5322`. - Any header that does not have a specified syntax is treated as - unstructured. The classic example of an unstructured header is the - :mailheader:`Subject` header. - - In :rfc:`5322`, an unstructured header is a run of arbitrary text in the - ASCII character set. :rfc:`2047`, however, has an :rfc:`5322` compatible - mechanism for encoding non-ASCII text as ASCII characters within a header - value. When a *value* containing encoded words is passed to the - constructor, the ``UnstructuredHeader`` parser converts such encoded words - into unicode, following the :rfc:`2047` rules for unstructured text. The - parser uses heuristics to attempt to decode certain non-compliant encoded - words. Defects are registered in such cases, as well as defects for issues - such as invalid characters within the encoded words or the non-encoded text. - - This header type provides no additional attributes. - - -.. class:: DateHeader - - :rfc:`5322` specifies a very specific format for dates within email headers. - The ``DateHeader`` parser recognizes that date format, as well as - recognizing a number of variant forms that are sometimes found "in the - wild". - - This header type provides the following additional attributes: - - .. attribute:: datetime - - If the header value can be recognized as a valid date of one form or - another, this attribute will contain a :class:`~datetime.datetime` - instance representing that date. If the timezone of the input date is - specified as ``-0000`` (indicating it is in UTC but contains no - information about the source timezone), then :attr:`.datetime` will be a - naive :class:`~datetime.datetime`. If a specific timezone offset is - found (including `+0000`), then :attr:`.datetime` will contain an aware - ``datetime`` that uses :class:`datetime.timezone` to record the timezone - offset. - - The ``decoded`` value of the header is determined by formatting the - ``datetime`` according to the :rfc:`5322` rules; that is, it is set to:: - - email.utils.format_datetime(self.datetime) - - When creating a ``DateHeader``, *value* may be - :class:`~datetime.datetime` instance. This means, for example, that - the following code is valid and does what one would expect:: - - msg['Date'] = datetime(2011, 7, 15, 21) - - Because this is a naive ``datetime`` it will be interpreted as a UTC - timestamp, and the resulting value will have a timezone of ``-0000``. Much - more useful is to use the :func:`~email.utils.localtime` function from the - :mod:`~email.utils` module:: - - msg['Date'] = utils.localtime() - - This example sets the date header to the current time and date using - the current timezone offset. - - -.. class:: AddressHeader - - Address headers are one of the most complex structured header types. - The ``AddressHeader`` class provides a generic interface to any address - header. - - This header type provides the following additional attributes: - - - .. attribute:: groups - - A tuple of :class:`.Group` objects encoding the - addresses and groups found in the header value. Addresses that are - not part of a group are represented in this list as single-address - ``Groups`` whose :attr:`~.Group.display_name` is ``None``. - - - .. attribute:: addresses - - A tuple of :class:`.Address` objects encoding all - of the individual addresses from the header value. If the header value - contains any groups, the individual addresses from the group are included - in the list at the point where the group occurs in the value (that is, - the list of addresses is "flattened" into a one dimensional list). - - The ``decoded`` value of the header will have all encoded words decoded to - unicode. :class:`~encodings.idna` encoded domain names are also decoded to - unicode. The ``decoded`` value is set by :attr:`~str.join`\ ing the - :class:`str` value of the elements of the ``groups`` attribute with ``', - '``. - - A list of :class:`.Address` and :class:`.Group` objects in any combination - may be used to set the value of an address header. ``Group`` objects whose - ``display_name`` is ``None`` will be interpreted as single addresses, which - allows an address list to be copied with groups intact by using the list - obtained from the ``groups`` attribute of the source header. - - -.. class:: SingleAddressHeader - - A subclass of :class:`.AddressHeader` that adds one - additional attribute: - - - .. attribute:: address - - The single address encoded by the header value. If the header value - actually contains more than one address (which would be a violation of - the RFC under the default :mod:`~email.policy`), accessing this attribute - will result in a :exc:`ValueError`. - - -Many of the above classes also have a ``Unique`` variant (for example, -``UniqueUnstructuredHeader``). The only difference is that in the ``Unique`` -variant, :attr:`~.BaseHeader.max_count` is set to 1. - - -.. class:: MIMEVersionHeader - - There is really only one valid value for the :mailheader:`MIME-Version` - header, and that is ``1.0``. For future proofing, this header class - supports other valid version numbers. If a version number has a valid value - per :rfc:`2045`, then the header object will have non-``None`` values for - the following attributes: - - .. attribute:: version - - The version number as a string, with any whitespace and/or comments - removed. - - .. attribute:: major - - The major version number as an integer - - .. attribute:: minor - - The minor version number as an integer - - -.. class:: ParameterizedMIMEHeader - - MIME headers all start with the prefix 'Content-'. Each specific header has - a certain value, described under the class for that header. Some can - also take a list of supplemental parameters, which have a common format. - This class serves as a base for all the MIME headers that take parameters. - - .. attribute:: params - - A dictionary mapping parameter names to parameter values. - - -.. class:: ContentTypeHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Type` header. - - .. attribute:: content_type - - The content type string, in the form ``maintype/subtype``. - - .. attribute:: maintype - - .. attribute:: subtype - - -.. class:: ContentDispositionHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Disposition` header. - - .. attribute:: content-disposition - - ``inline`` and ``attachment`` are the only valid values in common use. - - -.. class:: ContentTransferEncoding - - Handles the :mailheader:`Content-Transfer-Encoding` header. - - .. attribute:: cte - - Valid values are ``7bit``, ``8bit``, ``base64``, and - ``quoted-printable``. See :rfc:`2045` for more information. - - - -.. class:: HeaderRegistry(base_class=BaseHeader, \ - default_class=UnstructuredHeader, \ - use_default_map=True) - - This is the factory used by :class:`~email.policy.EmailPolicy` by default. - ``HeaderRegistry`` builds the class used to create a header instance - dynamically, using *base_class* and a specialized class retrieved from a - registry that it holds. When a given header name does not appear in the - registry, the class specified by *default_class* is used as the specialized - class. When *use_default_map* is ``True`` (the default), the standard - mapping of header names to classes is copied in to the registry during - initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. - - The default mappings are: - - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader - - ``HeaderRegistry`` has the following methods: - - - .. method:: map_to_type(self, name, cls) - - *name* is the name of the header to be mapped. It will be converted to - lower case in the registry. *cls* is the specialized class to be used, - along with *base_class*, to create the class used to instantiate headers - that match *name*. - - - .. method:: __getitem__(name) - - Construct and return a class to handle creating a *name* header. - - - .. method:: __call__(name, value) - - Retrieves the specialized header associated with *name* from the - registry (using *default_class* if *name* does not appear in the - registry) and composes it with *base_class* to produce a class, - calls the constructed class's constructor, passing it the same - argument list, and finally returns the class instance created thereby. - - -The following classes are the classes used to represent data parsed from -structured headers and can, in general, be used by an application program to -construct structured values to assign to specific headers. - - -.. class:: Address(display_name='', username='', domain='', addr_spec=None) - - The class used to represent an email address. The general form of an - address is:: - - [display_name] - - or:: - - username@domain - - where each part must conform to specific syntax rules spelled out in - :rfc:`5322`. - - As a convenience *addr_spec* can be specified instead of *username* and - *domain*, in which case *username* and *domain* will be parsed from the - *addr_spec*. An *addr_spec* must be a properly RFC quoted string; if it is - not ``Address`` will raise an error. Unicode characters are allowed and - will be property encoded when serialized. However, per the RFCs, unicode is - *not* allowed in the username portion of the address. - - .. attribute:: display_name - - The display name portion of the address, if any, with all quoting - removed. If the address does not have a display name, this attribute - will be an empty string. - - .. attribute:: username - - The ``username`` portion of the address, with all quoting removed. - - .. attribute:: domain - - The ``domain`` portion of the address. - - .. attribute:: addr_spec - - The ``username@domain`` portion of the address, correctly quoted - for use as a bare address (the second form shown above). This - attribute is not mutable. - - .. method:: __str__() - - The ``str`` value of the object is the address quoted according to - :rfc:`5322` rules, but with no Content Transfer Encoding of any non-ASCII - characters. - - To support SMTP (:rfc:`5321`), ``Address`` handles one special case: if - ``username`` and ``domain`` are both the empty string (or ``None``), then - the string value of the ``Address`` is ``<>``. - - -.. class:: Group(display_name=None, addresses=None) - - The class used to represent an address group. The general form of an - address group is:: - - display_name: [address-list]; - - As a convenience for processing lists of addresses that consist of a mixture - of groups and single addresses, a ``Group`` may also be used to represent - single addresses that are not part of a group by setting *display_name* to - ``None`` and providing a list of the single address as *addresses*. - - .. attribute:: display_name - - The ``display_name`` of the group. If it is ``None`` and there is - exactly one ``Address`` in ``addresses``, then the ``Group`` represents a - single address that is not in a group. - - .. attribute:: addresses - - A possibly empty tuple of :class:`.Address` objects representing the - addresses in the group. - - .. method:: __str__() - - The ``str`` value of a ``Group`` is formatted according to :rfc:`5322`, - but with no Content Transfer Encoding of any non-ASCII characters. If - ``display_name`` is none and there is a single ``Address`` in the - ``addresses`` list, the ``str`` value will be the same as the ``str`` of - that single ``Address``. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.3 as a :term:`provisional module ` diff --git a/third_party/python/Doc/library/email.iterators.rst b/third_party/python/Doc/library/email.iterators.rst deleted file mode 100644 index d53ab33b8..000000000 --- a/third_party/python/Doc/library/email.iterators.rst +++ /dev/null @@ -1,83 +0,0 @@ -:mod:`email.iterators`: Iterators ---------------------------------- - -.. module:: email.iterators - :synopsis: Iterate over a message object tree. - -**Source code:** :source:`Lib/email/iterators.py` - --------------- - -Iterating over a message object tree is fairly easy with the -:meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over -message object trees. - - -.. function:: body_line_iterator(msg, decode=False) - - This iterates over all the payloads in all the subparts of *msg*, returning the - string payloads line-by-line. It skips over all the subpart headers, and it - skips over any subpart with a payload that isn't a Python string. This is - somewhat equivalent to reading the flat text representation of the message from - a file using :meth:`~io.TextIOBase.readline`, skipping over all the - intervening headers. - - Optional *decode* is passed through to :meth:`Message.get_payload - `. - - -.. function:: typed_subpart_iterator(msg, maintype='text', subtype=None) - - This iterates over all the subparts of *msg*, returning only those subparts that - match the MIME type specified by *maintype* and *subtype*. - - Note that *subtype* is optional; if omitted, then subpart MIME type matching is - done only with the main type. *maintype* is optional too; it defaults to - :mimetype:`text`. - - Thus, by default :func:`typed_subpart_iterator` returns each subpart that has a - MIME type of :mimetype:`text/\*`. - - -The following function has been added as a useful debugging tool. It should -*not* be considered part of the supported public interface for the package. - -.. function:: _structure(msg, fp=None, level=0, include_default=False) - - Prints an indented representation of the content types of the message object - structure. For example: - - .. testsetup:: - - import email - from email.iterators import _structure - somefile = open('../Lib/test/test_email/data/msg_02.txt') - - .. doctest:: - - >>> msg = email.message_from_file(somefile) - >>> _structure(msg) - multipart/mixed - text/plain - text/plain - multipart/digest - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - text/plain - - .. testcleanup:: - - somefile.close() - - Optional *fp* is a file-like object to print the output to. It must be - suitable for Python's :func:`print` function. *level* is used internally. - *include_default*, if true, prints the default type as well. diff --git a/third_party/python/Doc/library/email.message.rst b/third_party/python/Doc/library/email.message.rst deleted file mode 100644 index 77b8099a7..000000000 --- a/third_party/python/Doc/library/email.message.rst +++ /dev/null @@ -1,751 +0,0 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- - -.. module:: email.message - :synopsis: The base class representing email messages. -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray , - Barry A. Warsaw - -**Source code:** :source:`Lib/email/message.py` - --------------- - -.. versionadded:: 3.6 [1]_ - -The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for -the :mod:`email` object model. :class:`EmailMessage` provides the core -functionality for setting and querying header fields, for accessing message -bodies, and for creating or modifying structured messages. - -An email message consists of *headers* and a *payload* (which is also referred -to as the *content*). Headers are :rfc:`5322` or :rfc:`6532` style field names -and values, where the field name and value are separated by a colon. The colon -is not part of either the field name or the field value. The payload may be a -simple text message, or a binary object, or a structured sequence of -sub-messages each with their own set of headers and their own payload. The -latter type of payload is indicated by the message having a MIME type such as -:mimetype:`multipart/\*` or :mimetype:`message/rfc822`. - -The conceptual model provided by an :class:`EmailMessage` object is that of an -ordered dictionary of headers coupled with a *payload* that represents the -:rfc:`5322` body of the message, which might be a list of sub-``EmailMessage`` -objects. In addition to the normal dictionary methods for accessing the header -names and values, there are methods for accessing specialized information from -the headers (for example the MIME content type), for operating on the payload, -for generating a serialized version of the message, and for recursively walking -over the object tree. - -The :class:`EmailMessage` dictionary-like interface is indexed by the header -names, which must be ASCII values. The values of the dictionary are strings -with some extra methods. Headers are stored and returned in case-preserving -form, but field names are matched case-insensitively. Unlike a real dict, -there is an ordering to the keys, and there can be duplicate keys. Additional -methods are provided for working with headers that have duplicate keys. - -The *payload* is either a string or bytes object, in the case of simple message -objects, or a list of :class:`EmailMessage` objects, for MIME container -documents such as :mimetype:`multipart/\*` and :mimetype:`message/rfc822` -message objects. - - -.. class:: EmailMessage(policy=default) - - If *policy* is specified use the rules it specifies to update and serialize - the representation of the message. If *policy* is not set, use the - :class:`~email.policy.default` policy, which follows the rules of the email - RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses - the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. - - .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) - - Return the entire message flattened as a string. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compatibility - with the base :class:`~email.message.Message` class *maxheaderlen* is - accepted, but defaults to ``None``, which means that by default the line - length is controlled by the - :attr:`~email.policy.EmailPolicy.max_line_length` of the policy. The - *policy* argument may be used to override the default policy obtained - from the message instance. This can be used to control some of the - formatting produced by the method, since the specified *policy* will be - passed to the :class:`~email.generator.Generator`. - - Flattening the message may trigger changes to the :class:`EmailMessage` - if defaults need to be filled in to complete the transformation to a - string (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not be the - most useful way to serialize messages in your application, especially if - you are dealing with multiple messages. See - :class:`email.generator.Generator` for a more flexible API for - serializing messages. Note also that this method is restricted to - producing messages serialized as "7 bit clean" when - :attr:`~email.policy.EmailPolicy.utf8` is ``False``, which is the default. - - .. versionchanged:: 3.6 the default behavior when *maxheaderlen* - is not specified was changed from defaulting to 0 to defaulting - to the value of *max_line_length* from the policy. - - - .. method:: __str__() - - Equivalent to ``as_string(policy=self.policy.clone(utf8=True))``. Allows - ``str(msg)`` to produce a string containing the serialized message in a - readable format. - - .. versionchanged:: 3.4 the method was changed to use ``utf8=True``, - thus producing an :rfc:`6531`-like message representation, instead of - being a direct alias for :meth:`as_string`. - - - .. method:: as_bytes(unixfrom=False, policy=None) - - Return the entire message flattened as a bytes object. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the - :class:`~email.generator.BytesGenerator`. - - Flattening the message may trigger changes to the :class:`EmailMessage` - if defaults need to be filled in to complete the transformation to a - string (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not be the - most useful way to serialize messages in your application, especially if - you are dealing with multiple messages. See - :class:`email.generator.BytesGenerator` for a more flexible API for - serializing messages. - - - .. method:: __bytes__() - - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a - bytes object containing the serialized message. - - - .. method:: is_multipart() - - Return ``True`` if the message's payload is a list of - sub-\ :class:`EmailMessage` objects, otherwise return ``False``. When - :meth:`is_multipart` returns ``False``, the payload should be a string - object (which might be a CTE encoded binary payload). Note that - :meth:`is_multipart` returning ``True`` does not necessarily mean that - "msg.get_content_maintype() == 'multipart'" will return the ``True``. - For example, ``is_multipart`` will return ``True`` when the - :class:`EmailMessage` is of type ``message/rfc822``. - - - .. method:: set_unixfrom(unixfrom) - - Set the message's envelope header to *unixfrom*, which should be a - string. (See :class:`~mailbox.mboxMessage` for a brief description of - this header.) - - - .. method:: get_unixfrom() - - Return the message's envelope header. Defaults to ``None`` if the - envelope header was never set. - - - The following methods implement the mapping-like interface for accessing the - message's headers. Note that there are some semantic differences - between these methods and a normal mapping (i.e. dictionary) interface. For - example, in a dictionary there are no duplicate keys, but here there may be - duplicate message headers. Also, in dictionaries there is no guaranteed - order to the keys returned by :meth:`keys`, but in an :class:`EmailMessage` - object, headers are always returned in the order they appeared in the - original message, or in which they were added to the message later. Any - header deleted and then re-added is always appended to the end of the - header list. - - These semantic differences are intentional and are biased toward - convenience in the most common use cases. - - Note that in all cases, any envelope header present in the message is not - included in the mapping interface. - - - .. method:: __len__() - - Return the total number of headers, including duplicates. - - - .. method:: __contains__(name) - - Return true if the message object has a field named *name*. Matching is - done without regard to case and *name* does not include the trailing - colon. Used for the ``in`` operator. For example:: - - if 'message-id' in myMessage: - print('Message-ID:', myMessage['message-id']) - - - .. method:: __getitem__(name) - - Return the value of the named header field. *name* does not include the - colon field separator. If the header is missing, ``None`` is returned; a - :exc:`KeyError` is never raised. - - Note that if the named field appears more than once in the message's - headers, exactly which of those field values will be returned is - undefined. Use the :meth:`get_all` method to get the values of all the - extant headers named *name*. - - Using the standard (non-``compat32``) policies, the returned value is an - instance of a subclass of :class:`email.headerregistry.BaseHeader`. - - - .. method:: __setitem__(name, val) - - Add a header to the message with field name *name* and value *val*. The - field is appended to the end of the message's existing headers. - - Note that this does *not* overwrite or delete any existing header with the same - name. If you want to ensure that the new header is the only one present in the - message with field name *name*, delete the field first, e.g.:: - - del msg['subject'] - msg['subject'] = 'Python roolz!' - - If the :mod:`policy` defines certain headers to be unique (as the standard - policies do), this method may raise a :exc:`ValueError` when an attempt - is made to assign a value to such a header when one already exists. This - behavior is intentional for consistency's sake, but do not depend on it - as we may choose to make such assignments do an automatic deletion of the - existing header in the future. - - - .. method:: __delitem__(name) - - Delete all occurrences of the field with name *name* from the message's - headers. No exception is raised if the named field isn't present in the - headers. - - - .. method:: keys() - - Return a list of all the message's header field names. - - - .. method:: values() - - Return a list of all the message's field values. - - - .. method:: items() - - Return a list of 2-tuples containing all the message's field headers and - values. - - - .. method:: get(name, failobj=None) - - Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the - named header is missing (*failobj* defaults to ``None``). - - - Here are some additional useful header related methods: - - - .. method:: get_all(name, failobj=None) - - Return a list of all the values for the field named *name*. If there are - no such named headers in the message, *failobj* is returned (defaults to - ``None``). - - - .. method:: add_header(_name, _value, **_params) - - Extended header setting. This method is similar to :meth:`__setitem__` - except that additional header parameters can be provided as keyword - arguments. *_name* is the header field to add and *_value* is the - *primary* value for the header. - - For each item in the keyword argument dictionary *_params*, the key is - taken as the parameter name, with underscores converted to dashes (since - dashes are illegal in Python identifiers). Normally, the parameter will - be added as ``key="value"`` unless the value is ``None``, in which case - only the key will be added. - - If the value contains non-ASCII characters, the charset and language may - be explicitly controlled by specifying the value as a three tuple in the - format ``(CHARSET, LANGUAGE, VALUE)``, where ``CHARSET`` is a string - naming the charset to be used to encode the value, ``LANGUAGE`` can - usually be set to ``None`` or the empty string (see :rfc:`2231` for other - possibilities), and ``VALUE`` is the string value containing non-ASCII - code points. If a three tuple is not passed and the value contains - non-ASCII characters, it is automatically encoded in :rfc:`2231` format - using a ``CHARSET`` of ``utf-8`` and a ``LANGUAGE`` of ``None``. - - Here is an example:: - - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - - This will add a header that looks like :: - - Content-Disposition: attachment; filename="bud.gif" - - An example of the extended interface with non-ASCII characters:: - - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fußballer.ppt')) - - - .. method:: replace_header(_name, _value) - - Replace a header. Replace the first header found in the message that - matches *_name*, retaining header order and field name case of the - original header. If no matching header is found, raise a - :exc:`KeyError`. - - - .. method:: get_content_type() - - Return the message's content type, coerced to lower case of the form - :mimetype:`maintype/subtype`. If there is no :mailheader:`Content-Type` - header in the message return the value returned by - :meth:`get_default_type`. If the :mailheader:`Content-Type` header is - invalid, return ``text/plain``. - - (According to :rfc:`2045`, messages always have a default type, - :meth:`get_content_type` will always return a value. :rfc:`2045` defines - a message's default type to be :mimetype:`text/plain` unless it appears - inside a :mimetype:`multipart/digest` container, in which case it would - be :mimetype:`message/rfc822`. If the :mailheader:`Content-Type` header - has an invalid type specification, :rfc:`2045` mandates that the default - type be :mimetype:`text/plain`.) - - - .. method:: get_content_maintype() - - Return the message's main content type. This is the :mimetype:`maintype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_content_subtype() - - Return the message's sub-content type. This is the :mimetype:`subtype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_default_type() - - Return the default content type. Most messages have a default content - type of :mimetype:`text/plain`, except for messages that are subparts of - :mimetype:`multipart/digest` containers. Such subparts have a default - content type of :mimetype:`message/rfc822`. - - - .. method:: set_default_type(ctype) - - Set the default content type. *ctype* should either be - :mimetype:`text/plain` or :mimetype:`message/rfc822`, although this is - not enforced. The default content type is not stored in the - :mailheader:`Content-Type` header, so it only affects the return value of - the ``get_content_type`` methods when no :mailheader:`Content-Type` - header is present in the message. - - - .. method:: set_param(param, value, header='Content-Type', requote=True, \ - charset=None, language='', replace=False) - - Set a parameter in the :mailheader:`Content-Type` header. If the - parameter already exists in the header, replace its value with *value*. - When *header* is ``Content-Type`` (the default) and the header does not - yet exist in the message, add it, set its value to - :mimetype:`text/plain`, and append the new parameter value. Optional - *header* specifies an alternative header to :mailheader:`Content-Type`. - - If the value contains non-ASCII characters, the charset and language may - be explicitly specified using the optional *charset* and *language* - parameters. Optional *language* specifies the :rfc:`2231` language, - defaulting to the empty string. Both *charset* and *language* should be - strings. The default is to use the ``utf8`` *charset* and ``None`` for - the *language*. - - If *replace* is ``False`` (the default) the header is moved to the - end of the list of headers. If *replace* is ``True``, the header - will be updated in place. - - Use of the *requote* parameter with :class:`EmailMessage` objects is - deprecated. - - Note that existing parameter values of headers may be accessed through - the :attr:`~email.headerregistry.BaseHeader.params` attribute of the - header value (for example, ``msg['Content-Type'].params['charset']``). - - .. versionchanged:: 3.4 ``replace`` keyword was added. - - - .. method:: del_param(param, header='content-type', requote=True) - - Remove the given parameter completely from the :mailheader:`Content-Type` - header. The header will be re-written in place without the parameter or - its value. Optional *header* specifies an alternative to - :mailheader:`Content-Type`. - - Use of the *requote* parameter with :class:`EmailMessage` objects is - deprecated. - - - .. method:: get_filename(failobj=None) - - Return the value of the ``filename`` parameter of the - :mailheader:`Content-Disposition` header of the message. If the header - does not have a ``filename`` parameter, this method falls back to looking - for the ``name`` parameter on the :mailheader:`Content-Type` header. If - neither is found, or the header is missing, then *failobj* is returned. - The returned string will always be unquoted as per - :func:`email.utils.unquote`. - - - .. method:: get_boundary(failobj=None) - - Return the value of the ``boundary`` parameter of the - :mailheader:`Content-Type` header of the message, or *failobj* if either - the header is missing, or has no ``boundary`` parameter. The returned - string will always be unquoted as per :func:`email.utils.unquote`. - - - .. method:: set_boundary(boundary) - - Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to - *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`~email.errors.HeaderParseError` is raised if the - message object has no :mailheader:`Content-Type` header. - - Note that using this method is subtly different from deleting the old - :mailheader:`Content-Type` header and adding a new one with the new - boundary via :meth:`add_header`, because :meth:`set_boundary` preserves - the order of the :mailheader:`Content-Type` header in the list of - headers. - - - .. method:: get_content_charset(failobj=None) - - Return the ``charset`` parameter of the :mailheader:`Content-Type` header, - coerced to lower case. If there is no :mailheader:`Content-Type` header, or if - that header has no ``charset`` parameter, *failobj* is returned. - - - .. method:: get_charsets(failobj=None) - - Return a list containing the character set names in the message. If the - message is a :mimetype:`multipart`, then the list will contain one element - for each subpart in the payload, otherwise, it will be a list of length 1. - - Each item in the list will be a string which is the value of the - ``charset`` parameter in the :mailheader:`Content-Type` header for the - represented subpart. If the subpart has no :mailheader:`Content-Type` - header, no ``charset`` parameter, or is not of the :mimetype:`text` main - MIME type, then that item in the returned list will be *failobj*. - - - .. method:: is_attachment - - Return ``True`` if there is a :mailheader:`Content-Disposition` header - and its (case insensitive) value is ``attachment``, ``False`` otherwise. - - .. versionchanged:: 3.4.2 - is_attachment is now a method instead of a property, for consistency - with :meth:`~email.message.Message.is_multipart`. - - - .. method:: get_content_disposition() - - Return the lowercased value (without parameters) of the message's - :mailheader:`Content-Disposition` header if it has one, or ``None``. The - possible values for this method are *inline*, *attachment* or ``None`` - if the message follows :rfc:`2183`. - - .. versionadded:: 3.5 - - - The following methods relate to interrogating and manipulating the content - (payload) of the message. - - - .. method:: walk() - - The :meth:`walk` method is an all-purpose generator which can be used to - iterate over all the parts and subparts of a message object tree, in - depth-first traversal order. You will typically use :meth:`walk` as the - iterator in a ``for`` loop; each iteration returns the next subpart. - - Here's an example that prints the MIME type of every part of a multipart - message structure: - - .. testsetup:: - - from email import message_from_binary_file - with open('../Lib/test/test_email/data/msg_16.txt', 'rb') as f: - msg = message_from_binary_file(f) - from email.iterators import _structure - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_type()) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - ``walk`` iterates over the subparts of any part where - :meth:`is_multipart` returns ``True``, even though - ``msg.get_content_maintype() == 'multipart'`` may return ``False``. We - can see this in our example by making use of the ``_structure`` debug - helper function: - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_maintype() == 'multipart', - ... part.is_multipart()) - True True - False False - False True - False False - False False - False True - False False - >>> _structure(msg) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - Here the ``message`` parts are not ``multiparts``, but they do contain - subparts. ``is_multipart()`` returns ``True`` and ``walk`` descends - into the subparts. - - - .. method:: get_body(preferencelist=('related', 'html', 'plain')) - - Return the MIME part that is the best candidate to be the "body" of the - message. - - *preferencelist* must be a sequence of strings from the set ``related``, - ``html``, and ``plain``, and indicates the order of preference for the - content type of the part returned. - - Start looking for candidate matches with the object on which the - ``get_body`` method is called. - - If ``related`` is not included in *preferencelist*, consider the root - part (or subpart of the root part) of any related encountered as a - candidate if the (sub-)part matches a preference. - - When encountering a ``multipart/related``, check the ``start`` parameter - and if a part with a matching :mailheader:`Content-ID` is found, consider - only it when looking for candidate matches. Otherwise consider only the - first (default root) part of the ``multipart/related``. - - If a part has a :mailheader:`Content-Disposition` header, only consider - the part a candidate match if the value of the header is ``inline``. - - If none of the candidates matches any of the preferences in - *preferencelist*, return ``None``. - - Notes: (1) For most applications the only *preferencelist* combinations - that really make sense are ``('plain',)``, ``('html', 'plain')``, and the - default ``('related', 'html', 'plain')``. (2) Because matching starts - with the object on which ``get_body`` is called, calling ``get_body`` on - a ``multipart/related`` will return the object itself unless - *preferencelist* has a non-default value. (3) Messages (or message parts) - that do not specify a :mailheader:`Content-Type` or whose - :mailheader:`Content-Type` header is invalid will be treated as if they - are of type ``text/plain``, which may occasionally cause ``get_body`` to - return unexpected results. - - - .. method:: iter_attachments() - - Return an iterator over all of the immediate sub-parts of the message - that are not candidate "body" parts. That is, skip the first occurrence - of each of ``text/plain``, ``text/html``, ``multipart/related``, or - ``multipart/alternative`` (unless they are explicitly marked as - attachments via :mailheader:`Content-Disposition: attachment`), and - return all remaining parts. When applied directly to a - ``multipart/related``, return an iterator over the all the related parts - except the root part (ie: the part pointed to by the ``start`` parameter, - or the first part if there is no ``start`` parameter or the ``start`` - parameter doesn't match the :mailheader:`Content-ID` of any of the - parts). When applied directly to a ``multipart/alternative`` or a - non-``multipart``, return an empty iterator. - - - .. method:: iter_parts() - - Return an iterator over all of the immediate sub-parts of the message, - which will be empty for a non-``multipart``. (See also - :meth:`~email.message.EmailMessage.walk`.) - - - .. method:: get_content(*args, content_manager=None, **kw) - - Call the :meth:`~email.contentmanager.ContentManager.get_content` method - of the *content_manager*, passing self as the message object, and passing - along any other arguments or keywords as additional arguments. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: set_content(*args, content_manager=None, **kw) - - Call the :meth:`~email.contentmanager.ContentManager.set_content` method - of the *content_manager*, passing self as the message object, and passing - along any other arguments or keywords as additional arguments. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: make_related(boundary=None) - - Convert a non-``multipart`` message into a ``multipart/related`` message, - moving any existing :mailheader:`Content-` headers and payload into a - (new) first part of the ``multipart``. If *boundary* is specified, use - it as the boundary string in the multipart, otherwise leave the boundary - to be automatically created when it is needed (for example, when the - message is serialized). - - - .. method:: make_alternative(boundary=None) - - Convert a non-``multipart`` or a ``multipart/related`` into a - ``multipart/alternative``, moving any existing :mailheader:`Content-` - headers and payload into a (new) first part of the ``multipart``. If - *boundary* is specified, use it as the boundary string in the multipart, - otherwise leave the boundary to be automatically created when it is - needed (for example, when the message is serialized). - - - .. method:: make_mixed(boundary=None) - - Convert a non-``multipart``, a ``multipart/related``, or a - ``multipart-alternative`` into a ``multipart/mixed``, moving any existing - :mailheader:`Content-` headers and payload into a (new) first part of the - ``multipart``. If *boundary* is specified, use it as the boundary string - in the multipart, otherwise leave the boundary to be automatically - created when it is needed (for example, when the message is serialized). - - - .. method:: add_related(*args, content_manager=None, **kw) - - If the message is a ``multipart/related``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, - and :meth:`~email.message.Message.attach` it to the ``multipart``. If - the message is a non-``multipart``, call :meth:`make_related` and then - proceed as above. If the message is any other type of ``multipart``, - raise a :exc:`TypeError`. If *content_manager* is not specified, use - the ``content_manager`` specified by the current :mod:`~email.policy`. - If the added part has no :mailheader:`Content-Disposition` header, - add one with the value ``inline``. - - - .. method:: add_alternative(*args, content_manager=None, **kw) - - If the message is a ``multipart/alternative``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart`` or ``multipart/related``, call - :meth:`make_alternative` and then proceed as above. If the message is - any other type of ``multipart``, raise a :exc:`TypeError`. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: add_attachment(*args, content_manager=None, **kw) - - If the message is a ``multipart/mixed``, create a new message object, - pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart``, ``multipart/related``, or - ``multipart/alternative``, call :meth:`make_mixed` and then proceed as - above. If *content_manager* is not specified, use the ``content_manager`` - specified by the current :mod:`~email.policy`. If the added part - has no :mailheader:`Content-Disposition` header, add one with the value - ``attachment``. This method can be used both for explicit attachments - (:mailheader:`Content-Disposition: attachment`) and ``inline`` attachments - (:mailheader:`Content-Disposition: inline`), by passing appropriate - options to the ``content_manager``. - - - .. method:: clear() - - Remove the payload and all of the headers. - - - .. method:: clear_content() - - Remove the payload and all of the :exc:`Content-` headers, leaving - all other headers intact and in their original order. - - - :class:`EmailMessage` objects have the following instance attributes: - - - .. attribute:: preamble - - The format of a MIME document allows for some text between the blank line - following the headers, and the first multipart boundary string. Normally, - this text is never visible in a MIME-aware mail reader because it falls - outside the standard MIME armor. However, when viewing the raw text of - the message, or when viewing the message in a non-MIME aware reader, this - text can become visible. - - The *preamble* attribute contains this leading extra-armor text for MIME - documents. When the :class:`~email.parser.Parser` discovers some text - after the headers but before the first boundary string, it assigns this - text to the message's *preamble* attribute. When the - :class:`~email.generator.Generator` is writing out the plain text - representation of a MIME message, and it finds the - message has a *preamble* attribute, it will write this text in the area - between the headers and the first boundary. See :mod:`email.parser` and - :mod:`email.generator` for details. - - Note that if the message object has no preamble, the *preamble* attribute - will be ``None``. - - - .. attribute:: epilogue - - The *epilogue* attribute acts the same way as the *preamble* attribute, - except that it contains text that appears between the last boundary and - the end of the message. As with the :attr:`~EmailMessage.preamble`, - if there is no epilog text this attribute will be ``None``. - - - .. attribute:: defects - - The *defects* attribute contains a list of all the problems found when - parsing this message. See :mod:`email.errors` for a detailed description - of the possible parsing defects. - - -.. class:: MIMEPart(policy=default) - - This class represents a subpart of a MIME message. It is identical to - :class:`EmailMessage`, except that no :mailheader:`MIME-Version` headers are - added when :meth:`~EmailMessage.set_content` is called, since sub-parts do - not need their own :mailheader:`MIME-Version` headers. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to - :ref:`compat32_message`. diff --git a/third_party/python/Doc/library/email.mime.rst b/third_party/python/Doc/library/email.mime.rst deleted file mode 100644 index f37f6aa28..000000000 --- a/third_party/python/Doc/library/email.mime.rst +++ /dev/null @@ -1,259 +0,0 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- - -.. module:: email.mime - :synopsis: Build MIME messages. - -**Source code:** :source:`Lib/email/mime/` - --------------- - -This module is part of the legacy (``Compat32``) email API. Its functionality -is partially replaced by the :mod:`~email.contentmanager` in the new API, but -in certain applications these classes may still be useful, even in non-legacy -code. - -Ordinarily, you get a message object structure by passing a file or some text to -a parser, which parses the text and returns the root message object. However -you can also build a complete message structure from scratch, or even individual -:class:`~email.message.Message` objects by hand. In fact, you can also take an -existing structure and add new :class:`~email.message.Message` objects, move them -around, etc. This makes a very convenient interface for slicing-and-dicing MIME -messages. - -You can create a new object structure by creating :class:`~email.message.Message` -instances, adding attachments and all the appropriate headers manually. For MIME -messages though, the :mod:`email` package provides some convenient subclasses to -make things easier. - -Here are the classes: - -.. currentmodule:: email.mime.base - -.. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params) - - Module: :mod:`email.mime.base` - - This is the base class for all the MIME-specific subclasses of - :class:`~email.message.Message`. Ordinarily you won't create instances - specifically of :class:`MIMEBase`, although you could. :class:`MIMEBase` - is provided primarily as a convenient base class for more specific - MIME-aware subclasses. - - *_maintype* is the :mailheader:`Content-Type` major type (e.g. :mimetype:`text` - or :mimetype:`image`), and *_subtype* is the :mailheader:`Content-Type` minor - type (e.g. :mimetype:`plain` or :mimetype:`gif`). *_params* is a parameter - key/value dictionary and is passed directly to :meth:`Message.add_header - `. - - If *policy* is specified, (defaults to the - :class:`compat32 ` policy) it will be passed to - :class:`~email.message.Message`. - - The :class:`MIMEBase` class always adds a :mailheader:`Content-Type` header - (based on *_maintype*, *_subtype*, and *_params*), and a - :mailheader:`MIME-Version` header (always set to ``1.0``). - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - - -.. currentmodule:: email.mime.nonmultipart - -.. class:: MIMENonMultipart() - - Module: :mod:`email.mime.nonmultipart` - - A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base - class for MIME messages that are not :mimetype:`multipart`. The primary - purpose of this class is to prevent the use of the - :meth:`~email.message.Message.attach` method, which only makes sense for - :mimetype:`multipart` messages. If :meth:`~email.message.Message.attach` - is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. - - -.. currentmodule:: email.mime.multipart - -.. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.multipart` - - A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base - class for MIME messages that are :mimetype:`multipart`. Optional *_subtype* - defaults to :mimetype:`mixed`, but can be used to specify the subtype of the - message. A :mailheader:`Content-Type` header of :mimetype:`multipart/_subtype` - will be added to the message object. A :mailheader:`MIME-Version` header will - also be added. - - Optional *boundary* is the multipart boundary string. When ``None`` (the - default), the boundary is calculated when needed (for example, when the - message is serialized). - - *_subparts* is a sequence of initial subparts for the payload. It must be - possible to convert this sequence to a list. You can always attach new subparts - to the message by using the :meth:`Message.attach - ` method. - - Optional *policy* argument defaults to :class:`compat32 `. - - Additional parameters for the :mailheader:`Content-Type` header are taken from - the keyword arguments, or passed into the *_params* argument, which is a keyword - dictionary. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.application - -.. class:: MIMEApplication(_data, _subtype='octet-stream', \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.application` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEApplication` class is used to represent MIME message objects of - major type :mimetype:`application`. *_data* is a string containing the raw - byte data. Optional *_subtype* specifies the MIME subtype and defaults to - :mimetype:`octet-stream`. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the data for transport. This callable takes one argument, which is - the :class:`MIMEApplication` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the base class constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.audio - -.. class:: MIMEAudio(_audiodata, _subtype=None, \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.audio` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEAudio` class is used to create MIME message objects of major type - :mimetype:`audio`. *_audiodata* is a string containing the raw audio data. If - this data can be decoded by the standard Python module :mod:`sndhdr`, then the - subtype will be automatically included in the :mailheader:`Content-Type` header. - Otherwise you can explicitly specify the audio subtype via the *_subtype* - argument. If the minor type could not be guessed and *_subtype* was not given, - then :exc:`TypeError` is raised. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the audio data for transport. This callable takes one argument, - which is the :class:`MIMEAudio` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the base class constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.image - -.. class:: MIMEImage(_imagedata, _subtype=None, \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.image` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEImage` class is used to create MIME message objects of major type - :mimetype:`image`. *_imagedata* is a string containing the raw image data. If - this data can be decoded by the standard Python module :mod:`imghdr`, then the - subtype will be automatically included in the :mailheader:`Content-Type` header. - Otherwise you can explicitly specify the image subtype via the *_subtype* - argument. If the minor type could not be guessed and *_subtype* was not given, - then :exc:`TypeError` is raised. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the image data for transport. This callable takes one argument, - which is the :class:`MIMEImage` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the :class:`~email.mime.base.MIMEBase` - constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.message - -.. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32) - - Module: :mod:`email.mime.message` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEMessage` class is used to create MIME objects of main type - :mimetype:`message`. *_msg* is used as the payload, and must be an instance - of class :class:`~email.message.Message` (or a subclass thereof), otherwise - a :exc:`TypeError` is raised. - - Optional *_subtype* sets the subtype of the message; it defaults to - :mimetype:`rfc822`. - - Optional *policy* argument defaults to :class:`compat32 `. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.text - -.. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32) - - Module: :mod:`email.mime.text` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEText` class is used to create MIME objects of major type - :mimetype:`text`. *_text* is the string for the payload. *_subtype* is the - minor type and defaults to :mimetype:`plain`. *_charset* is the character - set of the text and is passed as an argument to the - :class:`~email.mime.nonmultipart.MIMENonMultipart` constructor; it defaults - to ``us-ascii`` if the string contains only ``ascii`` code points, and - ``utf-8`` otherwise. The *_charset* parameter accepts either a string or a - :class:`~email.charset.Charset` instance. - - Unless the *_charset* argument is explicitly set to ``None``, the - MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` - header. This means that a subsequent ``set_payload`` call will not result - in an encoded payload, even if a charset is passed in the ``set_payload`` - command. You can "reset" this behavior by deleting the - ``Content-Transfer-Encoding`` header, after which a ``set_payload`` call - will automatically encode the new payload (and add a new - :mailheader:`Content-Transfer-Encoding` header). - - Optional *policy* argument defaults to :class:`compat32 `. - - .. versionchanged:: 3.5 - *_charset* also accepts :class:`~email.charset.Charset` instances. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. diff --git a/third_party/python/Doc/library/email.parser.rst b/third_party/python/Doc/library/email.parser.rst deleted file mode 100644 index d9a61616b..000000000 --- a/third_party/python/Doc/library/email.parser.rst +++ /dev/null @@ -1,320 +0,0 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- - -.. module:: email.parser - :synopsis: Parse flat text email messages to produce a message object structure. - -**Source code:** :source:`Lib/email/parser.py` - --------------- - -Message object structures can be created in one of two ways: they can be -created from whole cloth by creating an :class:`~email.message.EmailMessage` -object, adding headers using the dictionary interface, and adding payload(s) -using :meth:`~email.message.EmailMessage.set_content` and related methods, or -they can be created by parsing a serialized representation of the email -message. - -The :mod:`email` package provides a standard parser that understands most email -document structures, including MIME documents. You can pass the parser a -bytes, string or file object, and the parser will return to you the root -:class:`~email.message.EmailMessage` instance of the object structure. For -simple, non-MIME messages the payload of this root object will likely be a -string containing the text of the message. For MIME messages, the root object -will return ``True`` from its :meth:`~email.message.EmailMessage.is_multipart` -method, and the subparts can be accessed via the payload manipulation methods, -such as :meth:`~email.message.EmailMessage.get_body`, -:meth:`~email.message.EmailMessage.iter_parts`, and -:meth:`~email.message.EmailMessage.walk`. - -There are actually two parser interfaces available for use, the :class:`Parser` -API and the incremental :class:`FeedParser` API. The :class:`Parser` API is -most useful if you have the entire text of the message in memory, or if the -entire message lives in a file on the file system. :class:`FeedParser` is more -appropriate when you are reading the message from a stream which might block -waiting for more input (such as reading an email message from a socket). The -:class:`FeedParser` can consume and parse the message incrementally, and only -returns the root object when you close the parser. - -Note that the parser can be extended in limited ways, and of course you can -implement your own parser completely from scratch. All of the logic that -connects the :mod:`email` package's bundled parser and the -:class:`~email.message.EmailMessage` class is embodied in the :mod:`policy` -class, so a custom parser can create message object trees any way it finds -necessary by implementing custom versions of the appropriate :mod:`policy` -methods. - - -FeedParser API -^^^^^^^^^^^^^^ - -The :class:`BytesFeedParser`, imported from the :mod:`email.feedparser` module, -provides an API that is conducive to incremental parsing of email messages, -such as would be necessary when reading the text of an email message from a -source that can block (such as a socket). The :class:`BytesFeedParser` can of -course be used to parse an email message fully contained in a :term:`bytes-like -object`, string, or file, but the :class:`BytesParser` API may be more -convenient for such use cases. The semantics and results of the two parser -APIs are identical. - -The :class:`BytesFeedParser`'s API is simple; you create an instance, feed it a -bunch of bytes until there's no more to feed it, then close the parser to -retrieve the root message object. The :class:`BytesFeedParser` is extremely -accurate when parsing standards-compliant messages, and it does a very good job -of parsing non-compliant messages, providing information about how a message -was deemed broken. It will populate a message object's -:attr:`~email.message.EmailMessage.defects` attribute with a list of any -problems it found in a message. See the :mod:`email.errors` module for the -list of defects that it can find. - -Here is the API for the :class:`BytesFeedParser`: - - -.. class:: BytesFeedParser(_factory=None, *, policy=policy.compat32) - - Create a :class:`BytesFeedParser` instance. Optional *_factory* is a - no-argument callable; if not specified use the - :attr:`~email.policy.Policy.message_factory` from the *policy*. Call - *_factory* whenever a new message object is needed. - - If *policy* is specified use the rules it specifies to update the - representation of the message. If *policy* is not set, use the - :class:`compat32 ` policy, which maintains backward - compatibility with the Python 3.2 version of the email package and provides - :class:`~email.message.Message` as the default factory. All other policies - provide :class:`~email.message.EmailMessage` as the default *_factory*. For - more information on what else *policy* controls, see the - :mod:`~email.policy` documentation. - - Note: **The policy keyword should always be specified**; The default will - change to :data:`email.policy.default` in a future version of Python. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 Added the *policy* keyword. - .. versionchanged:: 3.6 *_factory* defaults to the policy ``message_factory``. - - - .. method:: feed(data) - - Feed the parser some more data. *data* should be a :term:`bytes-like - object` containing one or more lines. The lines can be partial and the - parser will stitch such partial lines together properly. The lines can - have any of the three common line endings: carriage return, newline, or - carriage return and newline (they can even be mixed). - - - .. method:: close() - - Complete the parsing of all previously fed data and return the root - message object. It is undefined what happens if :meth:`~feed` is called - after this method has been called. - - -.. class:: FeedParser(_factory=None, *, policy=policy.compat32) - - Works like :class:`BytesFeedParser` except that the input to the - :meth:`~BytesFeedParser.feed` method must be a string. This is of limited - utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is - ``True``, no binary attachments. - - .. versionchanged:: 3.3 Added the *policy* keyword. - - -Parser API -^^^^^^^^^^ - -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, -provides an API that can be used to parse a message when the complete contents -of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, -and header-only parsers, :class:`BytesHeaderParser` and -:class:`HeaderParser`, which can be used if you're only interested in the -headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` -can be much faster in these situations, since they do not attempt to parse the -message body, instead setting the payload to the raw body. - - -.. class:: BytesParser(_class=None, *, policy=policy.compat32) - - Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and semantics as the *_factory* - and *policy* arguments of :class:`BytesFeedParser`. - - Note: **The policy keyword should always be specified**; The default will - change to :data:`email.policy.default` in a future version of Python. - - .. versionchanged:: 3.3 - Removed the *strict* argument that was deprecated in 2.4. Added the - *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - - .. method:: parse(fp, headersonly=False) - - Read all the data from the binary file-like object *fp*, parse the - resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` - methods. - - The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) - style headers and header continuation lines, optionally preceded by an - envelope header. The header block is terminated either by the end of the - data or by a blank line. Following the header block is the body of the - message (which may contain MIME-encoded subparts, including subparts - with a :mailheader:`Content-Transfer-Encoding` of ``8bit``). - - Optional *headersonly* is a flag specifying whether to stop parsing after - reading the headers or not. The default is ``False``, meaning it parses - the entire contents of the file. - - - .. method:: parsebytes(bytes, headersonly=False) - - Similar to the :meth:`parse` method, except it takes a :term:`bytes-like - object` instead of a file-like object. Calling this method on a - :term:`bytes-like object` is equivalent to wrapping *bytes* in a - :class:`~io.BytesIO` instance first and calling :meth:`parse`. - - Optional *headersonly* is as with the :meth:`parse` method. - - .. versionadded:: 3.2 - - -.. class:: BytesHeaderParser(_class=None, *, policy=policy.compat32) - - Exactly like :class:`BytesParser`, except that *headersonly* - defaults to ``True``. - - .. versionadded:: 3.3 - - -.. class:: Parser(_class=None, *, policy=policy.compat32) - - This class is parallel to :class:`BytesParser`, but handles string input. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - - .. method:: parse(fp, headersonly=False) - - Read all the data from the text-mode file-like object *fp*, parse the - resulting text, and return the root message object. *fp* must support - both the :meth:`~io.TextIOBase.readline` and the - :meth:`~io.TextIOBase.read` methods on file-like objects. - - Other than the text mode requirement, this method operates like - :meth:`BytesParser.parse`. - - - .. method:: parsestr(text, headersonly=False) - - Similar to the :meth:`parse` method, except it takes a string object - instead of a file-like object. Calling this method on a string is - equivalent to wrapping *text* in a :class:`~io.StringIO` instance first - and calling :meth:`parse`. - - Optional *headersonly* is as with the :meth:`parse` method. - - -.. class:: HeaderParser(_class=None, *, policy=policy.compat32) - - Exactly like :class:`Parser`, except that *headersonly* - defaults to ``True``. - - -Since creating a message object structure from a string or a file object is such -a common task, four functions are provided as a convenience. They are available -in the top-level :mod:`email` package namespace. - -.. currentmodule:: email - - -.. function:: message_from_bytes(s, _class=None, *, policy=policy.compat32) - - Return a message object structure from a :term:`bytes-like object`. This is - equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and - *policy* are interpreted as with the :class:`~email.parser.BytesParser` class - constructor. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_binary_file(fp, _class=None, *, \ - policy=policy.compat32) - - Return a message object structure tree from an open binary :term:`file - object`. This is equivalent to ``BytesParser().parse(fp)``. *_class* and - *policy* are interpreted as with the :class:`~email.parser.BytesParser` class - constructor. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_string(s, _class=None, *, policy=policy.compat32) - - Return a message object structure from a string. This is equivalent to - ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as - with the :class:`~email.parser.Parser` class constructor. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_file(fp, _class=None, *, policy=policy.compat32) - - Return a message object structure tree from an open :term:`file object`. - This is equivalent to ``Parser().parse(fp)``. *_class* and *policy* are - interpreted as with the :class:`~email.parser.Parser` class constructor. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - -Here's an example of how you might use :func:`message_from_bytes` at an -interactive Python prompt:: - - >>> import email - >>> msg = email.message_from_bytes(myBytes) # doctest: +SKIP - - -Additional notes -^^^^^^^^^^^^^^^^ - -Here are some notes on the parsing semantics: - -* Most non-\ :mimetype:`multipart` type messages are parsed as a single message - object with a string payload. These objects will return ``False`` for - :meth:`~email.message.EmailMessage.is_multipart`, and - :meth:`~email.message.EmailMessage.iter_parts` will yield an empty list. - -* All :mimetype:`multipart` type messages will be parsed as a container message - object with a list of sub-message objects for their payload. The outer - container message will return ``True`` for - :meth:`~email.message.EmailMessage.is_multipart`, and - :meth:`~email.message.EmailMessage.iter_parts` will yield a list of subparts. - -* Most messages with a content type of :mimetype:`message/\*` (such as - :mimetype:`message/delivery-status` and :mimetype:`message/rfc822`) will also - be parsed as container object containing a list payload of length 1. Their - :meth:`~email.message.EmailMessage.is_multipart` method will return ``True``. - The single element yielded by :meth:`~email.message.EmailMessage.iter_parts` - will be a sub-message object. - -* Some non-standards-compliant messages may not be internally consistent about - their :mimetype:`multipart`\ -edness. Such messages may have a - :mailheader:`Content-Type` header of type :mimetype:`multipart`, but their - :meth:`~email.message.EmailMessage.is_multipart` method may return ``False``. - If such messages were parsed with the :class:`~email.parser.FeedParser`, - they will have an instance of the - :class:`~email.errors.MultipartInvariantViolationDefect` class in their - *defects* attribute list. See :mod:`email.errors` for details. diff --git a/third_party/python/Doc/library/email.policy.rst b/third_party/python/Doc/library/email.policy.rst deleted file mode 100644 index 8e7076259..000000000 --- a/third_party/python/Doc/library/email.policy.rst +++ /dev/null @@ -1,651 +0,0 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ - -.. module:: email.policy - :synopsis: Controlling the parsing and generating of messages - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/email/policy.py` - --------------- - -The :mod:`email` package's prime focus is the handling of email messages as -described by the various email and MIME RFCs. However, the general format of -email messages (a block of header fields each consisting of a name followed by -a colon followed by a value, the whole block followed by a blank line and an -arbitrary 'body'), is a format that has found utility outside of the realm of -email. Some of these uses conform fairly closely to the main email RFCs, some -do not. Even when working with email, there are times when it is desirable to -break strict compliance with the RFCs, such as generating emails that -interoperate with email servers that do not themselves follow the standards, or -that implement extensions you want to use in ways that violate the -standards. - -Policy objects give the email package the flexibility to handle all these -disparate use cases. - -A :class:`Policy` object encapsulates a set of attributes and methods that -control the behavior of various components of the email package during use. -:class:`Policy` instances can be passed to various classes and methods in the -email package to alter the default behavior. The settable values and their -defaults are described below. - -There is a default policy used by all classes in the email package. For all of -the :mod:`~email.parser` classes and the related convenience functions, and for -the :class:`~email.message.Message` class, this is the :class:`Compat32` -policy, via its corresponding pre-defined instance :const:`compat32`. This -policy provides for complete backward compatibility (in some cases, including -bug compatibility) with the pre-Python3.3 version of the email package. - -This default value for the *policy* keyword to -:class:`~email.message.EmailMessage` is the :class:`EmailPolicy` policy, via -its pre-defined instance :data:`~default`. - -When a :class:`~email.message.Message` or :class:`~email.message.EmailMessage` -object is created, it acquires a policy. If the message is created by a -:mod:`~email.parser`, a policy passed to the parser will be the policy used by -the message it creates. If the message is created by the program, then the -policy can be specified when it is created. When a message is passed to a -:mod:`~email.generator`, the generator uses the policy from the message by -default, but you can also pass a specific policy to the generator that will -override the one stored on the message object. - -The default value for the *policy* keyword for the :mod:`email.parser` classes -and the parser convenience functions **will be changing** in a future version of -Python. Therefore you should **always specify explicitly which policy you want -to use** when calling any of the classes and functions described in the -:mod:`~email.parser` module. - -The first part of this documentation covers the features of :class:`Policy`, an -:term:`abstract base class` that defines the features that are common to all -policy objects, including :const:`compat32`. This includes certain hook -methods that are called internally by the email package, which a custom policy -could override to obtain different behavior. The second part describes the -concrete classes :class:`EmailPolicy` and :class:`Compat32`, which implement -the hooks that provide the standard behavior and the backward compatible -behavior and features, respectively. - -:class:`Policy` instances are immutable, but they can be cloned, accepting the -same keyword arguments as the class constructor and returning a new -:class:`Policy` instance that is a copy of the original but with the specified -attributes values changed. - -As an example, the following code could be used to read an email message from a -file on disk and pass it to the system ``sendmail`` program on a Unix system: - -.. testsetup:: - - from unittest import mock - mocker = mock.patch('subprocess.Popen') - m = mocker.start() - proc = mock.MagicMock() - m.return_value = proc - proc.stdin.close.return_value = None - mymsg = open('mymsg.txt', 'w') - mymsg.write('To: abc@xyz.com\n\n') - mymsg.flush() - -.. doctest:: - - >>> from email import message_from_binary_file - >>> from email.generator import BytesGenerator - >>> from email import policy - >>> from subprocess import Popen, PIPE - >>> with open('mymsg.txt', 'rb') as f: - ... msg = message_from_binary_file(f, policy=policy.default) - >>> p = Popen(['sendmail', msg['To'].addresses[0]], stdin=PIPE) - >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n')) - >>> g.flatten(msg) - >>> p.stdin.close() - >>> rc = p.wait() - -.. testcleanup:: - - mymsg.close() - mocker.stop() - import os - os.remove('mymsg.txt') - -Here we are telling :class:`~email.generator.BytesGenerator` to use the RFC -correct line separator characters when creating the binary string to feed into -``sendmail's`` ``stdin``, where the default policy would use ``\n`` line -separators. - -Some email package methods accept a *policy* keyword argument, allowing the -policy to be overridden for that method. For example, the following code uses -the :meth:`~email.message.Message.as_bytes` method of the *msg* object from -the previous example and writes the message to a file using the native line -separators for the platform on which it is running:: - - >>> import os - >>> with open('converted.txt', 'wb') as f: - ... f.write(msg.as_bytes(policy=msg.policy.clone(linesep=os.linesep))) - 17 - -Policy objects can also be combined using the addition operator, producing a -policy object whose settings are a combination of the non-default values of the -summed objects:: - - >>> compat_SMTP = policy.compat32.clone(linesep='\r\n') - >>> compat_strict = policy.compat32.clone(raise_on_defect=True) - >>> compat_strict_SMTP = compat_SMTP + compat_strict - -This operation is not commutative; that is, the order in which the objects are -added matters. To illustrate:: - - >>> policy100 = policy.compat32.clone(max_line_length=100) - >>> policy80 = policy.compat32.clone(max_line_length=80) - >>> apolicy = policy100 + policy80 - >>> apolicy.max_line_length - 80 - >>> apolicy = policy80 + policy100 - >>> apolicy.max_line_length - 100 - - -.. class:: Policy(**kw) - - This is the :term:`abstract base class` for all policy classes. It provides - default implementations for a couple of trivial methods, as well as the - implementation of the immutability property, the :meth:`clone` method, and - the constructor semantics. - - The constructor of a policy class can be passed various keyword arguments. - The arguments that may be specified are any non-method properties on this - class, plus any additional non-method properties on the concrete class. A - value specified in the constructor will override the default value for the - corresponding attribute. - - This class defines the following properties, and thus values for the - following may be passed in the constructor of any policy class: - - - .. attribute:: max_line_length - - The maximum length of any line in the serialized output, not counting the - end of line character(s). Default is 78, per :rfc:`5322`. A value of - ``0`` or :const:`None` indicates that no line wrapping should be - done at all. - - - .. attribute:: linesep - - The string to be used to terminate lines in serialized output. The - default is ``\n`` because that's the internal end-of-line discipline used - by Python, though ``\r\n`` is required by the RFCs. - - - .. attribute:: cte_type - - Controls the type of Content Transfer Encodings that may be or are - required to be used. The possible values are: - - .. tabularcolumns:: |l|L| - - ======== =============================================================== - ``7bit`` all data must be "7 bit clean" (ASCII-only). This means that - where necessary data will be encoded using either - quoted-printable or base64 encoding. - - ``8bit`` data is not constrained to be 7 bit clean. Data in headers is - still required to be ASCII-only and so will be encoded (see - :meth:`fold_binary` and :attr:`~EmailPolicy.utf8` below for - exceptions), but body parts may use the ``8bit`` CTE. - ======== =============================================================== - - A ``cte_type`` value of ``8bit`` only works with ``BytesGenerator``, not - ``Generator``, because strings cannot contain binary data. If a - ``Generator`` is operating under a policy that specifies - ``cte_type=8bit``, it will act as if ``cte_type`` is ``7bit``. - - - .. attribute:: raise_on_defect - - If :const:`True`, any defects encountered will be raised as errors. If - :const:`False` (the default), defects will be passed to the - :meth:`register_defect` method. - - - .. attribute:: mangle_from\_ - - If :const:`True`, lines starting with *"From "* in the body are - escaped by putting a ``>`` in front of them. This parameter is used when - the message is being serialized by a generator. - Default: :const:`False`. - - .. versionadded:: 3.5 - The *mangle_from_* parameter. - - - .. attribute:: message_factory - - A factory function for constructing a new empty message object. Used - by the parser when building messages. Defaults to ``None``, in - which case :class:`~email.message.Message` is used. - - .. versionadded:: 3.6 - - The following :class:`Policy` method is intended to be called by code using - the email library to create policy instances with custom settings: - - - .. method:: clone(**kw) - - Return a new :class:`Policy` instance whose attributes have the same - values as the current instance, except where those attributes are - given new values by the keyword arguments. - - - The remaining :class:`Policy` methods are called by the email package code, - and are not intended to be called by an application using the email package. - A custom policy must implement all of these methods. - - - .. method:: handle_defect(obj, defect) - - Handle a *defect* found on *obj*. When the email package calls this - method, *defect* will always be a subclass of - :class:`~email.errors.Defect`. - - The default implementation checks the :attr:`raise_on_defect` flag. If - it is ``True``, *defect* is raised as an exception. If it is ``False`` - (the default), *obj* and *defect* are passed to :meth:`register_defect`. - - - .. method:: register_defect(obj, defect) - - Register a *defect* on *obj*. In the email package, *defect* will always - be a subclass of :class:`~email.errors.Defect`. - - The default implementation calls the ``append`` method of the ``defects`` - attribute of *obj*. When the email package calls :attr:`handle_defect`, - *obj* will normally have a ``defects`` attribute that has an ``append`` - method. Custom object types used with the email package (for example, - custom ``Message`` objects) should also provide such an attribute, - otherwise defects in parsed messages will raise unexpected errors. - - - .. method:: header_max_count(name) - - Return the maximum allowed number of headers named *name*. - - Called when a header is added to an :class:`~email.message.EmailMessage` - or :class:`~email.message.Message` object. If the returned value is not - ``0`` or ``None``, and there are already a number of headers with the - name *name* greater than or equal to the value returned, a - :exc:`ValueError` is raised. - - Because the default behavior of ``Message.__setitem__`` is to append the - value to the list of headers, it is easy to create duplicate headers - without realizing it. This method allows certain headers to be limited - in the number of instances of that header that may be added to a - ``Message`` programmatically. (The limit is not observed by the parser, - which will faithfully produce as many headers as exist in the message - being parsed.) - - The default implementation returns ``None`` for all header names. - - - .. method:: header_source_parse(sourcelines) - - The email package calls this method with a list of strings, each string - ending with the line separation characters found in the source being - parsed. The first line includes the field header name and separator. - All whitespace in the source is preserved. The method should return the - ``(name, value)`` tuple that is to be stored in the ``Message`` to - represent the parsed header. - - If an implementation wishes to retain compatibility with the existing - email package policies, *name* should be the case preserved name (all - characters up to the '``:``' separator), while *value* should be the - unfolded value (all line separator characters removed, but whitespace - kept intact), stripped of leading whitespace. - - *sourcelines* may contain surrogateescaped binary data. - - There is no default implementation - - - .. method:: header_store_parse(name, value) - - The email package calls this method with the name and value provided by - the application program when the application program is modifying a - ``Message`` programmatically (as opposed to a ``Message`` created by a - parser). The method should return the ``(name, value)`` tuple that is to - be stored in the ``Message`` to represent the header. - - If an implementation wishes to retain compatibility with the existing - email package policies, the *name* and *value* should be strings or - string subclasses that do not change the content of the passed in - arguments. - - There is no default implementation - - - .. method:: header_fetch_parse(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` when that header is requested by the - application program, and whatever the method returns is what is passed - back to the application as the value of the header being retrieved. - Note that there may be more than one header with the same name stored in - the ``Message``; the method is passed the specific name and value of the - header destined to be returned to the application. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the value returned by the method. - - There is no default implementation - - - .. method:: fold(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` for a given header. The method should return a - string that represents that header "folded" correctly (according to the - policy settings) by composing the *name* with the *value* and inserting - :attr:`linesep` characters at the appropriate places. See :rfc:`5322` - for a discussion of the rules for folding email headers. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the string returned by the method. - - - .. method:: fold_binary(name, value) - - The same as :meth:`fold`, except that the returned value should be a - bytes object rather than a string. - - *value* may contain surrogateescaped binary data. These could be - converted back into binary data in the returned bytes object. - - - -.. class:: EmailPolicy(**kw) - - This concrete :class:`Policy` provides behavior that is intended to be fully - compliant with the current email RFCs. These include (but are not limited - to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs. - - This policy adds new header parsing and folding algorithms. Instead of - simple strings, headers are ``str`` subclasses with attributes that depend - on the type of the field. The parsing and folding algorithm fully implement - :rfc:`2047` and :rfc:`5322`. - - The default value for the :attr:`~email.policy.Policy.message_factory` - attribute is :class:`~email.message.EmailMessage`. - - In addition to the settable attributes listed above that apply to all - policies, this policy adds the following additional attributes: - - .. versionadded:: 3.6 [1]_ - - - .. attribute:: utf8 - - If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages - formatted in this way may be passed to SMTP servers that support - the ``SMTPUTF8`` extension (:rfc:`6531`). - - - .. attribute:: refold_source - - If the value for a header in the ``Message`` object originated from a - :mod:`~email.parser` (as opposed to being set by a program), this - attribute indicates whether or not a generator should refold that value - when transforming the message back into serialized form. The possible - values are: - - ======== =============================================================== - ``none`` all source values use original folding - - ``long`` source values that have any line that is longer than - ``max_line_length`` will be refolded - - ``all`` all values are refolded. - ======== =============================================================== - - The default is ``long``. - - - .. attribute:: header_factory - - A callable that takes two arguments, ``name`` and ``value``, where - ``name`` is a header field name and ``value`` is an unfolded header field - value, and returns a string subclass that represents that header. A - default ``header_factory`` (see :mod:`~email.headerregistry`) is provided - that supports custom parsing for the various address and date :RFC:`5322` - header field types, and the major MIME header field stypes. Support for - additional custom parsing will be added in the future. - - - .. attribute:: content_manager - - An object with at least two methods: get_content and set_content. When - the :meth:`~email.message.EmailMessage.get_content` or - :meth:`~email.message.EmailMessage.set_content` method of an - :class:`~email.message.EmailMessage` object is called, it calls the - corresponding method of this object, passing it the message object as its - first argument, and any arguments or keywords that were passed to it as - additional arguments. By default ``content_manager`` is set to - :data:`~email.contentmanager.raw_data_manager`. - - .. versionadded:: 3.4 - - - The class provides the following concrete implementations of the abstract - methods of :class:`Policy`: - - - .. method:: header_max_count(name) - - Returns the value of the - :attr:`~email.headerregistry.BaseHeader.max_count` attribute of the - specialized class used to represent the header with the given name. - - - .. method:: header_source_parse(sourcelines) - - - The name is parsed as everything up to the '``:``' and returned - unmodified. The value is determined by stripping leading whitespace off - the remainder of the first line, joining all subsequent lines together, - and stripping any trailing carriage return or linefeed characters. - - - .. method:: header_store_parse(name, value) - - The name is returned unchanged. If the input value has a ``name`` - attribute and it matches *name* ignoring case, the value is returned - unchanged. Otherwise the *name* and *value* are passed to - ``header_factory``, and the resulting header object is returned as - the value. In this case a ``ValueError`` is raised if the input value - contains CR or LF characters. - - - .. method:: header_fetch_parse(name, value) - - If the value has a ``name`` attribute, it is returned to unmodified. - Otherwise the *name*, and the *value* with any CR or LF characters - removed, are passed to the ``header_factory``, and the resulting - header object is returned. Any surrogateescaped bytes get turned into - the unicode unknown-character glyph. - - - .. method:: fold(name, value) - - Header folding is controlled by the :attr:`refold_source` policy setting. - A value is considered to be a 'source value' if and only if it does not - have a ``name`` attribute (having a ``name`` attribute means it is a - header object of some sort). If a source value needs to be refolded - according to the policy, it is converted into a header object by - passing the *name* and the *value* with any CR and LF characters removed - to the ``header_factory``. Folding of a header object is done by - calling its ``fold`` method with the current policy. - - Source values are split into lines using :meth:`~str.splitlines`. If - the value is not to be refolded, the lines are rejoined using the - ``linesep`` from the policy and returned. The exception is lines - containing non-ascii binary data. In that case the value is refolded - regardless of the ``refold_source`` setting, which causes the binary data - to be CTE encoded using the ``unknown-8bit`` charset. - - - .. method:: fold_binary(name, value) - - The same as :meth:`fold` if :attr:`~Policy.cte_type` is ``7bit``, except - that the returned value is bytes. - - If :attr:`~Policy.cte_type` is ``8bit``, non-ASCII binary data is - converted back - into bytes. Headers with binary data are not refolded, regardless of the - ``refold_header`` setting, since there is no way to know whether the - binary data consists of single byte characters or multibyte characters. - - -The following instances of :class:`EmailPolicy` provide defaults suitable for -specific application domains. Note that in the future the behavior of these -instances (in particular the ``HTTP`` instance) may be adjusted to conform even -more closely to the RFCs relevant to their domains. - - -.. data:: default - - An instance of ``EmailPolicy`` with all defaults unchanged. This policy - uses the standard Python ``\n`` line endings rather than the RFC-correct - ``\r\n``. - - -.. data:: SMTP - - Suitable for serializing messages in conformance with the email RFCs. - Like ``default``, but with ``linesep`` set to ``\r\n``, which is RFC - compliant. - - -.. data:: SMTPUTF8 - - The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. - Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP transmission if the - sender or recipient addresses have non-ASCII characters (the - :meth:`smtplib.SMTP.send_message` method handles this automatically). - - -.. data:: HTTP - - Suitable for serializing headers with for use in HTTP traffic. Like - ``SMTP`` except that ``max_line_length`` is set to ``None`` (unlimited). - - -.. data:: strict - - Convenience instance. The same as ``default`` except that - ``raise_on_defect`` is set to ``True``. This allows any policy to be made - strict by writing:: - - somepolicy + policy.strict - - -With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of -the email package is changed from the Python 3.2 API in the following ways: - - * Setting a header on a :class:`~email.message.Message` results in that - header being parsed and a header object created. - - * Fetching a header value from a :class:`~email.message.Message` results - in that header being parsed and a header object created and - returned. - - * Any header object, or any header that is refolded due to the - policy settings, is folded using an algorithm that fully implements the - RFC folding algorithms, including knowing where encoded words are required - and allowed. - -From the application view, this means that any header obtained through the -:class:`~email.message.EmailMessage` is a header object with extra -attributes, whose string value is the fully decoded unicode value of the -header. Likewise, a header may be assigned a new value, or a new header -created, using a unicode string, and the policy will take care of converting -the unicode string into the correct RFC encoded form. - -The header objects and their attributes are described in -:mod:`~email.headerregistry`. - - - -.. class:: Compat32(**kw) - - This concrete :class:`Policy` is the backward compatibility policy. It - replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, - :const:`compat32`, that is used as the default policy. Thus the default - behavior of the email package is to maintain compatibility with Python 3.2. - - The following attributes have values that are different from the - :class:`Policy` default: - - - .. attribute:: mangle_from_ - - The default is ``True``. - - - The class provides the following concrete implementations of the - abstract methods of :class:`Policy`: - - - .. method:: header_source_parse(sourcelines) - - The name is parsed as everything up to the '``:``' and returned - unmodified. The value is determined by stripping leading whitespace off - the remainder of the first line, joining all subsequent lines together, - and stripping any trailing carriage return or linefeed characters. - - - .. method:: header_store_parse(name, value) - - The name and value are returned unmodified. - - - .. method:: header_fetch_parse(name, value) - - If the value contains binary data, it is converted into a - :class:`~email.header.Header` object using the ``unknown-8bit`` charset. - Otherwise it is returned unmodified. - - - .. method:: fold(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. Non-ASCII binary data are - CTE encoded using the ``unknown-8bit`` charset. - - - .. method:: fold_binary(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. If ``cte_type`` is - ``7bit``, non-ascii binary data is CTE encoded using the ``unknown-8bit`` - charset. Otherwise the original source header is used, with its existing - line breaks and any (RFC invalid) binary data it may contain. - - -.. data:: compat32 - - An instance of :class:`Compat32`, providing backward compatibility with the - behavior of the email package in Python 3.2. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.3 as a :term:`provisional feature `. diff --git a/third_party/python/Doc/library/email.rst b/third_party/python/Doc/library/email.rst deleted file mode 100644 index fae99cf3e..000000000 --- a/third_party/python/Doc/library/email.rst +++ /dev/null @@ -1,152 +0,0 @@ -:mod:`email` --- An email and MIME handling package -=================================================== - -.. module:: email - :synopsis: Package supporting the parsing, manipulating, and generating - email messages. -.. moduleauthor:: Barry A. Warsaw , - R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/__init__.py` - --------------- - -The :mod:`email` package is a library for managing email messages. It is -specifically *not* designed to do any sending of email messages to SMTP -(:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as -RFC-compliant as possible, supporting :rfc:`5233` and :rfc:`6532`, as well as -such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, -and :rfc:`2231`. - -The overall structure of the email package can be divided into three major -components, plus a fourth component that controls the behavior of the other -components. - -The central component of the package is an "object model" that represents email -messages. An application interacts with the package primarily through the -object model interface defined in the :mod:`~email.message` sub-module. The -application can use this API to ask questions about an existing email, to -construct a new email, or to add or remove email subcomponents that themselves -use the same object model interface. That is, following the nature of email -messages and their MIME subcomponents, the email object model is a tree -structure of objects that all provide the :class:`~email.message.EmailMessage` -API. - -The other two major components of the package are the :mod:`~email.parser` and -the :mod:`~email.generator`. The parser takes the serialized version of an -email message (a stream of bytes) and converts it into a tree of -:class:`~email.message.EmailMessage` objects. The generator takes an -:class:`~email.message.EmailMessage` and turns it back into a serialized byte -stream. (The parser and generator also handle streams of text characters, but -this usage is discouraged as it is too easy to end up with messages that are -not valid in one way or another.) - -The control component is the :mod:`~email.policy` module. Every -:class:`~email.message.EmailMessage`, every :mod:`~email.generator`, and every -:mod:`~email.parser` has an associated :mod:`~email.policy` object that -controls its behavior. Usually an application only needs to specify the policy -when an :class:`~email.message.EmailMessage` is created, either by directly -instantiating an :class:`~email.message.EmailMessage` to create a new email, -or by parsing an input stream using a :mod:`~email.parser`. But the policy can -be changed when the message is serialized using a :mod:`~email.generator`. -This allows, for example, a generic email message to be parsed from disk, but -to serialize it using standard SMTP settings when sending it to an email -server. - -The email package does its best to hide the details of the various governing -RFCs from the application. Conceptually the application should be able to -treat the email message as a structured tree of unicode text and binary -attachments, without having to worry about how these are represented when -serialized. In practice, however, it is often necessary to be aware of at -least some of the rules governing MIME messages and their structure, -specifically the names and nature of the MIME "content types" and how they -identify multipart documents. For the most part this knowledge should only be -required for more complex applications, and even then it should only be the -high level structure in question, and not the details of how those structures -are represented. Since MIME content types are used widely in modern internet -software (not just email), this will be a familiar concept to many programmers. - -The following sections describe the functionality of the :mod:`email` package. -We start with the :mod:`~email.message` object model, which is the primary -interface an application will use, and follow that with the -:mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the -:mod:`~email.policy` controls, which completes the treatment of the main -components of the library. - -The next three sections cover the exceptions the package may raise and the -defects (non-compliance with the RFCs) that the :mod:`~email.parser` may -detect. Then we cover the :mod:`~email.headerregistry` and the -:mod:`~email.contentmanager` sub-components, which provide tools for doing more -detailed manipulation of headers and payloads, respectively. Both of these -components contain features relevant to consuming and producing non-trivial -messages, but also document their extensibility APIs, which will be of interest -to advanced applications. - -Following those is a set of examples of using the fundamental parts of the APIs -covered in the preceding sections. - -The foregoing represent the modern (unicode friendly) API of the email package. -The remaining sections, starting with the :class:`~email.message.Message` -class, cover the legacy :data:`~email.policy.compat32` API that deals much more -directly with the details of how email messages are represented. The -:data:`~email.policy.compat32` API does *not* hide the details of the RFCs from -the application, but for applications that need to operate at that level, they -can be useful tools. This documentation is also relevant for applications that -are still using the :mod:`~email.policy.compat32` API for backward -compatibility reasons. - -.. versionchanged:: 3.6 - Docs reorganized and rewritten to promote the new - :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` - API. - -Contents of the :mod:`email` package documentation: - -.. toctree:: - - email.message.rst - email.parser.rst - email.generator.rst - email.policy.rst - - email.errors.rst - email.headerregistry.rst - email.contentmanager.rst - - email.examples.rst - -Legacy API: - -.. toctree:: - - email.compat32-message.rst - email.mime.rst - email.header.rst - email.charset.rst - email.encoders.rst - email.utils.rst - email.iterators.rst - - -.. seealso:: - - Module :mod:`smtplib` - SMTP (Simple Mail Transport Protocol) client - - Module :mod:`poplib` - POP (Post Office Protocol) client - - Module :mod:`imaplib` - IMAP (Internet Message Access Protocol) client - - Module :mod:`nntplib` - NNTP (Net News Transport Protocol) client - - Module :mod:`mailbox` - Tools for creating, reading, and managing collections of messages on disk - using a variety standard formats. - - Module :mod:`smtpd` - SMTP server framework (primarily useful for testing) diff --git a/third_party/python/Doc/library/email.utils.rst b/third_party/python/Doc/library/email.utils.rst deleted file mode 100644 index 63fae2ab8..000000000 --- a/third_party/python/Doc/library/email.utils.rst +++ /dev/null @@ -1,218 +0,0 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- - -.. module:: email.utils - :synopsis: Miscellaneous email package utilities. - -**Source code:** :source:`Lib/email/utils.py` - --------------- - -There are a couple of useful utilities provided in the :mod:`email.utils` -module: - -.. function:: localtime(dt=None) - - Return local time as an aware datetime object. If called without - arguments, return current time. Otherwise *dt* argument should be a - :class:`~datetime.datetime` instance, and it is converted to the local time - zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. In this - case, a positive or zero value for *isdst* causes ``localtime`` to presume - initially that summer time (for example, Daylight Saving Time) is or is not - (respectively) in effect for the specified time. A negative value for - *isdst* causes the ``localtime`` to attempt to divine whether summer time - is in effect for the specified time. - - .. versionadded:: 3.3 - - -.. function:: make_msgid(idstring=None, domain=None) - - Returns a string suitable for an :rfc:`2822`\ -compliant - :mailheader:`Message-ID` header. Optional *idstring* if given, is a string - used to strengthen the uniqueness of the message id. Optional *domain* if - given provides the portion of the msgid after the '@'. The default is the - local hostname. It is not normally necessary to override this default, but - may be useful certain cases, such as a constructing distributed system that - uses a consistent domain name across multiple hosts. - - .. versionchanged:: 3.2 - Added the *domain* keyword. - - -The remaining functions are part of the legacy (``Compat32``) email API. There -is no need to directly use these with the new API, since the parsing and -formatting they provide is done automatically by the header parsing machinery -of the new API. - - -.. function:: quote(str) - - Return a new string with backslashes in *str* replaced by two backslashes, and - double quotes replaced by backslash-double quote. - - -.. function:: unquote(str) - - Return a new string which is an *unquoted* version of *str*. If *str* ends and - begins with double quotes, they are stripped off. Likewise if *str* ends and - begins with angle brackets, they are stripped off. - - -.. function:: parseaddr(address) - - Parse address -- which should be the value of some address-containing field such - as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and - *email address* parts. Returns a tuple of that information, unless the parse - fails, in which case a 2-tuple of ``('', '')`` is returned. - - -.. function:: formataddr(pair, charset='utf-8') - - The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname, - email_address)`` and returns the string value suitable for a :mailheader:`To` or - :mailheader:`Cc` header. If the first element of *pair* is false, then the - second element is returned unmodified. - - Optional *charset* is the character set that will be used in the :rfc:`2047` - encoding of the ``realname`` if the ``realname`` contains non-ASCII - characters. Can be an instance of :class:`str` or a - :class:`~email.charset.Charset`. Defaults to ``utf-8``. - - .. versionchanged:: 3.3 - Added the *charset* option. - - -.. function:: getaddresses(fieldvalues) - - This method returns a list of 2-tuples of the form returned by ``parseaddr()``. - *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: - - from email.utils import getaddresses - - tos = msg.get_all('to', []) - ccs = msg.get_all('cc', []) - resent_tos = msg.get_all('resent-to', []) - resent_ccs = msg.get_all('resent-cc', []) - all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) - - -.. function:: parsedate(date) - - Attempts to parse a date according to the rules in :rfc:`2822`. however, some - mailers don't follow that format as specified, so :func:`parsedate` tries to - guess correctly in such cases. *date* is a string containing an :rfc:`2822` - date, such as ``"Mon, 20 Nov 1995 19:12:08 -0500"``. If it succeeds in parsing - the date, :func:`parsedate` returns a 9-tuple that can be passed directly to - :func:`time.mktime`; otherwise ``None`` will be returned. Note that indexes 6, - 7, and 8 of the result tuple are not usable. - - -.. function:: parsedate_tz(date) - - Performs the same function as :func:`parsedate`, but returns either ``None`` or - a 10-tuple; the first 9 elements make up a tuple that can be passed directly to - :func:`time.mktime`, and the tenth is the offset of the date's timezone from UTC - (which is the official term for Greenwich Mean Time) [#]_. If the input string - has no timezone, the last element of the tuple returned is ``None``. Note that - indexes 6, 7, and 8 of the result tuple are not usable. - - -.. function:: parsedate_to_datetime(date) - - The inverse of :func:`format_datetime`. Performs the same function as - :func:`parsedate`, but on success returns a :mod:`~datetime.datetime`. If - the input date has a timezone of ``-0000``, the ``datetime`` will be a naive - ``datetime``, and if the date is conforming to the RFCs it will represent a - time in UTC but with no indication of the actual source timezone of the - message the date comes from. If the input date has any other valid timezone - offset, the ``datetime`` will be an aware ``datetime`` with the - corresponding a :class:`~datetime.timezone` :class:`~datetime.tzinfo`. - - .. versionadded:: 3.3 - - -.. function:: mktime_tz(tuple) - - Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC - timestamp (seconds since the Epoch). If the timezone item in the - tuple is ``None``, assume local time. - - -.. function:: formatdate(timeval=None, localtime=False, usegmt=False) - - Returns a date string as per :rfc:`2822`, e.g.:: - - Fri, 09 Nov 2001 01:08:47 -0000 - - Optional *timeval* if given is a floating point time value as accepted by - :func:`time.gmtime` and :func:`time.localtime`, otherwise the current time is - used. - - Optional *localtime* is a flag that when ``True``, interprets *timeval*, and - returns a date relative to the local timezone instead of UTC, properly taking - daylight savings time into account. The default is ``False`` meaning UTC is - used. - - Optional *usegmt* is a flag that when ``True``, outputs a date string with the - timezone as an ascii string ``GMT``, rather than a numeric ``-0000``. This is - needed for some protocols (such as HTTP). This only applies when *localtime* is - ``False``. The default is ``False``. - - -.. function:: format_datetime(dt, usegmt=False) - - Like ``formatdate``, but the input is a :mod:`datetime` instance. If it is - a naive datetime, it is assumed to be "UTC with no information about the - source timezone", and the conventional ``-0000`` is used for the timezone. - If it is an aware ``datetime``, then the numeric timezone offset is used. - If it is an aware timezone with offset zero, then *usegmt* may be set to - ``True``, in which case the string ``GMT`` is used instead of the numeric - timezone offset. This provides a way to generate standards conformant HTTP - date headers. - - .. versionadded:: 3.3 - - -.. function:: decode_rfc2231(s) - - Decode the string *s* according to :rfc:`2231`. - - -.. function:: encode_rfc2231(s, charset=None, language=None) - - Encode the string *s* according to :rfc:`2231`. Optional *charset* and - *language*, if given is the character set name and language name to use. If - neither is given, *s* is returned as-is. If *charset* is given but *language* - is not, the string is encoded using the empty string for *language*. - - -.. function:: collapse_rfc2231_value(value, errors='replace', fallback_charset='us-ascii') - - When a header parameter is encoded in :rfc:`2231` format, - :meth:`Message.get_param ` may return a - 3-tuple containing the character set, - language, and value. :func:`collapse_rfc2231_value` turns this into a unicode - string. Optional *errors* is passed to the *errors* argument of :class:`str`'s - :func:`~str.encode` method; it defaults to ``'replace'``. Optional - *fallback_charset* specifies the character set to use if the one in the - :rfc:`2231` header is not known by Python; it defaults to ``'us-ascii'``. - - For convenience, if the *value* passed to :func:`collapse_rfc2231_value` is not - a tuple, it should be a string and it is returned unquoted. - - -.. function:: decode_params(params) - - Decode parameters list according to :rfc:`2231`. *params* is a sequence of - 2-tuples containing elements of the form ``(content-type, string-value)``. - - -.. rubric:: Footnotes - -.. [#] Note that the sign of the timezone offset is the opposite of the sign of the - ``time.timezone`` variable for the same timezone; the latter variable follows - the POSIX standard while this module follows :rfc:`2822`. diff --git a/third_party/python/Doc/library/ensurepip.rst b/third_party/python/Doc/library/ensurepip.rst deleted file mode 100644 index 652339e57..000000000 --- a/third_party/python/Doc/library/ensurepip.rst +++ /dev/null @@ -1,136 +0,0 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== - -.. module:: ensurepip - :synopsis: Bootstrapping the "pip" installer into an existing Python - installation or virtual environment. - -.. versionadded:: 3.4 - --------------- - -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` -installer into an existing Python installation or virtual environment. This -bootstrapping approach reflects the fact that ``pip`` is an independent -project with its own release cycle, and the latest available stable version -is bundled with maintenance and feature releases of the CPython reference -interpreter. - -In most cases, end users of Python shouldn't need to invoke this module -directly (as ``pip`` should be bootstrapped by default), but it may be -needed if installing ``pip`` was skipped when installing Python (or -when creating a virtual environment) or after explicitly uninstalling -``pip``. - -.. note:: - - This module *does not* access the internet. All of the components - needed to bootstrap ``pip`` are included as internal parts of the - package. - -.. seealso:: - - :ref:`installing-index` - The end user guide for installing Python packages - - :pep:`453`: Explicit bootstrapping of pip in Python installations - The original rationale and specification for this module. - - -Command line interface ----------------------- - -The command line interface is invoked using the interpreter's ``-m`` switch. - -The simplest possible invocation is:: - - python -m ensurepip - -This invocation will install ``pip`` if it is not already installed, -but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one bundled with ``ensurepip``, pass the -``--upgrade`` option:: - - python -m ensurepip --upgrade - -By default, ``pip`` is installed into the current virtual environment -(if one is active) or into the system site packages (if there is no -active virtual environment). The installation location can be controlled -through two additional command line options: - -* ``--root ``: Installs ``pip`` relative to the given root directory - rather than the root of the currently active virtual environment (if any) - or the default root for the current Python installation. -* ``--user``: Installs ``pip`` into the user site packages directory rather - than globally for the current Python installation (this option is not - permitted inside an active virtual environment). - -By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where -X.Y stands for the version of Python used to invoke ``ensurepip``). The -scripts installed can be controlled through two additional command line -options: - -* ``--altinstall``: if an alternate installation is requested, the ``pipX`` - script will *not* be installed. - -* ``--default-pip``: if a "default pip" installation is requested, the - ``pip`` script will be installed in addition to the two regular scripts. - -Providing both of the script selection options will trigger an exception. - -.. versionchanged:: 3.6.3 - The exit status is non-zero if the command fails. - - -Module API ----------- - -:mod:`ensurepip` exposes two functions for programmatic use: - -.. function:: version() - - Returns a string specifying the bundled version of pip that will be - installed when bootstrapping an environment. - -.. function:: bootstrap(root=None, upgrade=False, user=False, \ - altinstall=False, default_pip=False, \ - verbosity=0) - - Bootstraps ``pip`` into the current or designated environment. - - *root* specifies an alternative root directory to install relative to. - If *root* is ``None``, then installation uses the default install location - for the current environment. - - *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the bundled version. - - *user* indicates whether to use the user scheme rather than installing - globally. - - By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where - X.Y stands for the current version of Python). - - If *altinstall* is set, then ``pipX`` will *not* be installed. - - If *default_pip* is set, then ``pip`` will be installed in addition to - the two regular scripts. - - Setting both *altinstall* and *default_pip* will trigger - :exc:`ValueError`. - - *verbosity* controls the level of output to :data:`sys.stdout` from the - bootstrapping operation. - - .. note:: - - The bootstrapping process has side effects on both ``sys.path`` and - ``os.environ``. Invoking the command line interface in a subprocess - instead allows these side effects to be avoided. - - .. note:: - - The bootstrapping process may install additional modules required by - ``pip``, but other software should not assume those dependencies will - always be present by default (as the dependencies may be removed in a - future version of ``pip``). diff --git a/third_party/python/Doc/library/enum.rst b/third_party/python/Doc/library/enum.rst deleted file mode 100644 index 2bf4885d2..000000000 --- a/third_party/python/Doc/library/enum.rst +++ /dev/null @@ -1,1093 +0,0 @@ -:mod:`enum` --- Support for enumerations -======================================== - -.. module:: enum - :synopsis: Implementation of an enumeration class. - -.. moduleauthor:: Ethan Furman -.. sectionauthor:: Barry Warsaw -.. sectionauthor:: Eli Bendersky -.. sectionauthor:: Ethan Furman - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/enum.py` - ----------------- - -An enumeration is a set of symbolic names (members) bound to unique, -constant values. Within an enumeration, the members can be compared -by identity, and the enumeration itself can be iterated over. - - -Module Contents ---------------- - -This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and -:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one -helper, :class:`auto`. - -.. class:: Enum - - Base class for creating enumerated constants. See section - `Functional API`_ for an alternate construction syntax. - -.. class:: IntEnum - - Base class for creating enumerated constants that are also - subclasses of :class:`int`. - -.. class:: IntFlag - - Base class for creating enumerated constants that can be combined using - the bitwise operators without losing their :class:`IntFlag` membership. - :class:`IntFlag` members are also subclasses of :class:`int`. - -.. class:: Flag - - Base class for creating enumerated constants that can be combined using - the bitwise operations without losing their :class:`Flag` membership. - -.. function:: unique - - Enum class decorator that ensures only one name is bound to any one value. - -.. class:: auto - - Instances are replaced with an appropriate value for Enum members. - -.. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` - - -Creating an Enum ----------------- - -Enumerations are created using the :keyword:`class` syntax, which makes them -easy to read and write. An alternative creation method is described in -`Functional API`_. To define an enumeration, subclass :class:`Enum` as -follows:: - - >>> from enum import Enum - >>> class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - ... - -.. note:: Enum member values - - Member values can be anything: :class:`int`, :class:`str`, etc.. If - the exact value is unimportant you may use :class:`auto` instances and an - appropriate value will be chosen for you. Care must be taken if you mix - :class:`auto` with other values. - -.. note:: Nomenclature - - - The class :class:`Color` is an *enumeration* (or *enum*) - - The attributes :attr:`Color.RED`, :attr:`Color.GREEN`, etc., are - *enumeration members* (or *enum members*) and are functionally constants. - - The enum members have *names* and *values* (the name of - :attr:`Color.RED` is ``RED``, the value of :attr:`Color.BLUE` is - ``3``, etc.) - -.. note:: - - Even though we use the :keyword:`class` syntax to create Enums, Enums - are not normal Python classes. See `How are Enums different?`_ for - more details. - -Enumeration members have human readable string representations:: - - >>> print(Color.RED) - Color.RED - -...while their ``repr`` has more information:: - - >>> print(repr(Color.RED)) - - -The *type* of an enumeration member is the enumeration it belongs to:: - - >>> type(Color.RED) - - >>> isinstance(Color.GREEN, Color) - True - >>> - -Enum members also have a property that contains just their item name:: - - >>> print(Color.RED.name) - RED - -Enumerations support iteration, in definition order:: - - >>> class Shake(Enum): - ... VANILLA = 7 - ... CHOCOLATE = 4 - ... COOKIES = 9 - ... MINT = 3 - ... - >>> for shake in Shake: - ... print(shake) - ... - Shake.VANILLA - Shake.CHOCOLATE - Shake.COOKIES - Shake.MINT - -Enumeration members are hashable, so they can be used in dictionaries and sets:: - - >>> apples = {} - >>> apples[Color.RED] = 'red delicious' - >>> apples[Color.GREEN] = 'granny smith' - >>> apples == {Color.RED: 'red delicious', Color.GREEN: 'granny smith'} - True - - -Programmatic access to enumeration members and their attributes ---------------------------------------------------------------- - -Sometimes it's useful to access members in enumerations programmatically (i.e. -situations where ``Color.RED`` won't do because the exact color is not known -at program-writing time). ``Enum`` allows such access:: - - >>> Color(1) - - >>> Color(3) - - -If you want to access enum members by *name*, use item access:: - - >>> Color['RED'] - - >>> Color['GREEN'] - - -If you have an enum member and need its :attr:`name` or :attr:`value`:: - - >>> member = Color.RED - >>> member.name - 'RED' - >>> member.value - 1 - - -Duplicating enum members and values ------------------------------------ - -Having two enum members with the same name is invalid:: - - >>> class Shape(Enum): - ... SQUARE = 2 - ... SQUARE = 3 - ... - Traceback (most recent call last): - ... - TypeError: Attempted to reuse key: 'SQUARE' - -However, two enum members are allowed to have the same value. Given two members -A and B with the same value (and A defined first), B is an alias to A. By-value -lookup of the value of A and B will return A. By-name lookup of B will also -return A:: - - >>> class Shape(Enum): - ... SQUARE = 2 - ... DIAMOND = 1 - ... CIRCLE = 3 - ... ALIAS_FOR_SQUARE = 2 - ... - >>> Shape.SQUARE - - >>> Shape.ALIAS_FOR_SQUARE - - >>> Shape(2) - - -.. note:: - - Attempting to create a member with the same name as an already - defined attribute (another member, a method, etc.) or attempting to create - an attribute with the same name as a member is not allowed. - - -Ensuring unique enumeration values ----------------------------------- - -By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, the following decorator can be used to -ensure each value is used only once in the enumeration: - -.. decorator:: unique - -A :keyword:`class` decorator specifically for enumerations. It searches an -enumeration's :attr:`__members__` gathering any aliases it finds; if any are -found :exc:`ValueError` is raised with the details:: - - >>> from enum import Enum, unique - >>> @unique - ... class Mistake(Enum): - ... ONE = 1 - ... TWO = 2 - ... THREE = 3 - ... FOUR = 3 - ... - Traceback (most recent call last): - ... - ValueError: duplicate values found in : FOUR -> THREE - - -Using automatic values ----------------------- - -If the exact value is unimportant you can use :class:`auto`:: - - >>> from enum import Enum, auto - >>> class Color(Enum): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> list(Color) - [, , ] - -The values are chosen by :func:`_generate_next_value_`, which can be -overridden:: - - >>> class AutoName(Enum): - ... def _generate_next_value_(name, start, count, last_values): - ... return name - ... - >>> class Ordinal(AutoName): - ... NORTH = auto() - ... SOUTH = auto() - ... EAST = auto() - ... WEST = auto() - ... - >>> list(Ordinal) - [, , , ] - -.. note:: - - The goal of the default :meth:`_generate_next_value_` methods is to provide - the next :class:`int` in sequence with the last :class:`int` provided, but - the way it does this is an implementation detail and may change. - -Iteration ---------- - -Iterating over the members of an enum does not provide the aliases:: - - >>> list(Shape) - [, , ] - -The special attribute ``__members__`` is an ordered dictionary mapping names -to members. It includes all names defined in the enumeration, including the -aliases:: - - >>> for name, member in Shape.__members__.items(): - ... name, member - ... - ('SQUARE', ) - ('DIAMOND', ) - ('CIRCLE', ) - ('ALIAS_FOR_SQUARE', ) - -The ``__members__`` attribute can be used for detailed programmatic access to -the enumeration members. For example, finding all the aliases:: - - >>> [name for name, member in Shape.__members__.items() if member.name != name] - ['ALIAS_FOR_SQUARE'] - - -Comparisons ------------ - -Enumeration members are compared by identity:: - - >>> Color.RED is Color.RED - True - >>> Color.RED is Color.BLUE - False - >>> Color.RED is not Color.BLUE - True - -Ordered comparisons between enumeration values are *not* supported. Enum -members are not integers (but see `IntEnum`_ below):: - - >>> Color.RED < Color.BLUE - Traceback (most recent call last): - File "", line 1, in - TypeError: '<' not supported between instances of 'Color' and 'Color' - -Equality comparisons are defined though:: - - >>> Color.BLUE == Color.RED - False - >>> Color.BLUE != Color.RED - True - >>> Color.BLUE == Color.BLUE - True - -Comparisons against non-enumeration values will always compare not equal -(again, :class:`IntEnum` was explicitly designed to behave differently, see -below):: - - >>> Color.BLUE == 2 - False - - -Allowed members and attributes of enumerations ----------------------------------------------- - -The examples above use integers for enumeration values. Using integers is -short and handy (and provided by default by the `Functional API`_), but not -strictly enforced. In the vast majority of use-cases, one doesn't care what -the actual value of an enumeration is. But if the value *is* important, -enumerations can have arbitrary values. - -Enumerations are Python classes, and can have methods and special methods as -usual. If we have this enumeration:: - - >>> class Mood(Enum): - ... FUNKY = 1 - ... HAPPY = 3 - ... - ... def describe(self): - ... # self is the member here - ... return self.name, self.value - ... - ... def __str__(self): - ... return 'my custom str! {0}'.format(self.value) - ... - ... @classmethod - ... def favorite_mood(cls): - ... # cls here is the enumeration - ... return cls.HAPPY - ... - -Then:: - - >>> Mood.favorite_mood() - - >>> Mood.HAPPY.describe() - ('HAPPY', 3) - >>> str(Mood.FUNKY) - 'my custom str! 1' - -The rules for what is allowed are as follows: names that start and end with -a single underscore are reserved by enum and cannot be used; all other -attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.) and descriptors (methods are also descriptors). - -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then -whatever value(s) were given to the enum member will be passed into those -methods. See `Planet`_ for an example. - - -Restricted subclassing of enumerations --------------------------------------- - -Subclassing an enumeration is allowed only if the enumeration does not define -any members. So this is forbidden:: - - >>> class MoreColor(Color): - ... PINK = 17 - ... - Traceback (most recent call last): - ... - TypeError: Cannot extend enumerations - -But this is allowed:: - - >>> class Foo(Enum): - ... def some_behavior(self): - ... pass - ... - >>> class Bar(Foo): - ... HAPPY = 1 - ... SAD = 2 - ... - -Allowing subclassing of enums that define members would lead to a violation of -some important invariants of types and instances. On the other hand, it makes -sense to allow sharing some common behavior between a group of enumerations. -(See `OrderedEnum`_ for an example.) - - -Pickling --------- - -Enumerations can be pickled and unpickled:: - - >>> from test.test_enum import Fruit - >>> from pickle import dumps, loads - >>> Fruit.TOMATO is loads(dumps(Fruit.TOMATO)) - True - -The usual restrictions for pickling apply: picklable enums must be defined in -the top level of a module, since unpickling requires them to be importable -from that module. - -.. note:: - - With pickle protocol version 4 it is possible to easily pickle enums - nested in other classes. - -It is possible to modify how Enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. - - -Functional API --------------- - -The :class:`Enum` class is callable, providing the following functional API:: - - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG') - >>> Animal - - >>> Animal.ANT - - >>> Animal.ANT.value - 1 - >>> list(Animal) - [, , , ] - -The semantics of this API resemble :class:`~collections.namedtuple`. The first -argument of the call to :class:`Enum` is the name of the enumeration. - -The second argument is the *source* of enumeration member names. It can be a -whitespace-separated string of names, a sequence of names, a sequence of -2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to -values. The last two options enable assigning arbitrary values to -enumerations; the others auto-assign increasing integers starting with 1 (use -the ``start`` parameter to specify a different starting value). A -new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: - - >>> class Animal(Enum): - ... ANT = 1 - ... BEE = 2 - ... CAT = 3 - ... DOG = 4 - ... - -The reason for defaulting to ``1`` as the starting number and not ``0`` is -that ``0`` is ``False`` in a boolean sense, but enum members all evaluate -to ``True``. - -Pickling enums created with the functional API can be tricky as frame stack -implementation details are used to try and figure out which module the -enumeration is being created in (e.g. it will fail if you use a utility -function in separate module, and also may not work on IronPython or Jython). -The solution is to specify the module name explicitly as follows:: - - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', module=__name__) - -.. warning:: - - If ``module`` is not supplied, and Enum cannot determine what it is, - the new Enum members will not be unpicklable; to keep errors closer to - the source, pickling will be disabled. - -The new pickle protocol 4 also, in some circumstances, relies on -:attr:`~definition.__qualname__` being set to the location where pickle will be able -to find the class. For example, if the class was made available in class -SomeData in the global scope:: - - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', qualname='SomeData.Animal') - -The complete signature is:: - - Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=, start=1) - -:value: What the new Enum class will record as its name. - -:names: The Enum members. This can be a whitespace or comma separated string - (values will start at 1 unless otherwise specified):: - - 'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE' - - or an iterator of names:: - - ['RED', 'GREEN', 'BLUE'] - - or an iterator of (name, value) pairs:: - - [('CYAN', 4), ('MAGENTA', 5), ('YELLOW', 6)] - - or a mapping:: - - {'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42} - -:module: name of module where new Enum class can be found. - -:qualname: where in module new Enum class can be found. - -:type: type to mix in to new Enum class. - -:start: number to start counting at if only names are passed in. - -.. versionchanged:: 3.5 - The *start* parameter was added. - - -Derived Enumerations --------------------- - -IntEnum -^^^^^^^ - -The first variation of :class:`Enum` that is provided is also a subclass of -:class:`int`. Members of an :class:`IntEnum` can be compared to integers; -by extension, integer enumerations of different types can also be compared -to each other:: - - >>> from enum import IntEnum - >>> class Shape(IntEnum): - ... CIRCLE = 1 - ... SQUARE = 2 - ... - >>> class Request(IntEnum): - ... POST = 1 - ... GET = 2 - ... - >>> Shape == 1 - False - >>> Shape.CIRCLE == 1 - True - >>> Shape.CIRCLE == Request.POST - True - -However, they still can't be compared to standard :class:`Enum` enumerations:: - - >>> class Shape(IntEnum): - ... CIRCLE = 1 - ... SQUARE = 2 - ... - >>> class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... - >>> Shape.CIRCLE == Color.RED - False - -:class:`IntEnum` values behave like integers in other ways you'd expect:: - - >>> int(Shape.CIRCLE) - 1 - >>> ['a', 'b', 'c'][Shape.CIRCLE] - 'b' - >>> [i for i in range(Shape.SQUARE)] - [0, 1] - - -IntFlag -^^^^^^^ - -The next variation of :class:`Enum` provided, :class:`IntFlag`, is also based -on :class:`int`. The difference being :class:`IntFlag` members can be combined -using the bitwise operators (&, \|, ^, ~) and the result is still an -:class:`IntFlag` member. However, as the name implies, :class:`IntFlag` -members also subclass :class:`int` and can be used wherever an :class:`int` is -used. Any operation on an :class:`IntFlag` member besides the bit-wise -operations will lose the :class:`IntFlag` membership. - -.. versionadded:: 3.6 - -Sample :class:`IntFlag` class:: - - >>> from enum import IntFlag - >>> class Perm(IntFlag): - ... R = 4 - ... W = 2 - ... X = 1 - ... - >>> Perm.R | Perm.W - - >>> Perm.R + Perm.W - 6 - >>> RW = Perm.R | Perm.W - >>> Perm.R in RW - True - -It is also possible to name the combinations:: - - >>> class Perm(IntFlag): - ... R = 4 - ... W = 2 - ... X = 1 - ... RWX = 7 - >>> Perm.RWX - - >>> ~Perm.RWX - - -Another important difference between :class:`IntFlag` and :class:`Enum` is that -if no flags are set (the value is 0), its boolean evaluation is :data:`False`:: - - >>> Perm.R & Perm.X - - >>> bool(Perm.R & Perm.X) - False - -Because :class:`IntFlag` members are also subclasses of :class:`int` they can -be combined with them:: - - >>> Perm.X | 8 - - - -Flag -^^^^ - -The last variation is :class:`Flag`. Like :class:`IntFlag`, :class:`Flag` -members can be combined using the bitwise operators (&, \|, ^, ~). Unlike -:class:`IntFlag`, they cannot be combined with, nor compared against, any -other :class:`Flag` enumeration, nor :class:`int`. While it is possible to -specify the values directly it is recommended to use :class:`auto` as the -value and let :class:`Flag` select an appropriate value. - -.. versionadded:: 3.6 - -Like :class:`IntFlag`, if a combination of :class:`Flag` members results in no -flags being set, the boolean evaluation is :data:`False`:: - - >>> from enum import Flag, auto - >>> class Color(Flag): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.RED & Color.GREEN - - >>> bool(Color.RED & Color.GREEN) - False - -Individual flags should have values that are powers of two (1, 2, 4, 8, ...), -while combinations of flags won't:: - - >>> class Color(Flag): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... WHITE = RED | BLUE | GREEN - ... - >>> Color.WHITE - - -Giving a name to the "no flags set" condition does not change its boolean -value:: - - >>> class Color(Flag): - ... BLACK = 0 - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.BLACK - - >>> bool(Color.BLACK) - False - -.. note:: - - For the majority of new code, :class:`Enum` and :class:`Flag` are strongly - recommended, since :class:`IntEnum` and :class:`IntFlag` break some - semantic promises of an enumeration (by being comparable to integers, and - thus by transitivity to other unrelated enumerations). :class:`IntEnum` - and :class:`IntFlag` should be used only in cases where :class:`Enum` and - :class:`Flag` will not do; for example, when integer constants are replaced - with enumerations, or for interoperability with other systems. - - -Others -^^^^^^ - -While :class:`IntEnum` is part of the :mod:`enum` module, it would be very -simple to implement independently:: - - class IntEnum(int, Enum): - pass - -This demonstrates how similar derived enumerations can be defined; for example -a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`. - -Some rules: - -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` - example above. -2. While :class:`Enum` can have members of any type, once you mix in an - additional type, all the members must have values of that type, e.g. - :class:`int` above. This restriction does not apply to mix-ins which only - add methods and don't specify another data type such as :class:`int` or - :class:`str`. -3. When another data type is mixed in, the :attr:`value` attribute is *not the - same* as the enum member itself, although it is equivalent and will compare - equal. -4. %-style formatting: `%s` and `%r` call the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as - `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the mixed-in - type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or - :func:`repr` is desired, use the `!s` or `!r` format codes. - - -Interesting examples --------------------- - -While :class:`Enum`, :class:`IntEnum`, :class:`IntFlag`, and :class:`Flag` are -expected to cover the majority of use-cases, they cannot cover them all. Here -are recipes for some different types of enumerations that can be used directly, -or as examples for creating one's own. - - -Omitting values -^^^^^^^^^^^^^^^ - -In many use-cases one doesn't care what the actual value of an enumeration -is. There are several ways to define this type of simple enumeration: - -- use instances of :class:`auto` for the value -- use instances of :class:`object` as the value -- use a descriptive string as the value -- use a tuple as the value and a custom :meth:`__new__` to replace the - tuple with an :class:`int` value - -Using any of these methods signifies to the user that these values are not -important, and also enables one to add, remove, or reorder members without -having to renumber the remaining members. - -Whichever method you choose, you should provide a :meth:`repr` that also hides -the (unimportant) value:: - - >>> class NoValue(Enum): - ... def __repr__(self): - ... return '<%s.%s>' % (self.__class__.__name__, self.name) - ... - - -Using :class:`auto` -""""""""""""""""""" - -Using :class:`auto` would look like:: - - >>> class Color(NoValue): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.GREEN - - - -Using :class:`object` -""""""""""""""""""""" - -Using :class:`object` would look like:: - - >>> class Color(NoValue): - ... RED = object() - ... GREEN = object() - ... BLUE = object() - ... - >>> Color.GREEN - - - -Using a descriptive string -"""""""""""""""""""""""""" - -Using a string as the value would look like:: - - >>> class Color(NoValue): - ... RED = 'stop' - ... GREEN = 'go' - ... BLUE = 'too fast!' - ... - >>> Color.GREEN - - >>> Color.GREEN.value - 'go' - - -Using a custom :meth:`__new__` -"""""""""""""""""""""""""""""" - -Using an auto-numbering :meth:`__new__` would look like:: - - >>> class AutoNumber(NoValue): - ... def __new__(cls): - ... value = len(cls.__members__) + 1 - ... obj = object.__new__(cls) - ... obj._value_ = value - ... return obj - ... - >>> class Color(AutoNumber): - ... RED = () - ... GREEN = () - ... BLUE = () - ... - >>> Color.GREEN - - >>> Color.GREEN.value - 2 - - -.. note:: - - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after - class creation for lookup of existing members. - - -OrderedEnum -^^^^^^^^^^^ - -An ordered enumeration that is not based on :class:`IntEnum` and so maintains -the normal :class:`Enum` invariants (such as not being comparable to other -enumerations):: - - >>> class OrderedEnum(Enum): - ... def __ge__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value >= other.value - ... return NotImplemented - ... def __gt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value > other.value - ... return NotImplemented - ... def __le__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value <= other.value - ... return NotImplemented - ... def __lt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value < other.value - ... return NotImplemented - ... - >>> class Grade(OrderedEnum): - ... A = 5 - ... B = 4 - ... C = 3 - ... D = 2 - ... F = 1 - ... - >>> Grade.C < Grade.A - True - - -DuplicateFreeEnum -^^^^^^^^^^^^^^^^^ - -Raises an error if a duplicate member name is found instead of creating an -alias:: - - >>> class DuplicateFreeEnum(Enum): - ... def __init__(self, *args): - ... cls = self.__class__ - ... if any(self.value == e.value for e in cls): - ... a = self.name - ... e = cls(self.value).name - ... raise ValueError( - ... "aliases not allowed in DuplicateFreeEnum: %r --> %r" - ... % (a, e)) - ... - >>> class Color(DuplicateFreeEnum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - ... GRENE = 2 - ... - Traceback (most recent call last): - ... - ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' - -.. note:: - - This is a useful example for subclassing Enum to add or change other - behaviors as well as disallowing aliases. If the only desired change is - disallowing aliases, the :func:`unique` decorator can be used instead. - - -Planet -^^^^^^ - -If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member -will be passed to those methods:: - - >>> class Planet(Enum): - ... MERCURY = (3.303e+23, 2.4397e6) - ... VENUS = (4.869e+24, 6.0518e6) - ... EARTH = (5.976e+24, 6.37814e6) - ... MARS = (6.421e+23, 3.3972e6) - ... JUPITER = (1.9e+27, 7.1492e7) - ... SATURN = (5.688e+26, 6.0268e7) - ... URANUS = (8.686e+25, 2.5559e7) - ... NEPTUNE = (1.024e+26, 2.4746e7) - ... def __init__(self, mass, radius): - ... self.mass = mass # in kilograms - ... self.radius = radius # in meters - ... @property - ... def surface_gravity(self): - ... # universal gravitational constant (m3 kg-1 s-2) - ... G = 6.67300E-11 - ... return G * self.mass / (self.radius * self.radius) - ... - >>> Planet.EARTH.value - (5.976e+24, 6378140.0) - >>> Planet.EARTH.surface_gravity - 9.802652743337129 - - -How are Enums different? ------------------------- - -Enums have a custom metaclass that affects many aspects of both derived Enum -classes and their instances (members). - - -Enum Classes -^^^^^^^^^^^^ - -The :class:`EnumMeta` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that -allow one to do things with an :class:`Enum` class that fail on a typical -class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is -responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). - - -Enum Members (aka instances) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The most interesting thing about Enum members is that they are singletons. -:class:`EnumMeta` creates them all while it is creating the :class:`Enum` -class itself, and then puts a custom :meth:`__new__` in place to ensure -that no new ones are ever instantiated by returning only the existing -member instances. - - -Finer Points -^^^^^^^^^^^^ - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`__members__` is an :class:`OrderedDict` of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`_value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" - -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used by the `Functional API`_ and by - :class:`auto` to get an appropriate value for an enum member; may be - overridden - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_ - -.. note:: - - In Python 2 code the :attr:`_order_` attribute is necessary as definition - order is lost before it can be recorded. - -``Enum`` member type -"""""""""""""""""""" - -:class:`Enum` members are instances of their :class:`Enum` class, and are -normally accessed as ``EnumClass.member``. Under certain circumstances they -can also be accessed as ``EnumClass.member.member``, but you should never do -this as that lookup may fail or, worse, return something besides the -:class:`Enum` member you are looking for (this is another good reason to use -all-uppercase names for members):: - - >>> class FieldTypes(Enum): - ... name = 0 - ... value = 1 - ... size = 2 - ... - >>> FieldTypes.value.size - - >>> FieldTypes.size.value - 2 - -.. versionchanged:: 3.5 - - -Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" - -:class:`Enum` members that are mixed with non-:class:`Enum` types (such as -:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in -type's rules; otherwise, all members evaluate as :data:`True`. To make your -own Enum's boolean evaluation depend on the member's value add the following to -your class:: - - def __bool__(self): - return bool(self.value) - -:class:`Enum` classes always evaluate as :data:`True`. - - -``Enum`` classes with methods -""""""""""""""""""""""""""""" - -If you give your :class:`Enum` subclass extra methods, like the `Planet`_ -class above, those methods will show up in a :func:`dir` of the member, -but not of the class:: - - >>> dir(Planet) - ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] - >>> dir(Planet.EARTH) - ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] - - -Combining members of ``Flag`` -""""""""""""""""""""""""""""" - -If a combination of Flag members is not named, the :func:`repr` will include -all named flags and all named combinations of flags that are in the value:: - - >>> class Color(Flag): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - ... MAGENTA = RED | BLUE - ... YELLOW = RED | GREEN - ... CYAN = GREEN | BLUE - ... - >>> Color(3) # named combination - - >>> Color(7) # not named combination - - diff --git a/third_party/python/Doc/library/errno.rst b/third_party/python/Doc/library/errno.rst deleted file mode 100644 index 1cbd51c58..000000000 --- a/third_party/python/Doc/library/errno.rst +++ /dev/null @@ -1,639 +0,0 @@ -:mod:`errno` --- Standard errno system symbols -============================================== - -.. module:: errno - :synopsis: Standard errno system symbols. - ----------------- - -This module makes available standard ``errno`` system symbols. The value of each -symbol is the corresponding integer value. The names and descriptions are -borrowed from :file:`linux/include/errno.h`, which should be pretty -all-inclusive. - - -.. data:: errorcode - - Dictionary providing a mapping from the errno value to the string name in the - underlying system. For instance, ``errno.errorcode[errno.EPERM]`` maps to - ``'EPERM'``. - -To translate a numeric error code to an error message, use :func:`os.strerror`. - -Of the following list, symbols that are not used on the current platform are not -defined by the module. The specific list of defined symbols is available as -``errno.errorcode.keys()``. Symbols available can include: - - -.. data:: EPERM - - Operation not permitted - - -.. data:: ENOENT - - No such file or directory - - -.. data:: ESRCH - - No such process - - -.. data:: EINTR - - Interrupted system call. - - .. seealso:: - This error is mapped to the exception :exc:`InterruptedError`. - - -.. data:: EIO - - I/O error - - -.. data:: ENXIO - - No such device or address - - -.. data:: E2BIG - - Arg list too long - - -.. data:: ENOEXEC - - Exec format error - - -.. data:: EBADF - - Bad file number - - -.. data:: ECHILD - - No child processes - - -.. data:: EAGAIN - - Try again - - -.. data:: ENOMEM - - Out of memory - - -.. data:: EACCES - - Permission denied - - -.. data:: EFAULT - - Bad address - - -.. data:: ENOTBLK - - Block device required - - -.. data:: EBUSY - - Device or resource busy - - -.. data:: EEXIST - - File exists - - -.. data:: EXDEV - - Cross-device link - - -.. data:: ENODEV - - No such device - - -.. data:: ENOTDIR - - Not a directory - - -.. data:: EISDIR - - Is a directory - - -.. data:: EINVAL - - Invalid argument - - -.. data:: ENFILE - - File table overflow - - -.. data:: EMFILE - - Too many open files - - -.. data:: ENOTTY - - Not a typewriter - - -.. data:: ETXTBSY - - Text file busy - - -.. data:: EFBIG - - File too large - - -.. data:: ENOSPC - - No space left on device - - -.. data:: ESPIPE - - Illegal seek - - -.. data:: EROFS - - Read-only file system - - -.. data:: EMLINK - - Too many links - - -.. data:: EPIPE - - Broken pipe - - -.. data:: EDOM - - Math argument out of domain of func - - -.. data:: ERANGE - - Math result not representable - - -.. data:: EDEADLK - - Resource deadlock would occur - - -.. data:: ENAMETOOLONG - - File name too long - - -.. data:: ENOLCK - - No record locks available - - -.. data:: ENOSYS - - Function not implemented - - -.. data:: ENOTEMPTY - - Directory not empty - - -.. data:: ELOOP - - Too many symbolic links encountered - - -.. data:: EWOULDBLOCK - - Operation would block - - -.. data:: ENOMSG - - No message of desired type - - -.. data:: EIDRM - - Identifier removed - - -.. data:: ECHRNG - - Channel number out of range - - -.. data:: EL2NSYNC - - Level 2 not synchronized - - -.. data:: EL3HLT - - Level 3 halted - - -.. data:: EL3RST - - Level 3 reset - - -.. data:: ELNRNG - - Link number out of range - - -.. data:: EUNATCH - - Protocol driver not attached - - -.. data:: ENOCSI - - No CSI structure available - - -.. data:: EL2HLT - - Level 2 halted - - -.. data:: EBADE - - Invalid exchange - - -.. data:: EBADR - - Invalid request descriptor - - -.. data:: EXFULL - - Exchange full - - -.. data:: ENOANO - - No anode - - -.. data:: EBADRQC - - Invalid request code - - -.. data:: EBADSLT - - Invalid slot - - -.. data:: EDEADLOCK - - File locking deadlock error - - -.. data:: EBFONT - - Bad font file format - - -.. data:: ENOSTR - - Device not a stream - - -.. data:: ENODATA - - No data available - - -.. data:: ETIME - - Timer expired - - -.. data:: ENOSR - - Out of streams resources - - -.. data:: ENONET - - Machine is not on the network - - -.. data:: ENOPKG - - Package not installed - - -.. data:: EREMOTE - - Object is remote - - -.. data:: ENOLINK - - Link has been severed - - -.. data:: EADV - - Advertise error - - -.. data:: ESRMNT - - Srmount error - - -.. data:: ECOMM - - Communication error on send - - -.. data:: EPROTO - - Protocol error - - -.. data:: EMULTIHOP - - Multihop attempted - - -.. data:: EDOTDOT - - RFS specific error - - -.. data:: EBADMSG - - Not a data message - - -.. data:: EOVERFLOW - - Value too large for defined data type - - -.. data:: ENOTUNIQ - - Name not unique on network - - -.. data:: EBADFD - - File descriptor in bad state - - -.. data:: EREMCHG - - Remote address changed - - -.. data:: ELIBACC - - Can not access a needed shared library - - -.. data:: ELIBBAD - - Accessing a corrupted shared library - - -.. data:: ELIBSCN - - .lib section in a.out corrupted - - -.. data:: ELIBMAX - - Attempting to link in too many shared libraries - - -.. data:: ELIBEXEC - - Cannot exec a shared library directly - - -.. data:: EILSEQ - - Illegal byte sequence - - -.. data:: ERESTART - - Interrupted system call should be restarted - - -.. data:: ESTRPIPE - - Streams pipe error - - -.. data:: EUSERS - - Too many users - - -.. data:: ENOTSOCK - - Socket operation on non-socket - - -.. data:: EDESTADDRREQ - - Destination address required - - -.. data:: EMSGSIZE - - Message too long - - -.. data:: EPROTOTYPE - - Protocol wrong type for socket - - -.. data:: ENOPROTOOPT - - Protocol not available - - -.. data:: EPROTONOSUPPORT - - Protocol not supported - - -.. data:: ESOCKTNOSUPPORT - - Socket type not supported - - -.. data:: EOPNOTSUPP - - Operation not supported on transport endpoint - - -.. data:: EPFNOSUPPORT - - Protocol family not supported - - -.. data:: EAFNOSUPPORT - - Address family not supported by protocol - - -.. data:: EADDRINUSE - - Address already in use - - -.. data:: EADDRNOTAVAIL - - Cannot assign requested address - - -.. data:: ENETDOWN - - Network is down - - -.. data:: ENETUNREACH - - Network is unreachable - - -.. data:: ENETRESET - - Network dropped connection because of reset - - -.. data:: ECONNABORTED - - Software caused connection abort - - -.. data:: ECONNRESET - - Connection reset by peer - - -.. data:: ENOBUFS - - No buffer space available - - -.. data:: EISCONN - - Transport endpoint is already connected - - -.. data:: ENOTCONN - - Transport endpoint is not connected - - -.. data:: ESHUTDOWN - - Cannot send after transport endpoint shutdown - - -.. data:: ETOOMANYREFS - - Too many references: cannot splice - - -.. data:: ETIMEDOUT - - Connection timed out - - -.. data:: ECONNREFUSED - - Connection refused - - -.. data:: EHOSTDOWN - - Host is down - - -.. data:: EHOSTUNREACH - - No route to host - - -.. data:: EALREADY - - Operation already in progress - - -.. data:: EINPROGRESS - - Operation now in progress - - -.. data:: ESTALE - - Stale NFS file handle - - -.. data:: EUCLEAN - - Structure needs cleaning - - -.. data:: ENOTNAM - - Not a XENIX named type file - - -.. data:: ENAVAIL - - No XENIX semaphores available - - -.. data:: EISNAM - - Is a named type file - - -.. data:: EREMOTEIO - - Remote I/O error - - -.. data:: EDQUOT - - Quota exceeded - diff --git a/third_party/python/Doc/library/exceptions.rst b/third_party/python/Doc/library/exceptions.rst deleted file mode 100644 index 5c0cac899..000000000 --- a/third_party/python/Doc/library/exceptions.rst +++ /dev/null @@ -1,737 +0,0 @@ -.. _bltin-exceptions: - -Built-in Exceptions -=================== - -.. index:: - statement: try - statement: except - -In Python, all exceptions must be instances of a class that derives from -:class:`BaseException`. In a :keyword:`try` statement with an :keyword:`except` -clause that mentions a particular class, that clause also handles any exception -classes derived from that class (but not exception classes from which *it* is -derived). Two exception classes that are not related via subclassing are never -equivalent, even if they have the same name. - -.. index:: statement: raise - -The built-in exceptions listed below can be generated by the interpreter or -built-in functions. Except where mentioned, they have an "associated value" -indicating the detailed cause of the error. This may be a string or a tuple of -several items of information (e.g., an error code and a string explaining the -code). The associated value is usually passed as arguments to the exception -class's constructor. - -User code can raise built-in exceptions. This can be used to test an exception -handler or to report an error condition "just like" the situation in which the -interpreter raises the same exception; but beware that there is nothing to -prevent user code from raising an inappropriate error. - -The built-in exception classes can be subclassed to define new exceptions; -programmers are encouraged to derive new exceptions from the :exc:`Exception` -class or one of its subclasses, and not from :exc:`BaseException`. More -information on defining exceptions is available in the Python Tutorial under -:ref:`tut-userexceptions`. - -When raising (or re-raising) an exception in an :keyword:`except` or -:keyword:`finally` clause -:attr:`__context__` is automatically set to the last exception caught; if the -new exception is not handled the traceback that is eventually displayed will -include the originating exception(s) and the final exception. - -When raising a new exception (rather than using a bare ``raise`` to re-raise -the exception currently being handled), the implicit exception context can be -supplemented with an explicit cause by using :keyword:`from` with -:keyword:`raise`:: - - raise new_exc from original_exc - -The expression following :keyword:`from` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. - -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. - -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. - - -Base classes ------------- - -The following exceptions are used mostly as base classes for other exceptions. - -.. exception:: BaseException - - The base class for all built-in exceptions. It is not meant to be directly - inherited by user-defined classes (for that, use :exc:`Exception`). If - :func:`str` is called on an instance of this class, the representation of - the argument(s) to the instance are returned, or the empty string when - there were no arguments. - - .. attribute:: args - - The tuple of arguments given to the exception constructor. Some built-in - exceptions (like :exc:`OSError`) expect a certain number of arguments and - assign a special meaning to the elements of this tuple, while others are - usually called only with a single string giving an error message. - - .. method:: with_traceback(tb) - - This method sets *tb* as the new traceback for the exception and returns - the exception object. It is usually used in exception handling code like - this:: - - try: - ... - except SomeException: - tb = sys.exc_info()[2] - raise OtherException(...).with_traceback(tb) - - -.. exception:: Exception - - All built-in, non-system-exiting exceptions are derived from this class. All - user-defined exceptions should also be derived from this class. - - -.. exception:: ArithmeticError - - The base class for those built-in exceptions that are raised for various - arithmetic errors: :exc:`OverflowError`, :exc:`ZeroDivisionError`, - :exc:`FloatingPointError`. - - -.. exception:: BufferError - - Raised when a :ref:`buffer ` related operation cannot be - performed. - - -.. exception:: LookupError - - The base class for the exceptions that are raised when a key or index used on - a mapping or sequence is invalid: :exc:`IndexError`, :exc:`KeyError`. This - can be raised directly by :func:`codecs.lookup`. - - -Concrete exceptions -------------------- - -The following exceptions are the exceptions that are usually raised. - -.. exception:: AssertionError - - .. index:: statement: assert - - Raised when an :keyword:`assert` statement fails. - - -.. exception:: AttributeError - - Raised when an attribute reference (see :ref:`attribute-references`) or - assignment fails. (When an object does not support attribute references or - attribute assignments at all, :exc:`TypeError` is raised.) - - -.. exception:: EOFError - - Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and - :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) - - -.. exception:: FloatingPointError - - Raised when a floating point operation fails. This exception is always defined, - but can only be raised when Python is configured with the - ``--with-fpectl`` option, or the :const:`WANT_SIGFPE_HANDLER` symbol is - defined in the :file:`pyconfig.h` file. - - -.. exception:: GeneratorExit - - Raised when a :term:`generator` or :term:`coroutine` is closed; - see :meth:`generator.close` and :meth:`coroutine.close`. It - directly inherits from :exc:`BaseException` instead of :exc:`Exception` since - it is technically not an error. - - -.. exception:: ImportError - - Raised when the :keyword:`import` statement has troubles trying to - load a module. Also raised when the "from list" in ``from ... import`` - has a name that cannot be found. - - The :attr:`name` and :attr:`path` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the module - that was attempted to be imported and the path to any file which triggered - the exception, respectively. - - .. versionchanged:: 3.3 - Added the :attr:`name` and :attr:`path` attributes. - -.. exception:: ModuleNotFoundError - - A subclass of :exc:`ImportError` which is raised by :keyword:`import` - when a module could not be located. It is also raised when ``None`` - is found in :data:`sys.modules`. - - .. versionadded:: 3.6 - - -.. exception:: IndexError - - Raised when a sequence subscript is out of range. (Slice indices are - silently truncated to fall in the allowed range; if an index is not an - integer, :exc:`TypeError` is raised.) - - .. XXX xref to sequences - - -.. exception:: KeyError - - Raised when a mapping (dictionary) key is not found in the set of existing keys. - - .. XXX xref to mapping objects? - - -.. exception:: KeyboardInterrupt - - Raised when the user hits the interrupt key (normally :kbd:`Control-C` or - :kbd:`Delete`). During execution, a check for interrupts is made - regularly. The exception inherits from :exc:`BaseException` so as to not be - accidentally caught by code that catches :exc:`Exception` and thus prevent - the interpreter from exiting. - - -.. exception:: MemoryError - - Raised when an operation runs out of memory but the situation may still be - rescued (by deleting some objects). The associated value is a string indicating - what kind of (internal) operation ran out of memory. Note that because of the - underlying memory management architecture (C's :c:func:`malloc` function), the - interpreter may not always be able to completely recover from this situation; it - nevertheless raises an exception so that a stack traceback can be printed, in - case a run-away program was the cause. - - -.. exception:: NameError - - Raised when a local or global name is not found. This applies only to - unqualified names. The associated value is an error message that includes the - name that could not be found. - - -.. exception:: NotImplementedError - - This exception is derived from :exc:`RuntimeError`. In user defined base - classes, abstract methods should raise this exception when they require - derived classes to override the method, or while the class is being - developed to indicate that the real implementation still needs to be added. - - .. note:: - - It should not be used to indicate that an operator or method is not - meant to be supported at all -- in that case either leave the operator / - method undefined or, if a subclass, set it to :data:`None`. - - .. note:: - - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, - even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. - -.. exception:: OSError([arg]) - OSError(errno, strerror[, filename[, winerror[, filename2]]]) - - .. index:: module: errno - - This exception is raised when a system function returns a system-related - error, including I/O failures such as "file not found" or "disk full" - (not for illegal argument types or other incidental errors). - - The second form of the constructor sets the corresponding attributes, - described below. The attributes default to :const:`None` if not - specified. For backwards compatibility, if three arguments are passed, - the :attr:`~BaseException.args` attribute contains only a 2-tuple - of the first two constructor arguments. - - The constructor often actually returns a subclass of :exc:`OSError`, as - described in `OS exceptions`_ below. The particular subclass depends on - the final :attr:`.errno` value. This behaviour only occurs when - constructing :exc:`OSError` directly or via an alias, and is not - inherited when subclassing. - - .. attribute:: errno - - A numeric error code from the C variable :c:data:`errno`. - - .. attribute:: winerror - - Under Windows, this gives you the native - Windows error code. The :attr:`.errno` attribute is then an approximate - translation, in POSIX terms, of that native error code. - - Under Windows, if the *winerror* constructor argument is an integer, - the :attr:`.errno` attribute is determined from the Windows error code, - and the *errno* argument is ignored. On other platforms, the - *winerror* argument is ignored, and the :attr:`winerror` attribute - does not exist. - - .. attribute:: strerror - - The corresponding error message, as provided by - the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` - under Windows. - - .. attribute:: filename - filename2 - - For exceptions that involve a file system path (such as :func:`open` or - :func:`os.unlink`), :attr:`filename` is the file name passed to the function. - For functions that involve two file system paths (such as - :func:`os.rename`), :attr:`filename2` corresponds to the second - file name passed to the function. - - - .. versionchanged:: 3.3 - :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, - :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the - constructor may return a subclass. - - .. versionchanged:: 3.4 - The :attr:`filename` attribute is now the original file name passed to - the function, instead of the name encoded to or decoded from the - filesystem encoding. Also, the *filename2* constructor argument and - attribute was added. - - -.. exception:: OverflowError - - Raised when the result of an arithmetic operation is too large to be - represented. This cannot occur for integers (which would rather raise - :exc:`MemoryError` than give up). However, for historical reasons, - OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. - - -.. exception:: RecursionError - - This exception is derived from :exc:`RuntimeError`. It is raised when the - interpreter detects that the maximum recursion depth (see - :func:`sys.getrecursionlimit`) is exceeded. - - .. versionadded:: 3.5 - Previously, a plain :exc:`RuntimeError` was raised. - - -.. exception:: ReferenceError - - This exception is raised when a weak reference proxy, created by the - :func:`weakref.proxy` function, is used to access an attribute of the referent - after it has been garbage collected. For more information on weak references, - see the :mod:`weakref` module. - - -.. exception:: RuntimeError - - Raised when an error is detected that doesn't fall in any of the other - categories. The associated value is a string indicating what precisely went - wrong. - - -.. exception:: StopIteration - - Raised by built-in function :func:`next` and an :term:`iterator`\'s - :meth:`~iterator.__next__` method to signal that there are no further - items produced by the iterator. - - The exception object has a single attribute :attr:`value`, which is - given as an argument when constructing the exception, and defaults - to :const:`None`. - - When a :term:`generator` or :term:`coroutine` function - returns, a new :exc:`StopIteration` instance is - raised, and the value returned by the function is used as the - :attr:`value` parameter to the constructor of the exception. - - If a generator function defined in the presence of a ``from __future__ - import generator_stop`` directive raises :exc:`StopIteration`, it will be - converted into a :exc:`RuntimeError` (retaining the :exc:`StopIteration` - as the new exception's cause). - - .. versionchanged:: 3.3 - Added ``value`` attribute and the ability for generator functions to - use it to return a value. - - .. versionchanged:: 3.5 - Introduced the RuntimeError transformation. - -.. exception:: StopAsyncIteration - - Must be raised by :meth:`__anext__` method of an - :term:`asynchronous iterator` object to stop the iteration. - - .. versionadded:: 3.5 - -.. exception:: SyntaxError - - Raised when the parser encounters a syntax error. This may occur in an - :keyword:`import` statement, in a call to the built-in functions :func:`exec` - or :func:`eval`, or when reading the initial script or standard input - (also interactively). - - Instances of this class have attributes :attr:`filename`, :attr:`lineno`, - :attr:`offset` and :attr:`text` for easier access to the details. :func:`str` - of the exception instance returns only the message. - - -.. exception:: IndentationError - - Base class for syntax errors related to incorrect indentation. This is a - subclass of :exc:`SyntaxError`. - - -.. exception:: TabError - - Raised when indentation contains an inconsistent use of tabs and spaces. - This is a subclass of :exc:`IndentationError`. - - -.. exception:: SystemError - - Raised when the interpreter finds an internal error, but the situation does not - look so serious to cause it to abandon all hope. The associated value is a - string indicating what went wrong (in low-level terms). - - You should report this to the author or maintainer of your Python interpreter. - Be sure to report the version of the Python interpreter (``sys.version``; it is - also printed at the start of an interactive Python session), the exact error - message (the exception's associated value) and if possible the source of the - program that triggered the error. - - -.. exception:: SystemExit - - This exception is raised by the :func:`sys.exit` function. It inherits from - :exc:`BaseException` instead of :exc:`Exception` so that it is not accidentally - caught by code that catches :exc:`Exception`. This allows the exception to - properly propagate up and cause the interpreter to exit. When it is not - handled, the Python interpreter exits; no stack traceback is printed. The - constructor accepts the same optional argument passed to :func:`sys.exit`. - If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if - it has another type (such as a string), the object's value is printed and - the exit status is one. - - A call to :func:`sys.exit` is translated into an exception so that clean-up - handlers (:keyword:`finally` clauses of :keyword:`try` statements) can be - executed, and so that a debugger can execute a script without running the risk - of losing control. The :func:`os._exit` function can be used if it is - absolutely positively necessary to exit immediately (for example, in the child - process after a call to :func:`os.fork`). - - .. attribute:: code - - The exit status or error message that is passed to the constructor. - (Defaults to ``None``.) - - -.. exception:: TypeError - - Raised when an operation or function is applied to an object of inappropriate - type. The associated value is a string giving details about the type mismatch. - - This exception may be raised by user code to indicate that an attempted - operation on an object is not supported, and is not meant to be. If an object - is meant to support a given operation but has not yet provided an - implementation, :exc:`NotImplementedError` is the proper exception to raise. - - Passing arguments of the wrong type (e.g. passing a :class:`list` when an - :class:`int` is expected) should result in a :exc:`TypeError`, but passing - arguments with the wrong value (e.g. a number outside expected boundaries) - should result in a :exc:`ValueError`. - -.. exception:: UnboundLocalError - - Raised when a reference is made to a local variable in a function or method, but - no value has been bound to that variable. This is a subclass of - :exc:`NameError`. - - -.. exception:: UnicodeError - - Raised when a Unicode-related encoding or decoding error occurs. It is a - subclass of :exc:`ValueError`. - - :exc:`UnicodeError` has attributes that describe the encoding or decoding - error. For example, ``err.object[err.start:err.end]`` gives the particular - invalid input that the codec failed on. - - .. attribute:: encoding - - The name of the encoding that raised the error. - - .. attribute:: reason - - A string describing the specific codec error. - - .. attribute:: object - - The object the codec was attempting to encode or decode. - - .. attribute:: start - - The first index of invalid data in :attr:`object`. - - .. attribute:: end - - The index after the last invalid data in :attr:`object`. - - -.. exception:: UnicodeEncodeError - - Raised when a Unicode-related error occurs during encoding. It is a subclass of - :exc:`UnicodeError`. - - -.. exception:: UnicodeDecodeError - - Raised when a Unicode-related error occurs during decoding. It is a subclass of - :exc:`UnicodeError`. - - -.. exception:: UnicodeTranslateError - - Raised when a Unicode-related error occurs during translating. It is a subclass - of :exc:`UnicodeError`. - - -.. exception:: ValueError - - Raised when an operation or function receives an argument that has the - right type but an inappropriate value, and the situation is not described by a - more precise exception such as :exc:`IndexError`. - - -.. exception:: ZeroDivisionError - - Raised when the second argument of a division or modulo operation is zero. The - associated value is a string indicating the type of the operands and the - operation. - - -The following exceptions are kept for compatibility with previous versions; -starting from Python 3.3, they are aliases of :exc:`OSError`. - -.. exception:: EnvironmentError - -.. exception:: IOError - -.. exception:: WindowsError - - Only available on Windows. - - -OS exceptions -^^^^^^^^^^^^^ - -The following exceptions are subclasses of :exc:`OSError`, they get raised -depending on the system error code. - -.. exception:: BlockingIOError - - Raised when an operation would block on an object (e.g. socket) set - for non-blocking operation. - Corresponds to :c:data:`errno` ``EAGAIN``, ``EALREADY``, - ``EWOULDBLOCK`` and ``EINPROGRESS``. - - In addition to those of :exc:`OSError`, :exc:`BlockingIOError` can have - one more attribute: - - .. attribute:: characters_written - - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the - buffered I/O classes from the :mod:`io` module. - -.. exception:: ChildProcessError - - Raised when an operation on a child process failed. - Corresponds to :c:data:`errno` ``ECHILD``. - -.. exception:: ConnectionError - - A base class for connection-related issues. - - Subclasses are :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`, - :exc:`ConnectionRefusedError` and :exc:`ConnectionResetError`. - -.. exception:: BrokenPipeError - - A subclass of :exc:`ConnectionError`, raised when trying to write on a - pipe while the other end has been closed, or trying to write on a socket - which has been shutdown for writing. - Corresponds to :c:data:`errno` ``EPIPE`` and ``ESHUTDOWN``. - -.. exception:: ConnectionAbortedError - - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is aborted by the peer. - Corresponds to :c:data:`errno` ``ECONNABORTED``. - -.. exception:: ConnectionRefusedError - - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is refused by the peer. - Corresponds to :c:data:`errno` ``ECONNREFUSED``. - -.. exception:: ConnectionResetError - - A subclass of :exc:`ConnectionError`, raised when a connection is - reset by the peer. - Corresponds to :c:data:`errno` ``ECONNRESET``. - -.. exception:: FileExistsError - - Raised when trying to create a file or directory which already exists. - Corresponds to :c:data:`errno` ``EEXIST``. - -.. exception:: FileNotFoundError - - Raised when a file or directory is requested but doesn't exist. - Corresponds to :c:data:`errno` ``ENOENT``. - -.. exception:: InterruptedError - - Raised when a system call is interrupted by an incoming signal. - Corresponds to :c:data:`errno` :py:data:`~errno.EINTR`. - - .. versionchanged:: 3.5 - Python now retries system calls when a syscall is interrupted by a - signal, except if the signal handler raises an exception (see :pep:`475` - for the rationale), instead of raising :exc:`InterruptedError`. - -.. exception:: IsADirectoryError - - Raised when a file operation (such as :func:`os.remove`) is requested - on a directory. - Corresponds to :c:data:`errno` ``EISDIR``. - -.. exception:: NotADirectoryError - - Raised when a directory operation (such as :func:`os.listdir`) is requested - on something which is not a directory. - Corresponds to :c:data:`errno` ``ENOTDIR``. - -.. exception:: PermissionError - - Raised when trying to run an operation without the adequate access - rights - for example filesystem permissions. - Corresponds to :c:data:`errno` ``EACCES`` and ``EPERM``. - -.. exception:: ProcessLookupError - - Raised when a given process doesn't exist. - Corresponds to :c:data:`errno` ``ESRCH``. - -.. exception:: TimeoutError - - Raised when a system function timed out at the system level. - Corresponds to :c:data:`errno` ``ETIMEDOUT``. - -.. versionadded:: 3.3 - All the above :exc:`OSError` subclasses were added. - - -.. seealso:: - - :pep:`3151` - Reworking the OS and IO exception hierarchy - - -Warnings --------- - -The following exceptions are used as warning categories; see the :mod:`warnings` -module for more information. - -.. exception:: Warning - - Base class for warning categories. - - -.. exception:: UserWarning - - Base class for warnings generated by user code. - - -.. exception:: DeprecationWarning - - Base class for warnings about deprecated features. - - -.. exception:: PendingDeprecationWarning - - Base class for warnings about features which will be deprecated in the future. - - -.. exception:: SyntaxWarning - - Base class for warnings about dubious syntax. - - -.. exception:: RuntimeWarning - - Base class for warnings about dubious runtime behavior. - - -.. exception:: FutureWarning - - Base class for warnings about constructs that will change semantically in the - future. - - -.. exception:: ImportWarning - - Base class for warnings about probable mistakes in module imports. - - -.. exception:: UnicodeWarning - - Base class for warnings related to Unicode. - - -.. exception:: BytesWarning - - Base class for warnings related to :class:`bytes` and :class:`bytearray`. - - -.. exception:: ResourceWarning - - Base class for warnings related to resource usage. - - .. versionadded:: 3.2 - - - -Exception hierarchy -------------------- - -The class hierarchy for built-in exceptions is: - -.. literalinclude:: ../../Lib/test/exception_hierarchy.txt diff --git a/third_party/python/Doc/library/faulthandler.rst b/third_party/python/Doc/library/faulthandler.rst deleted file mode 100644 index 94ebd8763..000000000 --- a/third_party/python/Doc/library/faulthandler.rst +++ /dev/null @@ -1,172 +0,0 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= - -.. module:: faulthandler - :synopsis: Dump the Python traceback. - -.. versionadded:: 3.3 - ----------------- - -This module contains functions to dump Python tracebacks explicitly, on a fault, -after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also -enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment -variable or by using the :option:`-X` ``faulthandler`` command line option. - -The fault handler is compatible with system fault handlers like Apport or the -Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the -traceback even on a stack overflow. - -The fault handler is called on catastrophic cases and therefore can only use -signal-safe functions (e.g. it cannot allocate memory on the heap). Because of -this limitation traceback dumping is minimal compared to normal Python -tracebacks: - -* Only ASCII is supported. The ``backslashreplace`` error handler is used on - encoding. -* Each string is limited to 500 characters. -* Only the filename, the function name and the line number are - displayed. (no source code) -* It is limited to 100 frames and 100 threads. -* The order is reversed: the most recent call is shown first. - -By default, the Python traceback is written to :data:`sys.stderr`. To see -tracebacks, applications must be run in the terminal. A log file can -alternatively be passed to :func:`faulthandler.enable`. - -The module is implemented in C, so tracebacks can be dumped on a crash or when -Python is deadlocked. - - -Dumping the traceback ---------------------- - -.. function:: dump_traceback(file=sys.stderr, all_threads=True) - - Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - - -Fault handler state -------------------- - -.. function:: enable(file=sys.stderr, all_threads=True) - - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` - signals to dump the Python traceback. If *all_threads* is ``True``, - produce tracebacks for every running thread. Otherwise, dump only the current - thread. - - The *file* must be kept open until the fault handler is disabled: see - :ref:`issue with file descriptors `. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - - .. versionchanged:: 3.6 - On Windows, a handler for Windows exception is also installed. - -.. function:: disable() - - Disable the fault handler: uninstall the signal handlers installed by - :func:`enable`. - -.. function:: is_enabled() - - Check if the fault handler is enabled. - - -Dumping the tracebacks after a timeout --------------------------------------- - -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) - - Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or - every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any - cleanup like flushing file buffers.) If the function is called twice, the new - call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. - - The *file* must be kept open until the traceback is dumped or - :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file - descriptors `. - - This function is implemented using a watchdog thread and therefore is not - available if Python is compiled with threads disabled. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - -.. function:: cancel_dump_traceback_later() - - Cancel the last call to :func:`dump_traceback_later`. - - -Dumping the traceback on a user signal --------------------------------------- - -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) - - Register a user signal: install a handler for the *signum* signal to dump - the traceback of all threads, or of the current thread if *all_threads* is - ``False``, into *file*. Call the previous handler if chain is ``True``. - - The *file* must be kept open until the signal is unregistered by - :func:`unregister`: see :ref:`issue with file descriptors `. - - Not available on Windows. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - -.. function:: unregister(signum) - - Unregister a user signal: uninstall the handler of the *signum* signal - installed by :func:`register`. Return ``True`` if the signal was registered, - ``False`` otherwise. - - Not available on Windows. - - -.. _faulthandler-fd: - -Issue with file descriptors ---------------------------- - -:func:`enable`, :func:`dump_traceback_later` and :func:`register` keep the -file descriptor of their *file* argument. If the file is closed and its file -descriptor is reused by a new file, or if :func:`os.dup2` is used to replace -the file descriptor, the traceback will be written into a different file. Call -these functions again each time that the file is replaced. - - -Example -------- - -Example of a segmentation fault on Linux with and without enabling the fault -handler: - -.. code-block:: shell-session - - $ python3 -c "import ctypes; ctypes.string_at(0)" - Segmentation fault - - $ python3 -q -X faulthandler - >>> import ctypes - >>> ctypes.string_at(0) - Fatal Python error: Segmentation fault - - Current thread 0x00007fb899f39700 (most recent call first): - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at - File "", line 1 in - Segmentation fault - diff --git a/third_party/python/Doc/library/fcntl.rst b/third_party/python/Doc/library/fcntl.rst deleted file mode 100644 index 88112f6b7..000000000 --- a/third_party/python/Doc/library/fcntl.rst +++ /dev/null @@ -1,170 +0,0 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= - -.. module:: fcntl - :platform: Unix - :synopsis: The fcntl() and ioctl() system calls. - -.. sectionauthor:: Jaap Vermeulen - -.. index:: - pair: UNIX; file control - pair: UNIX; I/O control - ----------------- - -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. - -All functions in this module take a file descriptor *fd* as their first -argument. This can be an integer file descriptor, such as returned by -``sys.stdin.fileno()``, or an :class:`io.IOBase` object, such as ``sys.stdin`` -itself, which provides a :meth:`~io.IOBase.fileno` that returns a genuine file -descriptor. - -.. versionchanged:: 3.3 - Operations in this module used to raise an :exc:`IOError` where they now - raise an :exc:`OSError`. - - -The module defines the following functions: - - -.. function:: fcntl(fd, cmd, arg=0) - - Perform the operation *cmd* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). The values used - for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C - header files. The argument *arg* can either be an integer value, or a - :class:`bytes` object. With an integer value, the return value of this - function is the integer return value of the C :c:func:`fcntl` call. When - the argument is bytes it represents a binary structure, e.g. created by - :func:`struct.pack`. The binary data is copied to a buffer whose address is - passed to the C :c:func:`fcntl` call. The return value after a successful - call is the contents of the buffer, converted to a :class:`bytes` object. - The length of the returned object will be the same as the length of the - *arg* argument. This is limited to 1024 bytes. If the information returned - in the buffer by the operating system is larger than 1024 bytes, this is - most likely to result in a segmentation violation or a more subtle data - corruption. - - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. - - -.. function:: ioctl(fd, request, arg=0, mutate_flag=True) - - This function is identical to the :func:`~fcntl.fcntl` function, except - that the argument handling is even more complicated. - - The *request* parameter is limited to values that can fit in 32-bits. - Additional constants of interest for use as the *request* argument can be - found in the :mod:`termios` module, under the same names as used in - the relevant C header files. - - The parameter *arg* can be one of an integer, an object supporting the - read-only buffer interface (like :class:`bytes`) or an object supporting - the read-write buffer interface (like :class:`bytearray`). - - In all but the last case, behaviour is as for the :func:`~fcntl.fcntl` - function. - - If a mutable buffer is passed, then the behaviour is determined by the value of - the *mutate_flag* parameter. - - If it is false, the buffer's mutability is ignored and behaviour is as for a - read-only buffer, except that the 1024 byte limit mentioned above is avoided -- - so long as the buffer you pass is at least as long as what the operating system - wants to put there, things should work. - - If *mutate_flag* is true (the default), then the buffer is (in effect) passed - to the underlying :func:`ioctl` system call, the latter's return code is - passed back to the calling Python, and the buffer's new contents reflect the - action of the :func:`ioctl`. This is a slight simplification, because if the - supplied buffer is less than 1024 bytes long it is first copied into a static - buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back - into the supplied buffer. - - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. - - An example:: - - >>> import array, fcntl, struct, termios, os - >>> os.getpgrp() - 13341 - >>> struct.unpack('h', fcntl.ioctl(0, termios.TIOCGPGRP, " "))[0] - 13341 - >>> buf = array.array('h', [0]) - >>> fcntl.ioctl(0, termios.TIOCGPGRP, buf, 1) - 0 - >>> buf - array('h', [13341]) - - -.. function:: flock(fd, operation) - - Perform the lock operation *operation* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual - :manpage:`flock(2)` for details. (On some systems, this function is emulated - using :c:func:`fcntl`.) - - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. - - -.. function:: lockf(fd, cmd, len=0, start=0, whence=0) - - This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. - *fd* is the file descriptor of the file to lock or unlock, and *cmd* - is one of the following values: - - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock - - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an - :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the - operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a - file opened for writing. - - *len* is the number of bytes to lock, *start* is the byte offset at - which the lock starts, relative to *whence*, and *whence* is as with - :func:`io.IOBase.seek`, specifically: - - * :const:`0` -- relative to the start of the file (:data:`os.SEEK_SET`) - * :const:`1` -- relative to the current buffer position (:data:`os.SEEK_CUR`) - * :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`) - - The default for *start* is 0, which means to start at the beginning of the file. - The default for *len* is 0 which means to lock to the end of the file. The - default for *whence* is also 0. - -Examples (all on a SVR4 compliant system):: - - import struct, fcntl, os - - f = open(...) - rv = fcntl.fcntl(f, fcntl.F_SETFL, os.O_NDELAY) - - lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) - rv = fcntl.fcntl(f, fcntl.F_SETLKW, lockdata) - -Note that in the first example the return value variable *rv* will hold an -integer value; in the second example it will hold a :class:`bytes` object. The -structure lay-out for the *lockdata* variable is system dependent --- therefore -using the :func:`flock` call may be better. - - -.. seealso:: - - Module :mod:`os` - If the locking flags :data:`~os.O_SHLOCK` and :data:`~os.O_EXLOCK` are - present in the :mod:`os` module (on BSD only), the :func:`os.open` - function provides an alternative to the :func:`lockf` and :func:`flock` - functions. - diff --git a/third_party/python/Doc/library/filecmp.rst b/third_party/python/Doc/library/filecmp.rst deleted file mode 100644 index 31b9b4afa..000000000 --- a/third_party/python/Doc/library/filecmp.rst +++ /dev/null @@ -1,198 +0,0 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= - -.. module:: filecmp - :synopsis: Compare files efficiently. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/filecmp.py` - --------------- - -The :mod:`filecmp` module defines functions to compare files and directories, -with various optional time/correctness trade-offs. For comparing files, -see also the :mod:`difflib` module. - -The :mod:`filecmp` module defines the following functions: - - -.. function:: cmp(f1, f2, shallow=True) - - Compare the files named *f1* and *f2*, returning ``True`` if they seem equal, - ``False`` otherwise. - - If *shallow* is true, files with identical :func:`os.stat` signatures are - taken to be equal. Otherwise, the contents of the files are compared. - - Note that no external programs are called from this function, giving it - portability and efficiency. - - This function uses a cache for past comparisons and the results, - with cache entries invalidated if the :func:`os.stat` information for the - file changes. The entire cache may be cleared using :func:`clear_cache`. - - -.. function:: cmpfiles(dir1, dir2, common, shallow=True) - - Compare the files in the two directories *dir1* and *dir2* whose names are - given by *common*. - - Returns three lists of file names: *match*, *mismatch*, - *errors*. *match* contains the list of files that match, *mismatch* contains - the names of those that don't, and *errors* lists the names of files which - could not be compared. Files are listed in *errors* if they don't exist in - one of the directories, the user lacks permission to read them or if the - comparison could not be done for some other reason. - - The *shallow* parameter has the same meaning and default value as for - :func:`filecmp.cmp`. - - For example, ``cmpfiles('a', 'b', ['c', 'd/e'])`` will compare ``a/c`` with - ``b/c`` and ``a/d/e`` with ``b/d/e``. ``'c'`` and ``'d/e'`` will each be in - one of the three returned lists. - - -.. function:: clear_cache() - - Clear the filecmp cache. This may be useful if a file is compared so quickly - after it is modified that it is within the mtime resolution of - the underlying filesystem. - - .. versionadded:: 3.4 - - -.. _dircmp-objects: - -The :class:`dircmp` class -------------------------- - -.. class:: dircmp(a, b, ignore=None, hide=None) - - Construct a new directory comparison object, to compare the directories *a* - and *b*. *ignore* is a list of names to ignore, and defaults to - :attr:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and - defaults to ``[os.curdir, os.pardir]``. - - The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. - - The :class:`dircmp` class provides the following methods: - - .. method:: report() - - Print (to :data:`sys.stdout`) a comparison between *a* and *b*. - - .. method:: report_partial_closure() - - Print a comparison between *a* and *b* and common immediate - subdirectories. - - .. method:: report_full_closure() - - Print a comparison between *a* and *b* and common subdirectories - (recursively). - - The :class:`dircmp` class offers a number of interesting attributes that may be - used to get various bits of information about the directory trees being - compared. - - Note that via :meth:`__getattr__` hooks, all attributes are computed lazily, - so there is no speed penalty if only those attributes which are lightweight - to compute are used. - - - .. attribute:: left - - The directory *a*. - - - .. attribute:: right - - The directory *b*. - - - .. attribute:: left_list - - Files and subdirectories in *a*, filtered by *hide* and *ignore*. - - - .. attribute:: right_list - - Files and subdirectories in *b*, filtered by *hide* and *ignore*. - - - .. attribute:: common - - Files and subdirectories in both *a* and *b*. - - - .. attribute:: left_only - - Files and subdirectories only in *a*. - - - .. attribute:: right_only - - Files and subdirectories only in *b*. - - - .. attribute:: common_dirs - - Subdirectories in both *a* and *b*. - - - .. attribute:: common_files - - Files in both *a* and *b*. - - - .. attribute:: common_funny - - Names in both *a* and *b*, such that the type differs between the - directories, or names for which :func:`os.stat` reports an error. - - - .. attribute:: same_files - - Files which are identical in both *a* and *b*, using the class's - file comparison operator. - - - .. attribute:: diff_files - - Files which are in both *a* and *b*, whose contents differ according - to the class's file comparison operator. - - - .. attribute:: funny_files - - Files which are in both *a* and *b*, but could not be compared. - - - .. attribute:: subdirs - - A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp` - objects. - -.. attribute:: DEFAULT_IGNORES - - .. versionadded:: 3.4 - - List of directories ignored by :class:`dircmp` by default. - - -Here is a simplified example of using the ``subdirs`` attribute to search -recursively through two directories to show common different files:: - - >>> from filecmp import dircmp - >>> def print_diff_files(dcmp): - ... for name in dcmp.diff_files: - ... print("diff_file %s found in %s and %s" % (name, dcmp.left, - ... dcmp.right)) - ... for sub_dcmp in dcmp.subdirs.values(): - ... print_diff_files(sub_dcmp) - ... - >>> dcmp = dircmp('dir1', 'dir2') # doctest: +SKIP - >>> print_diff_files(dcmp) # doctest: +SKIP - diff --git a/third_party/python/Doc/library/fileformats.rst b/third_party/python/Doc/library/fileformats.rst deleted file mode 100644 index e9c2e1fbb..000000000 --- a/third_party/python/Doc/library/fileformats.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _fileformats: - -************ -File Formats -************ - -The modules described in this chapter parse various miscellaneous file formats -that aren't markup languages and are not related to e-mail. - - -.. toctree:: - - csv.rst - configparser.rst - netrc.rst - xdrlib.rst - plistlib.rst diff --git a/third_party/python/Doc/library/fileinput.rst b/third_party/python/Doc/library/fileinput.rst deleted file mode 100644 index 5881fefe2..000000000 --- a/third_party/python/Doc/library/fileinput.rst +++ /dev/null @@ -1,207 +0,0 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== - -.. module:: fileinput - :synopsis: Loop over standard input or a list of files. - -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/fileinput.py` - --------------- - -This module implements a helper class and functions to quickly write a -loop over standard input or a list of files. If you just want to read or -write one file see :func:`open`. - -The typical use is:: - - import fileinput - for line in fileinput.input(): - process(line) - -This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting -to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin``. To specify an alternative list of filenames, pass it -as the first argument to :func:`.input`. A single file name is also allowed. - -All files are opened in text mode by default, but you can override this by -specifying the *mode* parameter in the call to :func:`.input` or -:class:`FileInput`. If an I/O error occurs during opening or reading a file, -:exc:`OSError` is raised. - -.. versionchanged:: 3.3 - :exc:`IOError` used to be raised; it is now an alias of :exc:`OSError`. - -If ``sys.stdin`` is used more than once, the second and further use will return -no lines, except perhaps for interactive use, or if it has been explicitly reset -(e.g. using ``sys.stdin.seek(0)``). - -Empty files are opened and immediately closed; the only time their presence in -the list of filenames is noticeable at all is when the last file opened is -empty. - -Lines are returned with any newlines intact, which means that the last line in -a file may not have one. - -You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The -hook must be a function that takes two arguments, *filename* and *mode*, and -returns an accordingly opened file-like object. Two useful hooks are already -provided by this module. - -The following function is the primary interface of this module: - - -.. function:: input(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) - - Create an instance of the :class:`FileInput` class. The instance will be used - as global state for the functions of this module, and is also returned to use - during iteration. The parameters to this function will be passed along to the - constructor of the :class:`FileInput` class. - - The :class:`FileInput` instance can be used as a context manager in the - :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`with` statement is exited, even if an exception occurs:: - - with fileinput.input(files=('spam.txt', 'eggs.txt')) as f: - for line in f: - process(line) - - .. versionchanged:: 3.2 - Can be used as a context manager. - - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. - -The following functions use the global state created by :func:`fileinput.input`; -if there is no active state, :exc:`RuntimeError` is raised. - - -.. function:: filename() - - Return the name of the file currently being read. Before the first line has - been read, returns ``None``. - - -.. function:: fileno() - - Return the integer "file descriptor" for the current file. When no file is - opened (before the first line and between files), returns ``-1``. - - -.. function:: lineno() - - Return the cumulative line number of the line that has just been read. Before - the first line has been read, returns ``0``. After the last line of the last - file has been read, returns the line number of that line. - - -.. function:: filelineno() - - Return the line number in the current file. Before the first line has been - read, returns ``0``. After the last line of the last file has been read, - returns the line number of that line within the file. - - -.. function:: isfirstline() - - Returns true if the line just read is the first line of its file, otherwise - returns false. - - -.. function:: isstdin() - - Returns true if the last line was read from ``sys.stdin``, otherwise returns - false. - - -.. function:: nextfile() - - Close the current file so that the next iteration will read the first line from - the next file (if any); lines not read from the file will not count towards the - cumulative line count. The filename is not changed until after the first line - of the next file has been read. Before the first line has been read, this - function has no effect; it cannot be used to skip the first file. After the - last line of the last file has been read, this function has no effect. - - -.. function:: close() - - Close the sequence. - -The class which implements the sequence behavior provided by the module is -available for subclassing as well: - - -.. class:: FileInput(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None) - - Class :class:`FileInput` is the implementation; its methods :meth:`filename`, - :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, - :meth:`isstdin`, :meth:`nextfile` and :meth:`close` correspond to the - functions of the same name in the module. In addition it has a - :meth:`~io.TextIOBase.readline` method which returns the next input line, - and a :meth:`__getitem__` method which implements the sequence behavior. - The sequence must be accessed in strictly sequential order; random access - and :meth:`~io.TextIOBase.readline` cannot be mixed. - - With *mode* you can specify which file mode will be passed to :func:`open`. It - must be one of ``'r'``, ``'rU'``, ``'U'`` and ``'rb'``. - - The *openhook*, when given, must be a function that takes two arguments, - *filename* and *mode*, and returns an accordingly opened file-like object. You - cannot use *inplace* and *openhook* together. - - A :class:`FileInput` instance can be used as a context manager in the - :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`with` statement is exited, even if an exception occurs:: - - with FileInput(files=('spam.txt', 'eggs.txt')) as input: - process(input) - - .. versionchanged:: 3.2 - Can be used as a context manager. - - .. deprecated:: 3.4 - The ``'rU'`` and ``'U'`` modes. - - .. deprecated-removed:: 3.6 3.8 - The *bufsize* parameter. - - -**Optional in-place filtering:** if the keyword argument ``inplace=True`` is -passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the -file is moved to a backup file and standard output is directed to the input file -(if a file of the same name as the backup file already exists, it will be -replaced silently). This makes it possible to write a filter that rewrites its -input file in place. If the *backup* parameter is given (typically as -``backup='.'``), it specifies the extension for the backup file, -and the backup file remains around; by default, the extension is ``'.bak'`` and -it is deleted when the output file is closed. In-place filtering is disabled -when standard input is read. - - -The two following opening hooks are provided by this module: - -.. function:: hook_compressed(filename, mode) - - Transparently opens files compressed with gzip and bzip2 (recognized by the - extensions ``'.gz'`` and ``'.bz2'``) using the :mod:`gzip` and :mod:`bz2` - modules. If the filename extension is not ``'.gz'`` or ``'.bz2'``, the file is - opened normally (ie, using :func:`open` without any decompression). - - Usage example: ``fi = fileinput.FileInput(openhook=fileinput.hook_compressed)`` - - -.. function:: hook_encoded(encoding, errors=None) - - Returns a hook which opens each file with :func:`open`, using the given - *encoding* and *errors* to read the file. - - Usage example: ``fi = - fileinput.FileInput(openhook=fileinput.hook_encoded("utf-8", - "surrogateescape"))`` - - .. versionchanged:: 3.6 - Added the optional *errors* parameter. diff --git a/third_party/python/Doc/library/filesys.rst b/third_party/python/Doc/library/filesys.rst deleted file mode 100644 index 03e085b45..000000000 --- a/third_party/python/Doc/library/filesys.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _filesys: - -************************* -File and Directory Access -************************* - -The modules described in this chapter deal with disk files and directories. For -example, there are modules for reading the properties of files, manipulating -paths in a portable way, and creating temporary files. The full list of modules -in this chapter is: - - -.. toctree:: - - pathlib.rst - os.path.rst - fileinput.rst - stat.rst - filecmp.rst - tempfile.rst - glob.rst - fnmatch.rst - linecache.rst - shutil.rst - macpath.rst - - -.. seealso:: - - Module :mod:`os` - Operating system interfaces, including functions to work with files at a - lower level than Python :term:`file objects `. - - Module :mod:`io` - Python's built-in I/O library, including both abstract classes and - some concrete classes such as file I/O. - - Built-in function :func:`open` - The standard way to open files for reading and writing with Python. diff --git a/third_party/python/Doc/library/fnmatch.rst b/third_party/python/Doc/library/fnmatch.rst deleted file mode 100644 index fb0a1e332..000000000 --- a/third_party/python/Doc/library/fnmatch.rst +++ /dev/null @@ -1,102 +0,0 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= - -.. module:: fnmatch - :synopsis: Unix shell style filename pattern matching. - -**Source code:** :source:`Lib/fnmatch.py` - -.. index:: single: filenames; wildcard expansion - -.. index:: module: re - --------------- - -This module provides support for Unix shell-style wildcards, which are *not* the -same as regular expressions (which are documented in the :mod:`re` module). The -special characters used in shell-style wildcards are: - -.. index:: - single: * (asterisk); in glob-style wildcards - single: ? (question mark); in glob-style wildcards - single: [] (square brackets); in glob-style wildcards - single: ! (exclamation); in glob-style wildcards - single: - (minus); in glob-style wildcards - -+------------+------------------------------------+ -| Pattern | Meaning | -+============+====================================+ -| ``*`` | matches everything | -+------------+------------------------------------+ -| ``?`` | matches any single character | -+------------+------------------------------------+ -| ``[seq]`` | matches any character in *seq* | -+------------+------------------------------------+ -| ``[!seq]`` | matches any character not in *seq* | -+------------+------------------------------------+ - -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - -.. index:: module: glob - -Note that the filename separator (``'/'`` on Unix) is *not* special to this -module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses -:func:`.filter` to match pathname segments). Similarly, filenames starting with -a period are not special for this module, and are matched by the ``*`` and ``?`` -patterns. - - -.. function:: fnmatch(filename, pattern) - - Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. Both parameters are case-normalized - using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a - case-sensitive comparison, regardless of whether that's standard for the - operating system. - - This example will print all file names in the current directory with the - extension ``.txt``:: - - import fnmatch - import os - - for file in os.listdir('.'): - if fnmatch.fnmatch(file, '*.txt'): - print(file) - - -.. function:: fnmatchcase(filename, pattern) - - Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive and does not apply - :func:`os.path.normcase`. - - -.. function:: filter(names, pattern) - - Return the subset of the list of *names* that match *pattern*. It is the same as - ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. - - -.. function:: translate(pattern) - - Return the shell-style *pattern* converted to a regular expression for - using with :func:`re.match`. - - Example: - - >>> import fnmatch, re - >>> - >>> regex = fnmatch.translate('*.txt') - >>> regex - '(?s:.*\\.txt)\\Z' - >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') - <_sre.SRE_Match object; span=(0, 10), match='foobar.txt'> - - -.. seealso:: - - Module :mod:`glob` - Unix shell-style path expansion. diff --git a/third_party/python/Doc/library/formatter.rst b/third_party/python/Doc/library/formatter.rst deleted file mode 100644 index 6c10ac6fa..000000000 --- a/third_party/python/Doc/library/formatter.rst +++ /dev/null @@ -1,351 +0,0 @@ -:mod:`formatter` --- Generic output formatting -============================================== - -.. module:: formatter - :synopsis: Generic output formatter and device interface. - :deprecated: - -.. deprecated:: 3.4 - Due to lack of usage, the formatter module has been deprecated. - --------------- - -This module supports two interface definitions, each with multiple -implementations: The *formatter* interface, and the *writer* interface which is -required by the formatter interface. - -Formatter objects transform an abstract flow of formatting events into specific -output events on writer objects. Formatters manage several stack structures to -allow various properties of a writer object to be changed and restored; writers -need not be able to handle relative changes nor any sort of "change back" -operation. Specific writer properties which may be controlled via formatter -objects are horizontal alignment, font, and left margin indentations. A -mechanism is provided which supports providing arbitrary, non-exclusive style -settings to a writer as well. Additional interfaces facilitate formatting -events which are not reversible, such as paragraph separation. - -Writer objects encapsulate device interfaces. Abstract devices, such as file -formats, are supported as well as physical devices. The provided -implementations all work with abstract devices. The interface makes available -mechanisms for setting the properties which formatter objects manage and -inserting data into the output. - - -.. _formatter-interface: - -The Formatter Interface ------------------------ - -Interfaces to create formatters are dependent on the specific formatter class -being instantiated. The interfaces described below are the required interfaces -which all formatters must support once initialized. - -One data element is defined at the module level: - - -.. data:: AS_IS - - Value which can be used in the font specification passed to the ``push_font()`` - method described below, or as the new value to any other ``push_property()`` - method. Pushing the ``AS_IS`` value allows the corresponding ``pop_property()`` - method to be called without having to track whether the property was changed. - -The following attributes are defined for formatter instance objects: - - -.. attribute:: formatter.writer - - The writer instance with which the formatter interacts. - - -.. method:: formatter.end_paragraph(blanklines) - - Close any open paragraphs and insert at least *blanklines* before the next - paragraph. - - -.. method:: formatter.add_line_break() - - Add a hard line break if one does not already exist. This does not break the - logical paragraph. - - -.. method:: formatter.add_hor_rule(*args, **kw) - - Insert a horizontal rule in the output. A hard break is inserted if there is - data in the current paragraph, but the logical paragraph is not broken. The - arguments and keywords are passed on to the writer's :meth:`send_line_break` - method. - - -.. method:: formatter.add_flowing_data(data) - - Provide data which should be formatted with collapsed whitespace. Whitespace - from preceding and successive calls to :meth:`add_flowing_data` is considered as - well when the whitespace collapse is performed. The data which is passed to - this method is expected to be word-wrapped by the output device. Note that any - word-wrapping still must be performed by the writer object due to the need to - rely on device and font information. - - -.. method:: formatter.add_literal_data(data) - - Provide data which should be passed to the writer unchanged. Whitespace, - including newline and tab characters, are considered legal in the value of - *data*. - - -.. method:: formatter.add_label_data(format, counter) - - Insert a label which should be placed to the left of the current left margin. - This should be used for constructing bulleted or numbered lists. If the - *format* value is a string, it is interpreted as a format specification for - *counter*, which should be an integer. The result of this formatting becomes the - value of the label; if *format* is not a string it is used as the label value - directly. The label value is passed as the only argument to the writer's - :meth:`send_label_data` method. Interpretation of non-string label values is - dependent on the associated writer. - - Format specifications are strings which, in combination with a counter value, - are used to compute label values. Each character in the format string is copied - to the label value, with some characters recognized to indicate a transform on - the counter value. Specifically, the character ``'1'`` represents the counter - value formatter as an Arabic number, the characters ``'A'`` and ``'a'`` - represent alphabetic representations of the counter value in upper and lower - case, respectively, and ``'I'`` and ``'i'`` represent the counter value in Roman - numerals, in upper and lower case. Note that the alphabetic and roman - transforms require that the counter value be greater than zero. - - -.. method:: formatter.flush_softspace() - - Send any pending whitespace buffered from a previous call to - :meth:`add_flowing_data` to the associated writer object. This should be called - before any direct manipulation of the writer object. - - -.. method:: formatter.push_alignment(align) - - Push a new alignment setting onto the alignment stack. This may be - :const:`AS_IS` if no change is desired. If the alignment value is changed from - the previous setting, the writer's :meth:`new_alignment` method is called with - the *align* value. - - -.. method:: formatter.pop_alignment() - - Restore the previous alignment. - - -.. method:: formatter.push_font((size, italic, bold, teletype)) - - Change some or all font properties of the writer object. Properties which are - not set to :const:`AS_IS` are set to the values passed in while others are - maintained at their current settings. The writer's :meth:`new_font` method is - called with the fully resolved font specification. - - -.. method:: formatter.pop_font() - - Restore the previous font. - - -.. method:: formatter.push_margin(margin) - - Increase the number of left margin indentations by one, associating the logical - tag *margin* with the new indentation. The initial margin level is ``0``. - Changed values of the logical tag must be true values; false values other than - :const:`AS_IS` are not sufficient to change the margin. - - -.. method:: formatter.pop_margin() - - Restore the previous margin. - - -.. method:: formatter.push_style(*styles) - - Push any number of arbitrary style specifications. All styles are pushed onto - the styles stack in order. A tuple representing the entire stack, including - :const:`AS_IS` values, is passed to the writer's :meth:`new_styles` method. - - -.. method:: formatter.pop_style(n=1) - - Pop the last *n* style specifications passed to :meth:`push_style`. A tuple - representing the revised stack, including :const:`AS_IS` values, is passed to - the writer's :meth:`new_styles` method. - - -.. method:: formatter.set_spacing(spacing) - - Set the spacing style for the writer. - - -.. method:: formatter.assert_line_data(flag=1) - - Inform the formatter that data has been added to the current paragraph - out-of-band. This should be used when the writer has been manipulated - directly. The optional *flag* argument can be set to false if the writer - manipulations produced a hard line break at the end of the output. - - -.. _formatter-impls: - -Formatter Implementations -------------------------- - -Two implementations of formatter objects are provided by this module. Most -applications may use one of these classes without modification or subclassing. - - -.. class:: NullFormatter(writer=None) - - A formatter which does nothing. If *writer* is omitted, a :class:`NullWriter` - instance is created. No methods of the writer are called by - :class:`NullFormatter` instances. Implementations should inherit from this - class if implementing a writer interface but don't need to inherit any - implementation. - - -.. class:: AbstractFormatter(writer) - - The standard formatter. This implementation has demonstrated wide applicability - to many writers, and may be used directly in most circumstances. It has been - used to implement a full-featured World Wide Web browser. - - -.. _writer-interface: - -The Writer Interface --------------------- - -Interfaces to create writers are dependent on the specific writer class being -instantiated. The interfaces described below are the required interfaces which -all writers must support once initialized. Note that while most applications can -use the :class:`AbstractFormatter` class as a formatter, the writer must -typically be provided by the application. - - -.. method:: writer.flush() - - Flush any buffered output or device control events. - - -.. method:: writer.new_alignment(align) - - Set the alignment style. The *align* value can be any object, but by convention - is a string or ``None``, where ``None`` indicates that the writer's "preferred" - alignment should be used. Conventional *align* values are ``'left'``, - ``'center'``, ``'right'``, and ``'justify'``. - - -.. method:: writer.new_font(font) - - Set the font style. The value of *font* will be ``None``, indicating that the - device's default font should be used, or a tuple of the form ``(size, - italic, bold, teletype)``. Size will be a string indicating the size of - font that should be used; specific strings and their interpretation must be - defined by the application. The *italic*, *bold*, and *teletype* values are - Boolean values specifying which of those font attributes should be used. - - -.. method:: writer.new_margin(margin, level) - - Set the margin level to the integer *level* and the logical tag to *margin*. - Interpretation of the logical tag is at the writer's discretion; the only - restriction on the value of the logical tag is that it not be a false value for - non-zero values of *level*. - - -.. method:: writer.new_spacing(spacing) - - Set the spacing style to *spacing*. - - -.. method:: writer.new_styles(styles) - - Set additional styles. The *styles* value is a tuple of arbitrary values; the - value :const:`AS_IS` should be ignored. The *styles* tuple may be interpreted - either as a set or as a stack depending on the requirements of the application - and writer implementation. - - -.. method:: writer.send_line_break() - - Break the current line. - - -.. method:: writer.send_paragraph(blankline) - - Produce a paragraph separation of at least *blankline* blank lines, or the - equivalent. The *blankline* value will be an integer. Note that the - implementation will receive a call to :meth:`send_line_break` before this call - if a line break is needed; this method should not include ending the last line - of the paragraph. It is only responsible for vertical spacing between - paragraphs. - - -.. method:: writer.send_hor_rule(*args, **kw) - - Display a horizontal rule on the output device. The arguments to this method - are entirely application- and writer-specific, and should be interpreted with - care. The method implementation may assume that a line break has already been - issued via :meth:`send_line_break`. - - -.. method:: writer.send_flowing_data(data) - - Output character data which may be word-wrapped and re-flowed as needed. Within - any sequence of calls to this method, the writer may assume that spans of - multiple whitespace characters have been collapsed to single space characters. - - -.. method:: writer.send_literal_data(data) - - Output character data which has already been formatted for display. Generally, - this should be interpreted to mean that line breaks indicated by newline - characters should be preserved and no new line breaks should be introduced. The - data may contain embedded newline and tab characters, unlike data provided to - the :meth:`send_formatted_data` interface. - - -.. method:: writer.send_label_data(data) - - Set *data* to the left of the current left margin, if possible. The value of - *data* is not restricted; treatment of non-string values is entirely - application- and writer-dependent. This method will only be called at the - beginning of a line. - - -.. _writer-impls: - -Writer Implementations ----------------------- - -Three implementations of the writer object interface are provided as examples by -this module. Most applications will need to derive new writer classes from the -:class:`NullWriter` class. - - -.. class:: NullWriter() - - A writer which only provides the interface definition; no actions are taken on - any methods. This should be the base class for all writers which do not need to - inherit any implementation methods. - - -.. class:: AbstractWriter() - - A writer which can be used in debugging formatters, but not much else. Each - method simply announces itself by printing its name and arguments on standard - output. - - -.. class:: DumbWriter(file=None, maxcol=72) - - Simple writer class which writes output on the :term:`file object` passed - in as *file* or, if *file* is omitted, on standard output. The output is - simply word-wrapped to the number of columns specified by *maxcol*. This - class is suitable for reflowing a sequence of paragraphs. - diff --git a/third_party/python/Doc/library/fpectl.rst b/third_party/python/Doc/library/fpectl.rst deleted file mode 100644 index 96607165b..000000000 --- a/third_party/python/Doc/library/fpectl.rst +++ /dev/null @@ -1,121 +0,0 @@ -:mod:`fpectl` --- Floating point exception control -================================================== - -.. module:: fpectl - :platform: Unix - :synopsis: Provide control for floating point exception handling. - -.. moduleauthor:: Lee Busby -.. sectionauthor:: Lee Busby - -.. note:: - - The :mod:`fpectl` module is not built by default, and its usage is discouraged - and may be dangerous except in the hands of experts. See also the section - :ref:`fpectl-limitations` on limitations for more details. - -.. index:: single: IEEE-754 - --------------- - -Most computers carry out floating point operations in conformance with the -so-called IEEE-754 standard. On any real computer, some floating point -operations produce results that cannot be expressed as a normal floating point -value. For example, try :: - - >>> import math - >>> math.exp(1000) - inf - >>> math.exp(1000) / math.exp(1000) - nan - -(The example above will work on many platforms. DEC Alpha may be one exception.) -"Inf" is a special, non-numeric value in IEEE-754 that stands for "infinity", -and "nan" means "not a number." Note that, other than the non-numeric results, -nothing special happened when you asked Python to carry out those calculations. -That is in fact the default behaviour prescribed in the IEEE-754 standard, and -if it works for you, stop reading now. - -In some circumstances, it would be better to raise an exception and stop -processing at the point where the faulty operation was attempted. The -:mod:`fpectl` module is for use in that situation. It provides control over -floating point units from several hardware manufacturers, allowing the user to -turn on the generation of :const:`SIGFPE` whenever any of the IEEE-754 -exceptions Division by Zero, Overflow, or Invalid Operation occurs. In tandem -with a pair of wrapper macros that are inserted into the C code comprising your -python system, :const:`SIGFPE` is trapped and converted into the Python -:exc:`FloatingPointError` exception. - -The :mod:`fpectl` module defines the following functions and may raise the given -exception: - - -.. function:: turnon_sigfpe() - - Turn on the generation of :const:`SIGFPE`, and set up an appropriate signal - handler. - - -.. function:: turnoff_sigfpe() - - Reset default handling of floating point exceptions. - - -.. exception:: FloatingPointError - - After :func:`turnon_sigfpe` has been executed, a floating point operation that - raises one of the IEEE-754 exceptions Division by Zero, Overflow, or Invalid - operation will in turn raise this standard Python exception. - - -.. _fpectl-example: - -Example -------- - -The following example demonstrates how to start up and test operation of the -:mod:`fpectl` module. :: - - >>> import fpectl - >>> import fpetest - >>> fpectl.turnon_sigfpe() - >>> fpetest.test() - overflow PASS - FloatingPointError: Overflow - - div by 0 PASS - FloatingPointError: Division by zero - [ more output from test elided ] - >>> import math - >>> math.exp(1000) - Traceback (most recent call last): - File "", line 1, in - FloatingPointError: in math_1 - - -.. _fpectl-limitations: - -Limitations and other considerations ------------------------------------- - -Setting up a given processor to trap IEEE-754 floating point errors currently -requires custom code on a per-architecture basis. You may have to modify -:mod:`fpectl` to control your particular hardware. - -Conversion of an IEEE-754 exception to a Python exception requires that the -wrapper macros ``PyFPE_START_PROTECT`` and ``PyFPE_END_PROTECT`` be inserted -into your code in an appropriate fashion. Python itself has been modified to -support the :mod:`fpectl` module, but many other codes of interest to numerical -analysts have not. - -The :mod:`fpectl` module is not thread-safe. - - -.. seealso:: - - Some files in the source distribution may be interesting in learning more about - how this module operates. The include file :file:`Include/pyfpe.h` discusses the - implementation of this module at some length. :file:`Modules/fpetestmodule.c` - gives several examples of use. Many additional examples can be found in - :file:`Objects/floatobject.c`. - diff --git a/third_party/python/Doc/library/fractions.rst b/third_party/python/Doc/library/fractions.rst deleted file mode 100644 index b5a818e1c..000000000 --- a/third_party/python/Doc/library/fractions.rst +++ /dev/null @@ -1,183 +0,0 @@ -:mod:`fractions` --- Rational numbers -===================================== - -.. module:: fractions - :synopsis: Rational numbers. - -.. moduleauthor:: Jeffrey Yasskin -.. sectionauthor:: Jeffrey Yasskin - -**Source code:** :source:`Lib/fractions.py` - --------------- - -The :mod:`fractions` module provides support for rational number arithmetic. - - -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. - -.. class:: Fraction(numerator=0, denominator=1) - Fraction(other_fraction) - Fraction(float) - Fraction(decimal) - Fraction(string) - - The first version requires that *numerator* and *denominator* are instances - of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is :const:`0`, it - raises a :exc:`ZeroDivisionError`. The second version requires that - *other_fraction* is an instance of :class:`numbers.Rational` and returns a - :class:`Fraction` instance with the same value. The next two versions accept - either a :class:`float` or a :class:`decimal.Decimal` instance, and return a - :class:`Fraction` instance with exactly the same value. Note that due to the - usual issues with binary floating-point (see :ref:`tut-fp-issues`), the - argument to ``Fraction(1.1)`` is not exactly equal to 11/10, and so - ``Fraction(1.1)`` does *not* return ``Fraction(11, 10)`` as one might expect. - (But see the documentation for the :meth:`limit_denominator` method below.) - The last version of the constructor expects a string or unicode instance. - The usual form for this instance is:: - - [sign] numerator ['/' denominator] - - where the optional ``sign`` may be either '+' or '-' and - ``numerator`` and ``denominator`` (if present) are strings of - decimal digits. In addition, any string that represents a finite - value and is accepted by the :class:`float` constructor is also - accepted by the :class:`Fraction` constructor. In either form the - input string may also have leading and/or trailing whitespace. - Here are some examples:: - - >>> from fractions import Fraction - >>> Fraction(16, -10) - Fraction(-8, 5) - >>> Fraction(123) - Fraction(123, 1) - >>> Fraction() - Fraction(0, 1) - >>> Fraction('3/7') - Fraction(3, 7) - >>> Fraction(' -3/7 ') - Fraction(-3, 7) - >>> Fraction('1.414213 \t\n') - Fraction(1414213, 1000000) - >>> Fraction('-.125') - Fraction(-1, 8) - >>> Fraction('7e-6') - Fraction(7, 1000000) - >>> Fraction(2.25) - Fraction(9, 4) - >>> Fraction(1.1) - Fraction(2476979795053773, 2251799813685248) - >>> from decimal import Decimal - >>> Fraction(Decimal('1.1')) - Fraction(11, 10) - - - The :class:`Fraction` class inherits from the abstract base class - :class:`numbers.Rational`, and implements all of the methods and - operations from that class. :class:`Fraction` instances are hashable, - and should be treated as immutable. In addition, - :class:`Fraction` has the following properties and methods: - - .. versionchanged:: 3.2 - The :class:`Fraction` constructor now accepts :class:`float` and - :class:`decimal.Decimal` instances. - - - .. attribute:: numerator - - Numerator of the Fraction in lowest term. - - .. attribute:: denominator - - Denominator of the Fraction in lowest term. - - - .. method:: from_float(flt) - - This class method constructs a :class:`Fraction` representing the exact - value of *flt*, which must be a :class:`float`. Beware that - ``Fraction.from_float(0.3)`` is not the same value as ``Fraction(3, 10)``. - - .. note:: - - From Python 3.2 onwards, you can also construct a - :class:`Fraction` instance directly from a :class:`float`. - - - .. method:: from_decimal(dec) - - This class method constructs a :class:`Fraction` representing the exact - value of *dec*, which must be a :class:`decimal.Decimal` instance. - - .. note:: - - From Python 3.2 onwards, you can also construct a - :class:`Fraction` instance directly from a :class:`decimal.Decimal` - instance. - - - .. method:: limit_denominator(max_denominator=1000000) - - Finds and returns the closest :class:`Fraction` to ``self`` that has - denominator at most max_denominator. This method is useful for finding - rational approximations to a given floating-point number: - - >>> from fractions import Fraction - >>> Fraction('3.1415926535897932').limit_denominator(1000) - Fraction(355, 113) - - or for recovering a rational number that's represented as a float: - - >>> from math import pi, cos - >>> Fraction(cos(pi/3)) - Fraction(4503599627370497, 9007199254740992) - >>> Fraction(cos(pi/3)).limit_denominator() - Fraction(1, 2) - >>> Fraction(1.1).limit_denominator() - Fraction(11, 10) - - - .. method:: __floor__() - - Returns the greatest :class:`int` ``<= self``. This method can - also be accessed through the :func:`math.floor` function: - - >>> from math import floor - >>> floor(Fraction(355, 113)) - 3 - - - .. method:: __ceil__() - - Returns the least :class:`int` ``>= self``. This method can - also be accessed through the :func:`math.ceil` function. - - - .. method:: __round__() - __round__(ndigits) - - The first version returns the nearest :class:`int` to ``self``, - rounding half to even. The second version rounds ``self`` to the - nearest multiple of ``Fraction(1, 10**ndigits)`` (logically, if - ``ndigits`` is negative), again rounding half toward even. This - method can also be accessed through the :func:`round` function. - - -.. function:: gcd(a, b) - - Return the greatest common divisor of the integers *a* and *b*. If either - *a* or *b* is nonzero, then the absolute value of ``gcd(a, b)`` is the - largest integer that divides both *a* and *b*. ``gcd(a,b)`` has the same - sign as *b* if *b* is nonzero; otherwise it takes the sign of *a*. ``gcd(0, - 0)`` returns ``0``. - - .. deprecated:: 3.5 - Use :func:`math.gcd` instead. - - -.. seealso:: - - Module :mod:`numbers` - The abstract base classes making up the numeric tower. diff --git a/third_party/python/Doc/library/frameworks.rst b/third_party/python/Doc/library/frameworks.rst deleted file mode 100644 index 15ceeec9c..000000000 --- a/third_party/python/Doc/library/frameworks.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _frameworks: - -****************** -Program Frameworks -****************** - -The modules described in this chapter are frameworks that will largely dictate -the structure of your program. Currently the modules described here are all -oriented toward writing command-line interfaces. - -The full list of modules described in this chapter is: - - -.. toctree:: - - turtle.rst - cmd.rst - shlex.rst diff --git a/third_party/python/Doc/library/ftplib.rst b/third_party/python/Doc/library/ftplib.rst deleted file mode 100644 index 6c39f9a59..000000000 --- a/third_party/python/Doc/library/ftplib.rst +++ /dev/null @@ -1,442 +0,0 @@ -:mod:`ftplib` --- FTP protocol client -===================================== - -.. module:: ftplib - :synopsis: FTP protocol client (requires sockets). - -**Source code:** :source:`Lib/ftplib.py` - -.. index:: - pair: FTP; protocol - single: FTP; ftplib (standard module) - --------------- - -This module defines the class :class:`FTP` and a few related items. The -:class:`FTP` class implements the client side of the FTP protocol. You can use -this to write Python programs that perform a variety of automated FTP jobs, such -as mirroring other FTP servers. It is also used by the module -:mod:`urllib.request` to handle URLs that use FTP. For more information on FTP -(File Transfer Protocol), see Internet :rfc:`959`. - -Here's a sample session using the :mod:`ftplib` module:: - - >>> from ftplib import FTP - >>> ftp = FTP('ftp.debian.org') # connect to host, default port - >>> ftp.login() # user anonymous, passwd anonymous@ - '230 Login successful.' - >>> ftp.cwd('debian') # change into "debian" directory - >>> ftp.retrlines('LIST') # list directory contents - -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README - ... - drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool - drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project - drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools - '226 Directory send OK.' - >>> ftp.retrbinary('RETR README', open('README', 'wb').write) - '226 Transfer complete.' - >>> ftp.quit() - - -The module defines the following items: - -.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None) - - Return a new instance of the :class:`FTP` class. When *host* is given, the - method call ``connect(host)`` is made. When *user* is given, additionally - the method call ``login(user, passwd, acct)`` is made (where *passwd* and - *acct* default to the empty string when not given). The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if is not specified, the global default timeout setting - will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket - to bind to as its source address before connecting. - - The :class:`FTP` class supports the :keyword:`with` statement, e.g.: - - >>> from ftplib import FTP - >>> with FTP("ftp1.at.proftpd.org") as ftp: - ... ftp.login() - ... ftp.dir() - ... # doctest: +SKIP - '230 Anonymous login ok, restrictions apply.' - dr-xr-xr-x 9 ftp ftp 154 May 6 10:43 . - dr-xr-xr-x 9 ftp ftp 154 May 6 10:43 .. - dr-xr-xr-x 5 ftp ftp 4096 May 6 10:43 CentOS - dr-xr-xr-x 3 ftp ftp 18 Jul 10 2008 Fedora - >>> - - .. versionchanged:: 3.2 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - -.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None) - - A :class:`FTP` subclass which adds TLS support to FTP as described in - :rfc:`4217`. - Connect as usual to port 21 implicitly securing the FTP control connection - before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* -- they - can point to PEM-formatted private key and certificate chain files - (respectively) for the SSL connection. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - Here's a sample session using the :class:`FTP_TLS` class:: - - >>> ftps = FTP_TLS('ftp.pureftpd.org') - >>> ftps.login() - '230 Anonymous user logged in' - >>> ftps.prot_p() - '200 Data protection level set to "private"' - >>> ftps.nlst() - ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] - - -.. exception:: error_reply - - Exception raised when an unexpected reply is received from the server. - - -.. exception:: error_temp - - Exception raised when an error code signifying a temporary error (response - codes in the range 400--499) is received. - - -.. exception:: error_perm - - Exception raised when an error code signifying a permanent error (response - codes in the range 500--599) is received. - - -.. exception:: error_proto - - Exception raised when a reply is received from the server that does not fit - the response specifications of the File Transfer Protocol, i.e. begin with a - digit in the range 1--5. - - -.. data:: all_errors - - The set of all exceptions (as a tuple) that methods of :class:`FTP` - instances may raise as a result of problems with the FTP connection (as - opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError`. - - -.. seealso:: - - Module :mod:`netrc` - Parser for the :file:`.netrc` file format. The file :file:`.netrc` is - typically used by FTP clients to load user authentication information - before prompting the user. - - -.. _ftp-objects: - -FTP Objects ------------ - -Several methods are available in two flavors: one for handling text files and -another for binary files. These are named for the command which is used -followed by ``lines`` for the text version or ``binary`` for the binary version. - -:class:`FTP` instances have the following methods: - - -.. method:: FTP.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. - - -.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) - - Connect to the given host and port. The default port number is ``21``, as - specified by the FTP protocol specification. It is rarely needed to specify a - different port number. This function should be called only once for each - instance; it should not be called at all if a host was given when the instance - was created. All other methods can only be used after a connection has been - made. - The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If no *timeout* is passed, the global default timeout - setting will be used. - *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as - its source address before connecting. - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - -.. method:: FTP.getwelcome() - - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) - - -.. method:: FTP.login(user='anonymous', passwd='', acct='') - - Log in as the given *user*. The *passwd* and *acct* parameters are optional and - default to the empty string. If no *user* is specified, it defaults to - ``'anonymous'``. If *user* is ``'anonymous'``, the default *passwd* is - ``'anonymous@'``. This function should be called only once for each instance, - after a connection has been established; it should not be called at all if a - host and user were given when the instance was created. Most FTP commands are - only allowed after the client has logged in. The *acct* parameter supplies - "accounting information"; few systems implement this. - - -.. method:: FTP.abort() - - Abort a file transfer that is in progress. Using this does not always work, but - it's worth a try. - - -.. method:: FTP.sendcmd(cmd) - - Send a simple command string to the server and return the response string. - - -.. method:: FTP.voidcmd(cmd) - - Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. - - -.. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) - - Retrieve a file in binary transfer mode. *cmd* should be an appropriate - ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single bytes argument giving the data - block. The optional *blocksize* argument specifies the maximum chunk size to - read on the low-level socket object created to do the actual transfer (which - will also be the largest size of the data blocks passed to *callback*). A - reasonable default is chosen. *rest* means the same thing as in the - :meth:`transfercmd` method. - - -.. method:: FTP.retrlines(cmd, callback=None) - - Retrieve a file or directory listing in ASCII transfer mode. *cmd* should be - an appropriate ``RETR`` command (see :meth:`retrbinary`) or a command such as - ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). - ``LIST`` retrieves a list of files and information about those files. - ``NLST`` retrieves a list of file names. - The *callback* function is called for each line with a string argument - containing the line with the trailing CRLF stripped. The default *callback* - prints the line to ``sys.stdout``. - - -.. method:: FTP.set_pasv(val) - - Enable "passive" mode if *val* is true, otherwise disable passive mode. - Passive mode is on by default. - - -.. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) - - Store a file in binary transfer mode. *cmd* should be an appropriate - ``STOR`` command: ``"STOR filename"``. *fp* is a :term:`file object` - (opened in binary mode) which is read until EOF using its :meth:`~io.IOBase.read` - method in blocks of size *blocksize* to provide the data to be stored. - The *blocksize* argument defaults to 8192. *callback* is an optional single - parameter callable that is called on each block of data after it is sent. - *rest* means the same thing as in the :meth:`transfercmd` method. - - .. versionchanged:: 3.2 - *rest* parameter added. - - -.. method:: FTP.storlines(cmd, fp, callback=None) - - Store a file in ASCII transfer mode. *cmd* should be an appropriate - ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the - :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` - method to provide the data to be stored. *callback* is an optional single - parameter callable that is called on each line after it is sent. - - -.. method:: FTP.transfercmd(cmd, rest=None) - - Initiate a transfer over the data connection. If the transfer is active, send an - ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and - accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` - command, connect to it, and start the transfer command. Either way, return the - socket for the connection. - - If optional *rest* is given, a ``REST`` command is sent to the server, passing - *rest* as an argument. *rest* is usually a byte offset into the requested file, - telling the server to restart sending the file's bytes at the requested offset, - skipping over the initial bytes. Note however that :rfc:`959` requires only that - *rest* be a string containing characters in the printable range from ASCII code - 33 to ASCII code 126. The :meth:`transfercmd` method, therefore, converts - *rest* to a string, but no check is performed on the string's contents. If the - server does not recognize the ``REST`` command, an :exc:`error_reply` exception - will be raised. If this happens, simply call :meth:`transfercmd` without a - *rest* argument. - - -.. method:: FTP.ntransfercmd(cmd, rest=None) - - Like :meth:`transfercmd`, but returns a tuple of the data connection and the - expected size of the data. If the expected size could not be computed, ``None`` - will be returned as the expected size. *cmd* and *rest* means the same thing as - in :meth:`transfercmd`. - - -.. method:: FTP.mlsd(path="", facts=[]) - - List a directory in a standardized format by using ``MLSD`` command - (:rfc:`3659`). If *path* is omitted the current directory is assumed. - *facts* is a list of strings representing the type of information desired - (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a - tuple of two elements for every file found in path. First element is the - file name, the second one is a dictionary containing facts about the file - name. Content of this dictionary might be limited by the *facts* argument - but server is not guaranteed to return all requested facts. - - .. versionadded:: 3.3 - - -.. method:: FTP.nlst(argument[, ...]) - - Return a list of file names as returned by the ``NLST`` command. The - optional *argument* is a directory to list (default is the current server - directory). Multiple arguments can be used to pass non-standard options to - the ``NLST`` command. - - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - - -.. method:: FTP.dir(argument[, ...]) - - Produce a directory listing as returned by the ``LIST`` command, printing it to - standard output. The optional *argument* is a directory to list (default is the - current server directory). Multiple arguments can be used to pass non-standard - options to the ``LIST`` command. If the last argument is a function, it is used - as a *callback* function as for :meth:`retrlines`; the default prints to - ``sys.stdout``. This method returns ``None``. - - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - - -.. method:: FTP.rename(fromname, toname) - - Rename file *fromname* on the server to *toname*. - - -.. method:: FTP.delete(filename) - - Remove the file named *filename* from the server. If successful, returns the - text of the response, otherwise raises :exc:`error_perm` on permission errors or - :exc:`error_reply` on other errors. - - -.. method:: FTP.cwd(pathname) - - Set the current directory on the server. - - -.. method:: FTP.mkd(pathname) - - Create a new directory on the server. - - -.. method:: FTP.pwd() - - Return the pathname of the current directory on the server. - - -.. method:: FTP.rmd(dirname) - - Remove the directory named *dirname* on the server. - - -.. method:: FTP.size(filename) - - Request the size of the file named *filename* on the server. On success, the - size of the file is returned as an integer, otherwise ``None`` is returned. - Note that the ``SIZE`` command is not standardized, but is supported by many - common server implementations. - - -.. method:: FTP.quit() - - Send a ``QUIT`` command to the server and close the connection. This is the - "polite" way to close a connection, but it may raise an exception if the server - responds with an error to the ``QUIT`` command. This implies a call to the - :meth:`close` method which renders the :class:`FTP` instance useless for - subsequent calls (see below). - - -.. method:: FTP.close() - - Close the connection unilaterally. This should not be applied to an already - closed connection such as after a successful call to :meth:`~FTP.quit`. - After this call the :class:`FTP` instance should not be used any more (after - a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the - connection by issuing another :meth:`login` method). - - -FTP_TLS Objects ---------------- - -:class:`FTP_TLS` class inherits from :class:`FTP`, defining these additional objects: - -.. attribute:: FTP_TLS.ssl_version - - The SSL version to use (defaults to :attr:`ssl.PROTOCOL_SSLv23`). - -.. method:: FTP_TLS.auth() - - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - -.. method:: FTP_TLS.ccc() - - Revert control channel back to plaintext. This can be useful to take - advantage of firewalls that know how to handle NAT with non-secure FTP - without opening fixed ports. - - .. versionadded:: 3.3 - -.. method:: FTP_TLS.prot_p() - - Set up secure data connection. - -.. method:: FTP_TLS.prot_c() - - Set up clear text data connection. diff --git a/third_party/python/Doc/library/functional.rst b/third_party/python/Doc/library/functional.rst deleted file mode 100644 index 5b6185a65..000000000 --- a/third_party/python/Doc/library/functional.rst +++ /dev/null @@ -1,15 +0,0 @@ -****************************** -Functional Programming Modules -****************************** - -The modules described in this chapter provide functions and classes that support -a functional programming style, and general operations on callables. - -The following modules are documented in this chapter: - - -.. toctree:: - - itertools.rst - functools.rst - operator.rst diff --git a/third_party/python/Doc/library/functions.rst b/third_party/python/Doc/library/functions.rst deleted file mode 100644 index bc528dd17..000000000 --- a/third_party/python/Doc/library/functions.rst +++ /dev/null @@ -1,1695 +0,0 @@ -.. XXX document all delegations to __special__ methods -.. _built-in-funcs: - -Built-in Functions -================== - -The Python interpreter has a number of functions and types built into it that -are always available. They are listed here in alphabetical order. - -=================== ================= ================== ================ ==================== -.. .. Built-in Functions .. .. -=================== ================= ================== ================ ==================== -:func:`abs` |func-dict|_ :func:`help` :func:`min` :func:`setattr` -:func:`all` :func:`dir` :func:`hex` :func:`next` :func:`slice` -:func:`any` :func:`divmod` :func:`id` :func:`object` :func:`sorted` -:func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` -:func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ -:func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` -|func-bytearray|_ :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -|func-bytes|_ :func:`float` :func:`iter` :func:`print` |func-tuple|_ -:func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` -:func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` -:func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` -:func:`compile` :func:`globals` :func:`map` :func:`reversed` :func:`__import__` -:func:`complex` :func:`hasattr` :func:`max` :func:`round` -:func:`delattr` :func:`hash` |func-memoryview|_ |func-set|_ -=================== ================= ================== ================ ==================== - -.. using :func:`dict` would create a link to another page, so local targets are - used, with replacement texts to make the output in the table consistent - -.. |func-dict| replace:: ``dict()`` -.. |func-frozenset| replace:: ``frozenset()`` -.. |func-memoryview| replace:: ``memoryview()`` -.. |func-set| replace:: ``set()`` -.. |func-list| replace:: ``list()`` -.. |func-str| replace:: ``str()`` -.. |func-tuple| replace:: ``tuple()`` -.. |func-range| replace:: ``range()`` -.. |func-bytearray| replace:: ``bytearray()`` -.. |func-bytes| replace:: ``bytes()`` - -.. function:: abs(x) - - Return the absolute value of a number. The argument may be an - integer or a floating point number. If the argument is a complex number, its - magnitude is returned. - - -.. function:: all(iterable) - - Return ``True`` if all elements of the *iterable* are true (or if the iterable - is empty). Equivalent to:: - - def all(iterable): - for element in iterable: - if not element: - return False - return True - - -.. function:: any(iterable) - - Return ``True`` if any element of the *iterable* is true. If the iterable - is empty, return ``False``. Equivalent to:: - - def any(iterable): - for element in iterable: - if element: - return True - return False - - -.. function:: ascii(object) - - As :func:`repr`, return a string containing a printable representation of an - object, but escape the non-ASCII characters in the string returned by - :func:`repr` using ``\x``, ``\u`` or ``\U`` escapes. This generates a string - similar to that returned by :func:`repr` in Python 2. - - -.. function:: bin(x) - - Convert an integer number to a binary string prefixed with "0b". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. Some - examples: - - >>> bin(3) - '0b11' - >>> bin(-10) - '-0b1010' - - If prefix "0b" is desired or not, you can use either of the following ways. - - >>> format(14, '#b'), format(14, 'b') - ('0b1110', '1110') - >>> f'{14:#b}', f'{14:b}' - ('0b1110', '1110') - - See also :func:`format` for more information. - - -.. class:: bool([x]) - - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false - or omitted, this returns ``False``; otherwise it returns ``True``. The - :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). - It cannot be subclassed further. Its only instances are ``False`` and - ``True`` (see :ref:`bltin-boolean-values`). - - .. index:: pair: Boolean; type - - -.. _func-bytearray: -.. class:: bytearray([source[, encoding[, errors]]]) - :noindex: - - Return a new array of bytes. The :class:`bytearray` class is a mutable - sequence of integers in the range 0 <= x < 256. It has most of the usual - methods of mutable sequences, described in :ref:`typesseq-mutable`, as well - as most methods that the :class:`bytes` type has, see :ref:`bytes-methods`. - - The optional *source* parameter can be used to initialize the array in a few - different ways: - - * If it is a *string*, you must also give the *encoding* (and optionally, - *errors*) parameters; :func:`bytearray` then converts the string to - bytes using :meth:`str.encode`. - - * If it is an *integer*, the array will have that size and will be - initialized with null bytes. - - * If it is an object conforming to the *buffer* interface, a read-only buffer - of the object will be used to initialize the bytes array. - - * If it is an *iterable*, it must be an iterable of integers in the range - ``0 <= x < 256``, which are used as the initial contents of the array. - - Without an argument, an array of size 0 is created. - - See also :ref:`binaryseq` and :ref:`typebytearray`. - - -.. _func-bytes: -.. class:: bytes([source[, encoding[, errors]]]) - :noindex: - - Return a new "bytes" object, which is an immutable sequence of integers in - the range ``0 <= x < 256``. :class:`bytes` is an immutable version of - :class:`bytearray` -- it has the same non-mutating methods and the same - indexing and slicing behavior. - - Accordingly, constructor arguments are interpreted as for :func:`bytearray`. - - Bytes objects can also be created with literals, see :ref:`strings`. - - See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. - - -.. function:: callable(object) - - Return :const:`True` if the *object* argument appears callable, - :const:`False` if not. If this returns true, it is still possible that a - call fails, but if it is false, calling *object* will never succeed. - Note that classes are callable (calling a class returns a new instance); - instances are callable if their class has a :meth:`__call__` method. - - .. versionadded:: 3.2 - This function was first removed in Python 3.0 and then brought back - in Python 3.2. - - -.. function:: chr(i) - - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(8364)`` returns the string ``'€'``. This is the inverse of :func:`ord`. - - The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. - - -.. decorator:: classmethod - - Transform a method into a class method. - - A class method receives the class as implicit first argument, just like an - instance method receives the instance. To declare a class method, use this - idiom:: - - class C: - @classmethod - def f(cls, arg1, arg2, ...): ... - - The ``@classmethod`` form is a function :term:`decorator` -- see the description - of function definitions in :ref:`function` for details. - - It can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). The instance is ignored except for its class. If a class - method is called for a derived class, the derived class object is passed as the - implied first argument. - - Class methods are different than C++ or Java static methods. If you want those, - see :func:`staticmethod` in this section. - - For more information on class methods, consult the documentation on the standard - type hierarchy in :ref:`types`. - - -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) - - Compile the *source* into a code or AST object. Code objects can be executed - by :func:`exec` or :func:`eval`. *source* can either be a normal string, a - byte string, or an AST object. Refer to the :mod:`ast` module documentation - for information on how to work with AST objects. - - The *filename* argument should give the file from which the code was read; - pass some recognizable value if it wasn't read from a file (``''`` is - commonly used). - - The *mode* argument specifies what kind of code must be compiled; it can be - ``'exec'`` if *source* consists of a sequence of statements, ``'eval'`` if it - consists of a single expression, or ``'single'`` if it consists of a single - interactive statement (in the latter case, expression statements that - evaluate to something other than ``None`` will be printed). - - The optional arguments *flags* and *dont_inherit* control which :ref:`future - statements ` affect the compilation of *source*. If neither - is present (or both are zero) the code is compiled with those future - statements that are in effect in the code that is calling :func:`compile`. If the - *flags* argument is given and *dont_inherit* is not (or is zero) then the - future statements specified by the *flags* argument are used in addition to - those that would be used anyway. If *dont_inherit* is a non-zero integer then - the *flags* argument is it -- the future statements in effect around the call - to compile are ignored. - - Future statements are specified by bits which can be bitwise ORed together to - specify multiple statements. The bitfield required to specify a given feature - can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on - the :class:`~__future__._Feature` instance in the :mod:`__future__` module. - - The argument *optimize* specifies the optimization level of the compiler; the - default value of ``-1`` selects the optimization level of the interpreter as - given by :option:`-O` options. Explicit levels are ``0`` (no optimization; - ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) - or ``2`` (docstrings are removed too). - - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. - - If you want to parse Python code into its AST representation, see - :func:`ast.parse`. - - .. note:: - - When compiling a string with multi-line code in ``'single'`` or - ``'eval'`` mode, input must be terminated by at least one newline - character. This is to facilitate detection of incomplete and complete - statements in the :mod:`code` module. - - .. warning:: - - It is possible to crash the Python interpreter with a - sufficiently large/complex string when compiling to an AST - object due to stack depth limitations in Python's AST compiler. - - .. versionchanged:: 3.2 - Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode - does not have to end in a newline anymore. Added the *optimize* parameter. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when null bytes were encountered - in *source*. - - -.. class:: complex([real[, imag]]) - - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. - - .. note:: - - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. - - The complex type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - -.. function:: delattr(object, name) - - This is a relative of :func:`setattr`. The arguments are an object and a - string. The string must be the name of one of the object's attributes. The - function deletes the named attribute, provided the object allows it. For - example, ``delattr(x, 'foobar')`` is equivalent to ``del x.foobar``. - - -.. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) - :noindex: - - Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. - - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. - - -.. function:: dir([object]) - - Without arguments, return the list of names in the current local scope. With an - argument, attempt to return a list of valid attributes for that object. - - If the object has a method named :meth:`__dir__`, this method will be called and - must return the list of attributes. This allows objects that implement a custom - :func:`__getattr__` or :func:`__getattribute__` function to customize the way - :func:`dir` reports their attributes. - - If the object does not provide :meth:`__dir__`, the function tries its best to - gather information from the object's :attr:`~object.__dict__` attribute, if defined, and - from its type object. The resulting list is not necessarily complete, and may - be inaccurate when the object has a custom :func:`__getattr__`. - - The default :func:`dir` mechanism behaves differently with different types of - objects, as it attempts to produce the most relevant, rather than complete, - information: - - * If the object is a module object, the list contains the names of the module's - attributes. - - * If the object is a type or class object, the list contains the names of its - attributes, and recursively of the attributes of its bases. - - * Otherwise, the list contains the object's attributes' names, the names of its - class's attributes, and recursively of the attributes of its class's base - classes. - - The resulting list is sorted alphabetically. For example: - - >>> import struct - >>> dir() # show the names in the module namespace - ['__builtins__', '__name__', 'struct'] - >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', - '__initializing__', '__loader__', '__name__', '__package__', - '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', - 'unpack', 'unpack_from'] - >>> class Shape: - ... def __dir__(self): - ... return ['area', 'perimeter', 'location'] - >>> s = Shape() - >>> dir(s) - ['area', 'location', 'perimeter'] - - .. note:: - - Because :func:`dir` is supplied primarily as a convenience for use at an - interactive prompt, it tries to supply an interesting set of names more - than it tries to supply a rigorously or consistently defined set of names, - and its detailed behavior may change across releases. For example, - metaclass attributes are not in the result list when the argument is a - class. - - -.. function:: divmod(a, b) - - Take two (non complex) numbers as arguments and return a pair of numbers - consisting of their quotient and remainder when using integer division. With - mixed operand types, the rules for binary arithmetic operators apply. For - integers, the result is the same as ``(a // b, a % b)``. For floating point - numbers the result is ``(q, a % b)``, where *q* is usually ``math.floor(a / - b)`` but may be 1 less than that. In any case ``q * b + a % b`` is very - close to *a*, if ``a % b`` is non-zero it has the same sign as *b*, and ``0 - <= abs(a % b) < abs(b)``. - - -.. function:: enumerate(iterable, start=0) - - Return an enumerate object. *iterable* must be a sequence, an - :term:`iterator`, or some other object which supports iteration. - The :meth:`~iterator.__next__` method of the iterator returned by - :func:`enumerate` returns a tuple containing a count (from *start* which - defaults to 0) and the values obtained from iterating over *iterable*. - - >>> seasons = ['Spring', 'Summer', 'Fall', 'Winter'] - >>> list(enumerate(seasons)) - [(0, 'Spring'), (1, 'Summer'), (2, 'Fall'), (3, 'Winter')] - >>> list(enumerate(seasons, start=1)) - [(1, 'Spring'), (2, 'Summer'), (3, 'Fall'), (4, 'Winter')] - - Equivalent to:: - - def enumerate(sequence, start=0): - n = start - for elem in sequence: - yield n, elem - n += 1 - - -.. function:: eval(expression, globals=None, locals=None) - - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. - - The *expression* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a - reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. - This means that *expression* normally has full - access to the standard :mod:`builtins` module and restricted environments are - propagated. If the *locals* dictionary is omitted it defaults to the *globals* - dictionary. If both dictionaries are omitted, the expression is executed in the - environment where :func:`eval` is called. The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: - - >>> x = 1 - >>> eval('x+1') - 2 - - This function can also be used to execute arbitrary code objects (such as - those created by :func:`compile`). In this case pass a code object instead - of a string. If the code object has been compiled with ``'exec'`` as the - *mode* argument, :func:`eval`\'s return value will be ``None``. - - Hints: dynamic execution of statements is supported by the :func:`exec` - function. The :func:`globals` and :func:`locals` functions - returns the current global and local dictionary, respectively, which may be - useful to pass around for use by :func:`eval` or :func:`exec`. - - See :func:`ast.literal_eval` for a function that can safely evaluate strings - with expressions containing only literals. - -.. index:: builtin: exec - -.. function:: exec(object[, globals[, locals]]) - - This function supports dynamic execution of Python code. *object* must be - either a string or a code object. If it is a string, the string is parsed as - a suite of Python statements which is then executed (unless a syntax error - occurs). [#]_ If it is a code object, it is simply executed. In all cases, - the code that's executed is expected to be valid as file input (see the - section "File input" in the Reference Manual). Be aware that the - :keyword:`return` and :keyword:`yield` statements may not be used outside of - function definitions even within the context of code passed to the - :func:`exec` function. The return value is ``None``. - - In all cases, if the optional parts are omitted, the code is executed in the - current scope. If only *globals* is provided, it must be a dictionary, which - will be used for both the global and the local variables. If *globals* and - *locals* are given, they are used for the global and local variables, - respectively. If provided, *locals* can be any mapping object. Remember - that at module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. - - If the *globals* dictionary does not contain a value for the key - ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. - - .. note:: - - The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around - for use as the second and third argument to :func:`exec`. - - .. note:: - - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. - Pass an explicit *locals* dictionary if you need to see effects of the - code on *locals* after function :func:`exec` returns. - - -.. function:: filter(function, iterable) - - Construct an iterator from those elements of *iterable* for which *function* - returns true. *iterable* may be either a sequence, a container which - supports iteration, or an iterator. If *function* is ``None``, the identity - function is assumed, that is, all elements of *iterable* that are false are - removed. - - Note that ``filter(function, iterable)`` is equivalent to the generator - expression ``(item for item in iterable if function(item))`` if function is - not ``None`` and ``(item for item in iterable if item)`` if function is - ``None``. - - See :func:`itertools.filterfalse` for the complementary function that returns - elements of *iterable* for which *function* returns false. - - -.. class:: float([x]) - - .. index:: - single: NaN - single: Infinity - - Return a floating point number constructed from a number or string *x*. - - If the argument is a string, it should contain a decimal number, optionally - preceded by a sign, and optionally embedded in whitespace. The optional - sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value - produced. The argument may also be a string representing a NaN - (not-a-number), or a positive or negative infinity. More precisely, the - input must conform to the following grammar after leading and trailing - whitespace characters are removed: - - .. productionlist:: - sign: "+" | "-" - infinity: "Infinity" | "inf" - nan: "nan" - numeric_value: `floatnumber` | `infinity` | `nan` - numeric_string: [`sign`] `numeric_value` - - Here ``floatnumber`` is the form of a Python floating-point literal, - described in :ref:`floating`. Case is not significant, so, for example, - "inf", "Inf", "INFINITY" and "iNfINity" are all acceptable spellings for - positive infinity. - - Otherwise, if the argument is an integer or a floating point number, a - floating point number with the same value (within Python's floating point - precision) is returned. If the argument is outside the range of a Python - float, an :exc:`OverflowError` will be raised. - - For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. - - If no argument is given, ``0.0`` is returned. - - Examples:: - - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf - - The float type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - -.. index:: - single: __format__ - single: string; format() (built-in function) - -.. function:: format(value[, format_spec]) - - Convert a *value* to a "formatted" representation, as controlled by - *format_spec*. The interpretation of *format_spec* will depend on the type - of the *value* argument, however there is a standard formatting syntax that - is used by most built-in types: :ref:`formatspec`. - - The default *format_spec* is an empty string which usually gives the same - effect as calling :func:`str(value) `. - - A call to ``format(value, format_spec)`` is translated to - ``type(value).__format__(value, format_spec)`` which bypasses the instance - dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches - :mod:`object` and the *format_spec* is non-empty, or if either the - *format_spec* or the return value are not strings. - - .. versionchanged:: 3.4 - ``object().__format__(format_spec)`` raises :exc:`TypeError` - if *format_spec* is not an empty string. - - -.. _func-frozenset: -.. class:: frozenset([iterable]) - :noindex: - - Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and - :ref:`types-set` for documentation about this class. - - For other containers see the built-in :class:`set`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. - - -.. function:: getattr(object, name[, default]) - - Return the value of the named attribute of *object*. *name* must be a string. - If the string is the name of one of the object's attributes, the result is the - value of that attribute. For example, ``getattr(x, 'foobar')`` is equivalent to - ``x.foobar``. If the named attribute does not exist, *default* is returned if - provided, otherwise :exc:`AttributeError` is raised. - - -.. function:: globals() - - Return a dictionary representing the current global symbol table. This is always - the dictionary of the current module (inside a function or method, this is the - module where it is defined, not the module from which it is called). - - -.. function:: hasattr(object, name) - - The arguments are an object and a string. The result is ``True`` if the - string is the name of one of the object's attributes, ``False`` if not. (This - is implemented by calling ``getattr(object, name)`` and seeing whether it - raises an :exc:`AttributeError` or not.) - - -.. function:: hash(object) - - Return the hash value of the object (if it has one). Hash values are - integers. They are used to quickly compare dictionary keys during a - dictionary lookup. Numeric values that compare equal have the same hash - value (even if they are of different types, as is the case for 1 and 1.0). - - .. note:: - - For objects with custom :meth:`__hash__` methods, note that :func:`hash` - truncates the return value based on the bit width of the host machine. - See :meth:`__hash__` for details. - -.. function:: help([object]) - - Invoke the built-in help system. (This function is intended for interactive - use.) If no argument is given, the interactive help system starts on the - interpreter console. If the argument is a string, then the string is looked up - as the name of a module, function, class, method, keyword, or documentation - topic, and a help page is printed on the console. If the argument is any other - kind of object, a help page on the object is generated. - - This function is added to the built-in namespace by the :mod:`site` module. - - .. versionchanged:: 3.4 - Changes to :mod:`pydoc` and :mod:`inspect` mean that the reported - signatures for callables are now more comprehensive and consistent. - - -.. function:: hex(x) - - Convert an integer number to a lowercase hexadecimal string prefixed with - "0x". If *x* is not a Python :class:`int` object, it has to define an - :meth:`__index__` method that returns an integer. Some examples: - - >>> hex(255) - '0xff' - >>> hex(-42) - '-0x2a' - - If you want to convert an integer number to an uppercase or lower hexadecimal - string with prefix or not, you can use either of the following ways: - - >>> '%#x' % 255, '%x' % 255, '%X' % 255 - ('0xff', 'ff', 'FF') - >>> format(255, '#x'), format(255, 'x'), format(255, 'X') - ('0xff', 'ff', 'FF') - >>> f'{255:#x}', f'{255:x}', f'{255:X}' - ('0xff', 'ff', 'FF') - - See also :func:`format` for more information. - - See also :func:`int` for converting a hexadecimal string to an - integer using a base of 16. - - .. note:: - - To obtain a hexadecimal string representation for a float, use the - :meth:`float.hex` method. - - -.. function:: id(object) - - Return the "identity" of an object. This is an integer which - is guaranteed to be unique and constant for this object during its lifetime. - Two objects with non-overlapping lifetimes may have the same :func:`id` - value. - - .. impl-detail:: This is the address of the object in memory. - - -.. function:: input([prompt]) - - If the *prompt* argument is present, it is written to standard output without - a trailing newline. The function then reads a line from input, converts it - to a string (stripping a trailing newline), and returns that. When EOF is - read, :exc:`EOFError` is raised. Example:: - - >>> s = input('--> ') # doctest: +SKIP - --> Monty Python's Flying Circus - >>> s # doctest: +SKIP - "Monty Python's Flying Circus" - - If the :mod:`readline` module was loaded, then :func:`input` will use it - to provide elaborate line editing and history features. - - -.. class:: int(x=0) - int(x, base=10) - - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`__trunc__`, - it returns ``x.__trunc__()``. - For floating point numbers, this truncates towards zero. - - If *x* is not a number or if *base* is given, then *x* must be a string, - :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer - literal ` in radix *base*. Optionally, the literal can be - preceded by ``+`` or ``-`` (with no space in between) and surrounded by - whitespace. A base-n literal consists of the digits 0 to n-1, with ``a`` - to ``z`` (or ``A`` to ``Z``) having - values 10 to 35. The default *base* is 10. The allowed values are 0 and 2--36. - Base-2, -8, and -16 literals can be optionally prefixed with ``0b``/``0B``, - ``0o``/``0O``, or ``0x``/``0X``, as with integer literals in code. Base 0 - means to interpret exactly as a code literal, so that the actual base is 2, - 8, 10, or 16, and so that ``int('010', 0)`` is not legal, while - ``int('010')`` is, as well as ``int('010', 8)``. - - The integer type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.4 - If *base* is not an instance of :class:`int` and the *base* object has a - :meth:`base.__index__ ` method, that method is called - to obtain an integer for the base. Previous versions used - :meth:`base.__int__ ` instead of :meth:`base.__index__ - `. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - -.. function:: isinstance(object, classinfo) - - Return true if the *object* argument is an instance of the *classinfo* - argument, or of a (direct, indirect or :term:`virtual `) subclass thereof. If *object* is not - an object of the given type, the function always returns false. - If *classinfo* is a tuple of type objects (or recursively, other such - tuples), return true if *object* is an instance of any of the types. - If *classinfo* is not a type or tuple of types and such tuples, - a :exc:`TypeError` exception is raised. - - -.. function:: issubclass(class, classinfo) - - Return true if *class* is a subclass (direct, indirect or :term:`virtual - `) of *classinfo*. A - class is considered a subclass of itself. *classinfo* may be a tuple of class - objects, in which case every entry in *classinfo* will be checked. In any other - case, a :exc:`TypeError` exception is raised. - - -.. function:: iter(object[, sentinel]) - - Return an :term:`iterator` object. The first argument is interpreted very - differently depending on the presence of the second argument. Without a - second argument, *object* must be a collection object which supports the - iteration protocol (the :meth:`__iter__` method), or it must support the - sequence protocol (the :meth:`__getitem__` method with integer arguments - starting at ``0``). If it does not support either of those protocols, - :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, - then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its - :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will - be returned. - - See also :ref:`typeiter`. - - One useful application of the second form of :func:`iter` is to read lines of - a file until a certain line is reached. The following example reads a file - until the :meth:`~io.TextIOBase.readline` method returns an empty string:: - - with open('mydata.txt') as fp: - for line in iter(fp.readline, ''): - process_line(line) - - -.. function:: len(s) - - Return the length (the number of items) of an object. The argument may be a - sequence (such as a string, bytes, tuple, list, or range) or a collection - (such as a dictionary, set, or frozen set). - - -.. _func-list: -.. class:: list([iterable]) - :noindex: - - Rather than being a function, :class:`list` is actually a mutable - sequence type, as documented in :ref:`typesseq-list` and :ref:`typesseq`. - - -.. function:: locals() - - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. - - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. - -.. function:: map(function, iterable, ...) - - Return an iterator that applies *function* to every item of *iterable*, - yielding the results. If additional *iterable* arguments are passed, - *function* must take that many arguments and is applied to the items from all - iterables in parallel. With multiple iterables, the iterator stops when the - shortest iterable is exhausted. For cases where the function inputs are - already arranged into argument tuples, see :func:`itertools.starmap`\. - - -.. function:: max(iterable, *[, key, default]) - max(arg1, arg2, *args[, key]) - - Return the largest item in an iterable or the largest of two or more - arguments. - - If one positional argument is provided, it should be an :term:`iterable`. - The largest item in the iterable is returned. If two or more positional - arguments are provided, the largest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. - - If multiple items are maximal, the function returns the first one - encountered. This is consistent with other sort-stability preserving tools - such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and - ``heapq.nlargest(1, iterable, key=keyfunc)``. - - .. versionadded:: 3.4 - The *default* keyword-only argument. - - -.. _func-memoryview: -.. function:: memoryview(obj) - :noindex: - - Return a "memory view" object created from the given argument. See - :ref:`typememoryview` for more information. - - -.. function:: min(iterable, *[, key, default]) - min(arg1, arg2, *args[, key]) - - Return the smallest item in an iterable or the smallest of two or more - arguments. - - If one positional argument is provided, it should be an :term:`iterable`. - The smallest item in the iterable is returned. If two or more positional - arguments are provided, the smallest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. - - If multiple items are minimal, the function returns the first one - encountered. This is consistent with other sort-stability preserving tools - such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, - iterable, key=keyfunc)``. - - .. versionadded:: 3.4 - The *default* keyword-only argument. - - -.. function:: next(iterator[, default]) - - Retrieve the next item from the *iterator* by calling its - :meth:`~iterator.__next__` method. If *default* is given, it is returned - if the iterator is exhausted, otherwise :exc:`StopIteration` is raised. - - -.. class:: object() - - Return a new featureless object. :class:`object` is a base for all classes. - It has the methods that are common to all instances of Python classes. This - function does not accept any arguments. - - .. note:: - - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. - - -.. function:: oct(x) - - Convert an integer number to an octal string prefixed with "0o". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`__index__` method that returns an integer. For - example: - - >>> oct(8) - '0o10' - >>> oct(-56) - '-0o70' - - If you want to convert an integer number to octal string either with prefix - "0o" or not, you can use either of the following ways. - - >>> '%#o' % 10, '%o' % 10 - ('0o12', '12') - >>> format(10, '#o'), format(10, 'o') - ('0o12', '12') - >>> f'{10:#o}', f'{10:o}' - ('0o12', '12') - - See also :func:`format` for more information. - - .. index:: - single: file object; open() built-in function - -.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) - - Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. - - *file* is a :term:`path-like object` giving the pathname (absolute or - relative to the current working directory) of the file to be opened or an - integer file descriptor of the file to be wrapped. (If a file descriptor is - given, it is closed when the returned I/O object is closed, unless *closefd* - is set to ``False``.) - - *mode* is an optional string that specifies the mode in which the file is - opened. It defaults to ``'r'`` which means open for reading in text mode. - Other common values are ``'w'`` for writing (truncating the file if it - already exists), ``'x'`` for exclusive creation and ``'a'`` for appending - (which on *some* Unix systems, means that *all* writes append to the end of - the file regardless of the current seek position). In text mode, if - *encoding* is not specified the encoding used is platform dependent: - ``locale.getpreferredencoding(False)`` is called to get the current locale - encoding. (For reading and writing raw bytes use binary mode and leave - *encoding* unspecified.) The available modes are: - - .. _filemodes: - - .. index:: - pair: file; modes - - ========= =============================================================== - Character Meaning - ========= =============================================================== - ``'r'`` open for reading (default) - ``'w'`` open for writing, truncating the file first - ``'x'`` open for exclusive creation, failing if the file already exists - ``'a'`` open for writing, appending to the end of the file if it exists - ``'b'`` binary mode - ``'t'`` text mode (default) - ``'+'`` open a disk file for updating (reading and writing) - ``'U'`` :term:`universal newlines` mode (deprecated) - ========= =============================================================== - - The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). - For binary read-write access, the mode ``'w+b'`` opens and truncates the file - to 0 bytes. ``'r+b'`` opens the file without truncation. - - As mentioned in the :ref:`io-overview`, Python distinguishes between binary - and text I/O. Files opened in binary mode (including ``'b'`` in the *mode* - argument) return contents as :class:`bytes` objects without any decoding. In - text mode (the default, or when ``'t'`` is included in the *mode* argument), - the contents of the file are returned as :class:`str`, the bytes having been - first decoded using a platform-dependent encoding or using the specified - *encoding* if given. - - .. note:: - - Python doesn't depend on the underlying operating system's notion of text - files; all the processing is done by Python itself, and is therefore - platform-independent. - - *buffering* is an optional integer used to set the buffering policy. Pass 0 - to switch buffering off (only allowed in binary mode), 1 to select line - buffering (only usable in text mode), and an integer > 1 to indicate the size - in bytes of a fixed-size chunk buffer. When no *buffering* argument is - given, the default buffering policy works as follows: - - * Binary files are buffered in fixed-size chunks; the size of the buffer is - chosen using a heuristic trying to determine the underlying device's "block - size" and falling back on :attr:`io.DEFAULT_BUFFER_SIZE`. On many systems, - the buffer will typically be 4096 or 8192 bytes long. - - * "Interactive" text files (files for which :meth:`~io.IOBase.isatty` - returns ``True``) use line buffering. Other text files use the policy - described above for binary files. - - *encoding* is the name of the encoding used to decode or encode the file. - This should only be used in text mode. The default encoding is platform - dependent (whatever :func:`locale.getpreferredencoding` returns), but any - :term:`text encoding` supported by Python - can be used. See the :mod:`codecs` module for - the list of supported encodings. - - *errors* is an optional string that specifies how encoding and decoding - errors are to be handled—this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with - :func:`codecs.register_error` is also valid. The standard names - include: - - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. - - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. - - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as code - points in the Unicode Private Use Area ranging from U+DC80 to - U+DCFF. These private code points will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference ``&#nnn;``. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. - - .. index:: - single: universal newlines; open() built-in function - - *newline* controls how :term:`universal newlines` mode works (it only - applies to text mode). It can be ``None``, ``''``, ``'\n'``, ``'\r'``, and - ``'\r\n'``. It works as follows: - - * When reading input from the stream, if *newline* is ``None``, universal - newlines mode is enabled. Lines in the input can end in ``'\n'``, - ``'\r'``, or ``'\r\n'``, and these are translated into ``'\n'`` before - being returned to the caller. If it is ``''``, universal newlines mode is - enabled, but line endings are returned to the caller untranslated. If it - has any of the other legal values, input lines are only terminated by the - given string, and the line ending is returned to the caller untranslated. - - * When writing output to the stream, if *newline* is ``None``, any ``'\n'`` - characters written are translated to the system default line separator, - :data:`os.linesep`. If *newline* is ``''`` or ``'\n'``, no translation - takes place. If *newline* is any of the other legal values, any ``'\n'`` - characters written are translated to the given string. - - If *closefd* is ``False`` and a file descriptor rather than a filename was - given, the underlying file descriptor will be kept open when the file is - closed. If a filename is given *closefd* must be ``True`` (the default) - otherwise an error will be raised. - - A custom opener can be used by passing a callable as *opener*. The underlying - file descriptor for the file object is then obtained by calling *opener* with - (*file*, *flags*). *opener* must return an open file descriptor (passing - :mod:`os.open` as *opener* results in functionality similar to passing - ``None``). - - The newly created file is :ref:`non-inheritable `. - - The following example uses the :ref:`dir_fd ` parameter of the - :func:`os.open` function to open a file relative to a given directory:: - - >>> import os - >>> dir_fd = os.open('somedir', os.O_RDONLY) - >>> def opener(path, flags): - ... return os.open(path, flags, dir_fd=dir_fd) - ... - >>> with open('spamspam.txt', 'w', opener=opener) as f: - ... print('This will be written to somedir/spamspam.txt', file=f) - ... - >>> os.close(dir_fd) # don't leak a file descriptor - - The type of :term:`file object` returned by the :func:`open` function - depends on the mode. When :func:`open` is used to open a file in a text - mode (``'w'``, ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of - :class:`io.TextIOBase` (specifically :class:`io.TextIOWrapper`). When used - to open a file in a binary mode with buffering, the returned class is a - subclass of :class:`io.BufferedIOBase`. The exact class varies: in read - binary mode, it returns an :class:`io.BufferedReader`; in write binary and - append binary modes, it returns an :class:`io.BufferedWriter`, and in - read/write mode, it returns an :class:`io.BufferedRandom`. When buffering is - disabled, the raw stream, a subclass of :class:`io.RawIOBase`, - :class:`io.FileIO`, is returned. - - .. index:: - single: line-buffered I/O - single: unbuffered I/O - single: buffer size, I/O - single: I/O control; buffering - single: binary mode - single: text mode - module: sys - - See also the file handling modules, such as, :mod:`fileinput`, :mod:`io` - (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, - and :mod:`shutil`. - - .. versionchanged:: - 3.3 - - * The *opener* parameter was added. - * The ``'x'`` mode was added. - * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - * :exc:`FileExistsError` is now raised if the file opened in exclusive - creation mode (``'x'``) already exists. - - .. versionchanged:: - 3.4 - - * The file is now non-inheritable. - - .. deprecated-removed:: 3.4 4.0 - - The ``'U'`` mode. - - .. versionchanged:: - 3.5 - - * If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - * The ``'namereplace'`` error handler was added. - - .. versionchanged:: - 3.6 - - * Support added to accept objects implementing :class:`os.PathLike`. - * On Windows, opening a console buffer may return a subclass of - :class:`io.RawIOBase` other than :class:`io.FileIO`. - -.. function:: ord(c) - - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, - ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) - returns ``8364``. This is the inverse of :func:`chr`. - - -.. function:: pow(x, y[, z]) - - Return *x* to the power *y*; if *z* is present, return *x* to the power *y*, - modulo *z* (computed more efficiently than ``pow(x, y) % z``). The two-argument - form ``pow(x, y)`` is equivalent to using the power operator: ``x**y``. - - The arguments must have numeric types. With mixed operand types, the - coercion rules for binary arithmetic operators apply. For :class:`int` - operands, the result has the same type as the operands (after coercion) - unless the second argument is negative; in that case, all arguments are - converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. If the second argument is - negative, the third argument must be omitted. If *z* is present, *x* and *y* - must be of integer types, and *y* must be non-negative. - - -.. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) - - Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword - arguments. - - All non-keyword arguments are converted to strings like :func:`str` does and - written to the stream, separated by *sep* and followed by *end*. Both *sep* - and *end* must be strings; they can also be ``None``, which means to use the - default values. If no *objects* are given, :func:`print` will just write - *end*. - - The *file* argument must be an object with a ``write(string)`` method; if it - is not present or ``None``, :data:`sys.stdout` will be used. Since printed - arguments are converted to text strings, :func:`print` cannot be used with - binary mode file objects. For these, use ``file.write(...)`` instead. - - Whether output is buffered is usually determined by *file*, but if the - *flush* keyword argument is true, the stream is forcibly flushed. - - .. versionchanged:: 3.3 - Added the *flush* keyword argument. - - -.. class:: property(fget=None, fset=None, fdel=None, doc=None) - - Return a property attribute. - - *fget* is a function for getting an attribute value. *fset* is a function - for setting an attribute value. *fdel* is a function for deleting an attribute - value. And *doc* creates a docstring for the attribute. - - A typical use is to define a managed attribute ``x``:: - - class C: - def __init__(self): - self._x = None - - def getx(self): - return self._x - - def setx(self, value): - self._x = value - - def delx(self): - del self._x - - x = property(getx, setx, delx, "I'm the 'x' property.") - - If *c* is an instance of *C*, ``c.x`` will invoke the getter, - ``c.x = value`` will invoke the setter and ``del c.x`` the deleter. - - If given, *doc* will be the docstring of the property attribute. Otherwise, the - property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: - - class Parrot: - def __init__(self): - self._voltage = 100000 - - @property - def voltage(self): - """Get the current voltage.""" - return self._voltage - - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" - for a read-only attribute with the same name, and it sets the docstring for - *voltage* to "Get the current voltage." - - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: - - class C: - def __init__(self): - self._x = None - - @property - def x(self): - """I'm the 'x' property.""" - return self._x - - @x.setter - def x(self, value): - self._x = value - - @x.deleter - def x(self): - del self._x - - This code is exactly equivalent to the first example. Be sure to give the - additional functions the same name as the original property (``x`` in this - case.) - - The returned property object also has the attributes ``fget``, ``fset``, and - ``fdel`` corresponding to the constructor arguments. - - .. versionchanged:: 3.5 - The docstrings of property objects are now writeable. - - -.. _func-range: -.. function:: range(stop) - range(start, stop[, step]) - :noindex: - - Rather than being a function, :class:`range` is actually an immutable - sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. - - -.. function:: repr(object) - - Return a string containing a printable representation of an object. For many - types, this function makes an attempt to return a string that would yield an - object with the same value when passed to :func:`eval`, otherwise the - representation is a string enclosed in angle brackets that contains the name - of the type of the object together with additional information often - including the name and address of the object. A class can control what this - function returns for its instances by defining a :meth:`__repr__` method. - - -.. function:: reversed(seq) - - Return a reverse :term:`iterator`. *seq* must be an object which has - a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`__getitem__` method with integer - arguments starting at ``0``). - - -.. function:: round(number[, ndigits]) - - Return *number* rounded to *ndigits* precision after the decimal - point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. - - For the built-in types supporting :func:`round`, values are rounded to the - closest multiple of 10 to the power minus *ndigits*; if two multiples are - equally close, rounding is done toward the even choice (so, for example, - both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). Any integer value is valid for *ndigits* (positive, zero, or - negative). The return value is an integer if *ndigits* is omitted or - ``None``. - Otherwise the return value has the same type as *number*. - - For a general Python object ``number``, ``round`` delegates to - ``number.__round__``. - - .. note:: - - The behavior of :func:`round` for floats can be surprising: for example, - ``round(2.675, 2)`` gives ``2.67`` instead of the expected ``2.68``. - This is not a bug: it's a result of the fact that most decimal fractions - can't be represented exactly as a float. See :ref:`tut-fp-issues` for - more information. - - -.. _func-set: -.. class:: set([iterable]) - :noindex: - - Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and - :ref:`types-set` for documentation about this class. - - For other containers see the built-in :class:`frozenset`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. - - -.. function:: setattr(object, name, value) - - This is the counterpart of :func:`getattr`. The arguments are an object, a - string and an arbitrary value. The string may name an existing attribute or a - new attribute. The function assigns the value to the attribute, provided the - object allows it. For example, ``setattr(x, 'foobar', 123)`` is equivalent to - ``x.foobar = 123``. - - -.. class:: slice(stop) - slice(start, stop[, step]) - - .. index:: single: Numerical Python - - Return a :term:`slice` object representing the set of indices specified by - ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop` and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however they are used by Numerical Python and other third party extensions. - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. - - -.. function:: sorted(iterable, *, key=None, reverse=False) - - Return a new sorted list from the items in *iterable*. - - Has two optional arguments which must be specified as keyword arguments. - - *key* specifies a function of one argument that is used to extract a comparison - key from each element in *iterable* (for example, ``key=str.lower``). The - default value is ``None`` (compare the elements directly). - - *reverse* is a boolean value. If set to ``True``, then the list elements are - sorted as if each comparison were reversed. - - Use :func:`functools.cmp_to_key` to convert an old-style *cmp* function to a - *key* function. - - The built-in :func:`sorted` function is guaranteed to be stable. A sort is - stable if it guarantees not to change the relative order of elements that - compare equal --- this is helpful for sorting in multiple passes (for - example, sort by department, then by salary grade). - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - -.. decorator:: staticmethod - - Transform a method into a static method. - - A static method does not receive an implicit first argument. To declare a static - method, use this idiom:: - - class C: - @staticmethod - def f(arg1, arg2, ...): ... - - The ``@staticmethod`` form is a function :term:`decorator` -- see the - description of function definitions in :ref:`function` for details. - - It can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). The instance is ignored except for its class. - - Static methods in Python are similar to those found in Java or C++. Also see - :func:`classmethod` for a variant that is useful for creating alternate class - constructors. - - Like all decorators, it is also possible to call ``staticmethod`` as - a regular function and do something with its result. This is needed - in some cases where you need a reference to a function from a class - body and you want to avoid the automatic transformation to instance - method. For these cases, use this idiom:: - - class C: - builtin_open = staticmethod(open) - - For more information on static methods, consult the documentation on the - standard type hierarchy in :ref:`types`. - - -.. index:: - single: string; str() (built-in function) - -.. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - :noindex: - - Return a :class:`str` version of *object*. See :func:`str` for details. - - ``str`` is the built-in string :term:`class`. For general information - about strings, see :ref:`textseq`. - - -.. function:: sum(iterable[, start]) - - Sums *start* and the items of an *iterable* from left to right and returns the - total. *start* defaults to ``0``. The *iterable*'s items are normally numbers, - and the start value is not allowed to be a string. - - For some use cases, there are good alternatives to :func:`sum`. - The preferred, fast way to concatenate a sequence of strings is by calling - ``''.join(sequence)``. To add floating point values with extended precision, - see :func:`math.fsum`\. To concatenate a series of iterables, consider using - :func:`itertools.chain`. - -.. function:: super([type[, object-or-type]]) - - Return a proxy object that delegates method calls to a parent or sibling - class of *type*. This is useful for accessing inherited methods that have - been overridden in a class. The search order is same as that used by - :func:`getattr` except that the *type* itself is skipped. - - The :attr:`~class.__mro__` attribute of the *type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. - - If the second argument is omitted, the super object returned is unbound. If - the second argument is an object, ``isinstance(obj, type)`` must be true. If - the second argument is a type, ``issubclass(type2, type)`` must be true (this - is useful for classmethods). - - There are two typical use cases for *super*. In a class hierarchy with - single inheritance, *super* can be used to refer to parent classes without - naming them explicitly, thus making the code more maintainable. This use - closely parallels the use of *super* in other programming languages. - - The second use case is to support cooperative multiple inheritance in a - dynamic execution environment. This use case is unique to Python and is - not found in statically compiled languages or languages that only support - single inheritance. This makes it possible to implement "diamond diagrams" - where multiple base classes implement the same method. Good design dictates - that this method have the same calling signature in every case (because the - order of calls is determined at runtime, because that order adapts - to changes in the class hierarchy, and because that order can include - sibling classes that are unknown prior to runtime). - - For both use cases, a typical superclass call looks like this:: - - class C(B): - def method(self, arg): - super().method(arg) # This does the same thing as: - # super(C, self).method(arg) - - Note that :func:`super` is implemented as part of the binding process for - explicit dotted attribute lookups such as ``super().__getitem__(name)``. - It does so by implementing its own :meth:`__getattribute__` method for searching - classes in a predictable order that supports cooperative multiple inheritance. - Accordingly, :func:`super` is undefined for implicit lookups using statements or - operators such as ``super()[name]``. - - Also note that, aside from the zero argument form, :func:`super` is not - limited to use inside methods. The two argument form specifies the - arguments exactly and makes the appropriate references. The zero - argument form only works inside a class definition, as the compiler fills - in the necessary details to correctly retrieve the class being defined, - as well as accessing the current instance for ordinary methods. - - For practical suggestions on how to design cooperative classes using - :func:`super`, see `guide to using super() - `_. - - -.. _func-tuple: -.. function:: tuple([iterable]) - :noindex: - - Rather than being a function, :class:`tuple` is actually an immutable - sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. - - -.. class:: type(object) - type(name, bases, dict) - - .. index:: object: type - - With one argument, return the type of an *object*. The return value is a - type object and generally the same object as returned by - :attr:`object.__class__ `. - - The :func:`isinstance` built-in function is recommended for testing the type - of an object, because it takes subclasses into account. - - - With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~definition.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and is copied to a standard dictionary to become the - :attr:`~object.__dict__` attribute. For example, the following two - statements create identical :class:`type` objects: - - >>> class X: - ... a = 1 - ... - >>> X = type('X', (object,), dict(a=1)) - - See also :ref:`bltin-type-objects`. - - .. versionchanged:: 3.6 - Subclasses of :class:`type` which don't override ``type.__new__`` may no - longer use the one-argument form to get the type of an object. - -.. function:: vars([object]) - - Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`~object.__dict__` attribute. - - Objects such as modules and instances have an updateable :attr:`~object.__dict__` - attribute; however, other objects may have write restrictions on their - :attr:`~object.__dict__` attributes (for example, classes use a - :class:`types.MappingProxyType` to prevent direct dictionary updates). - - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. - - -.. function:: zip(*iterables) - - Make an iterator that aggregates elements from each of the iterables. - - Returns an iterator of tuples, where the *i*-th tuple contains - the *i*-th element from each of the argument sequences or iterables. The - iterator stops when the shortest input iterable is exhausted. With a single - iterable argument, it returns an iterator of 1-tuples. With no arguments, - it returns an empty iterator. Equivalent to:: - - def zip(*iterables): - # zip('ABCD', 'xy') --> Ax By - sentinel = object() - iterators = [iter(it) for it in iterables] - while iterators: - result = [] - for it in iterators: - elem = next(it, sentinel) - if elem is sentinel: - return - result.append(elem) - yield tuple(result) - - The left-to-right evaluation order of the iterables is guaranteed. This - makes possible an idiom for clustering a data series into n-length groups - using ``zip(*[iter(s)]*n)``. This repeats the *same* iterator ``n`` times - so that each output tuple has the result of ``n`` calls to the iterator. - This has the effect of dividing the input into n-length chunks. - - :func:`zip` should only be used with unequal length inputs when you don't - care about trailing, unmatched values from the longer iterables. If those - values are important, use :func:`itertools.zip_longest` instead. - - :func:`zip` in conjunction with the ``*`` operator can be used to unzip a - list:: - - >>> x = [1, 2, 3] - >>> y = [4, 5, 6] - >>> zipped = zip(x, y) - >>> list(zipped) - [(1, 4), (2, 5), (3, 6)] - >>> x2, y2 = zip(*zip(x, y)) - >>> x == list(x2) and y == list(y2) - True - - -.. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) - - .. index:: - statement: import - module: imp - - .. note:: - - This is an advanced function that is not needed in everyday Python - programming, unlike :func:`importlib.import_module`. - - This function is invoked by the :keyword:`import` statement. It can be - replaced (by importing the :mod:`builtins` module and assigning to - ``builtins.__import__``) in order to change semantics of the - :keyword:`import` statement, but doing so is **strongly** discouraged as it - is usually simpler to use import hooks (see :pep:`302`) to attain the same - goals and does not cause issues with code which assumes the default import - implementation is in use. Direct use of :func:`__import__` is also - discouraged in favor of :func:`importlib.import_module`. - - The function imports the module *name*, potentially using the given *globals* - and *locals* to determine how to interpret the name in a package context. - The *fromlist* gives the names of objects or submodules that should be - imported from the module given by *name*. The standard implementation does - not use its *locals* argument at all, and uses its *globals* only to - determine the package context of the :keyword:`import` statement. - - *level* specifies whether to use absolute or relative imports. ``0`` (the - default) means only perform absolute imports. Positive values for - *level* indicate the number of parent directories to search relative to the - directory of the module calling :func:`__import__` (see :pep:`328` for the - details). - - When the *name* variable is of the form ``package.module``, normally, the - top-level package (the name up till the first dot) is returned, *not* the - module named by *name*. However, when a non-empty *fromlist* argument is - given, the module named by *name* is returned. - - For example, the statement ``import spam`` results in bytecode resembling the - following code:: - - spam = __import__('spam', globals(), locals(), [], 0) - - The statement ``import spam.ham`` results in this call:: - - spam = __import__('spam.ham', globals(), locals(), [], 0) - - Note how :func:`__import__` returns the toplevel module here because this is - the object that is bound to a name by the :keyword:`import` statement. - - On the other hand, the statement ``from spam.ham import eggs, sausage as - saus`` results in :: - - _temp = __import__('spam.ham', globals(), locals(), ['eggs', 'sausage'], 0) - eggs = _temp.eggs - saus = _temp.sausage - - Here, the ``spam.ham`` module is returned from :func:`__import__`. From this - object, the names to import are retrieved and assigned to their respective - names. - - If you simply want to import a module (potentially within a package) by name, - use :func:`importlib.import_module`. - - .. versionchanged:: 3.3 - Negative values for *level* are no longer supported (which also changes - the default value to 0). - - -.. rubric:: Footnotes - -.. [#] Note that the parser only accepts the Unix-style end of line convention. - If you are reading the code from a file, make sure to use newline conversion - mode to convert Windows or Mac-style newlines. diff --git a/third_party/python/Doc/library/functools.rst b/third_party/python/Doc/library/functools.rst deleted file mode 100644 index 41c06c647..000000000 --- a/third_party/python/Doc/library/functools.rst +++ /dev/null @@ -1,486 +0,0 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== - -.. module:: functools - :synopsis: Higher-order functions and operations on callable objects. - -.. moduleauthor:: Peter Harris -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Peter Harris - -**Source code:** :source:`Lib/functools.py` - --------------- - -The :mod:`functools` module is for higher-order functions: functions that act on -or return other functions. In general, any callable object can be treated as a -function for the purposes of this module. - -The :mod:`functools` module defines the following functions: - -.. function:: cmp_to_key(func) - - Transform an old-style comparison function to a :term:`key function`. Used - with tools that accept key functions (such as :func:`sorted`, :func:`min`, - :func:`max`, :func:`heapq.nlargest`, :func:`heapq.nsmallest`, - :func:`itertools.groupby`). This function is primarily used as a transition - tool for programs being converted from Python 2 which supported the use of - comparison functions. - - A comparison function is any callable that accept two arguments, compares them, - and returns a negative number for less-than, zero for equality, or a positive - number for greater-than. A key function is a callable that accepts one - argument and returns another value to be used as the sort key. - - Example:: - - sorted(iterable, key=cmp_to_key(locale.strcoll)) # locale-aware sort order - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - - .. versionadded:: 3.2 - - -.. decorator:: lru_cache(maxsize=128, typed=False) - - Decorator to wrap a function with a memoizing callable that saves up to the - *maxsize* most recent calls. It can save time when an expensive or I/O bound - function is periodically called with the same arguments. - - Since a dictionary is used to cache results, the positional and keyword - arguments to the function must be hashable. - - If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. - - If *typed* is set to true, function arguments of different types will be - cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated - as distinct calls with distinct results. - - To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` - function that returns a :term:`named tuple` showing *hits*, *misses*, - *maxsize* and *currsize*. In a multi-threaded environment, the hits - and misses are approximate. - - The decorator also provides a :func:`cache_clear` function for clearing or - invalidating the cache. - - The original underlying function is accessible through the - :attr:`__wrapped__` attribute. This is useful for introspection, for - bypassing the cache, or for rewrapping the function with a different cache. - - An `LRU (least recently used) cache - `_ works - best when the most recent calls are the best predictors of upcoming calls (for - example, the most popular articles on a news server tend to change each day). - The cache's size limit assures that the cache does not grow without bound on - long-running processes such as web servers. - - In general, the LRU cache should only be used when you want to reuse - previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). - - Example of an LRU cache for static web content:: - - @lru_cache(maxsize=32) - def get_pep(num): - 'Retrieve text of a Python Enhancement Proposal' - resource = 'http://www.python.org/dev/peps/pep-%04d/' % num - try: - with urllib.request.urlopen(resource) as s: - return s.read() - except urllib.error.HTTPError: - return 'Not Found' - - >>> for n in 8, 290, 308, 320, 8, 218, 320, 279, 289, 320, 9991: - ... pep = get_pep(n) - ... print(n, len(pep)) - - >>> get_pep.cache_info() - CacheInfo(hits=3, misses=8, maxsize=32, currsize=8) - - Example of efficiently computing - `Fibonacci numbers `_ - using a cache to implement a - `dynamic programming `_ - technique:: - - @lru_cache(maxsize=None) - def fib(n): - if n < 2: - return n - return fib(n-1) + fib(n-2) - - >>> [fib(n) for n in range(16)] - [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610] - - >>> fib.cache_info() - CacheInfo(hits=28, misses=16, maxsize=None, currsize=16) - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *typed* option. - -.. decorator:: total_ordering - - Given a class defining one or more rich comparison ordering methods, this - class decorator supplies the rest. This simplifies the effort involved - in specifying all of the possible rich comparison operations: - - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. - - For example:: - - @total_ordering - class Student: - def _is_valid_operand(self, other): - return (hasattr(other, "lastname") and - hasattr(other, "firstname")) - def __eq__(self, other): - if not self._is_valid_operand(other): - return NotImplemented - return ((self.lastname.lower(), self.firstname.lower()) == - (other.lastname.lower(), other.firstname.lower())) - def __lt__(self, other): - if not self._is_valid_operand(other): - return NotImplemented - return ((self.lastname.lower(), self.firstname.lower()) < - (other.lastname.lower(), other.firstname.lower())) - - .. note:: - - While this decorator makes it easy to create well behaved totally - ordered types, it *does* come at the cost of slower execution and - more complex stack traces for the derived comparison methods. If - performance benchmarking indicates this is a bottleneck for a given - application, implementing all six rich comparison methods instead is - likely to provide an easy speed boost. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for - unrecognised types is now supported. - -.. function:: partial(func, *args, **keywords) - - Return a new :ref:`partial object` which when called - will behave like *func* called with the positional arguments *args* - and keyword arguments *keywords*. If more arguments are supplied to the - call, they are appended to *args*. If additional keyword arguments are - supplied, they extend and override *keywords*. - Roughly equivalent to:: - - def partial(func, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*args, *fargs, **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc - - The :func:`partial` is used for partial function application which "freezes" - some portion of a function's arguments and/or keywords resulting in a new object - with a simplified signature. For example, :func:`partial` can be used to create - a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: - - >>> from functools import partial - >>> basetwo = partial(int, base=2) - >>> basetwo.__doc__ = 'Convert base 2 string to an int.' - >>> basetwo('10010') - 18 - - -.. class:: partialmethod(func, *args, **keywords) - - Return a new :class:`partialmethod` descriptor which behaves - like :class:`partial` except that it is designed to be used as a method - definition rather than being directly callable. - - *func* must be a :term:`descriptor` or a callable (objects which are both, - like normal functions, are handled as descriptors). - - When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or - another instance of :class:`partialmethod`), calls to ``__get__`` are - delegated to the underlying descriptor, and an appropriate - :ref:`partial object` returned as the result. - - When *func* is a non-descriptor callable, an appropriate bound method is - created dynamically. This behaves like a normal Python function when - used as a method: the *self* argument will be inserted as the first - positional argument, even before the *args* and *keywords* supplied to - the :class:`partialmethod` constructor. - - Example:: - - >>> class Cell(object): - ... def __init__(self): - ... self._alive = False - ... @property - ... def alive(self): - ... return self._alive - ... def set_state(self, state): - ... self._alive = bool(state) - ... set_alive = partialmethod(set_state, True) - ... set_dead = partialmethod(set_state, False) - ... - >>> c = Cell() - >>> c.alive - False - >>> c.set_alive() - >>> c.alive - True - - .. versionadded:: 3.4 - - -.. function:: reduce(function, iterable[, initializer]) - - Apply *function* of two arguments cumulatively to the items of *sequence*, from - left to right, so as to reduce the sequence to a single value. For example, - ``reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])`` calculates ``((((1+2)+3)+4)+5)``. - The left argument, *x*, is the accumulated value and the right argument, *y*, is - the update value from the *sequence*. If the optional *initializer* is present, - it is placed before the items of the sequence in the calculation, and serves as - a default when the sequence is empty. If *initializer* is not given and - *sequence* contains only one item, the first item is returned. - - Roughly equivalent to:: - - def reduce(function, iterable, initializer=None): - it = iter(iterable) - if initializer is None: - value = next(it) - else: - value = initializer - for element in it: - value = function(value, element) - return value - - -.. decorator:: singledispatch - - Transform a function into a :term:`single-dispatch ` :term:`generic function`. - - To define a generic function, decorate it with the ``@singledispatch`` - decorator. Note that the dispatch happens on the type of the first argument, - create your function accordingly:: - - >>> from functools import singledispatch - >>> @singledispatch - ... def fun(arg, verbose=False): - ... if verbose: - ... print("Let me just say,", end=" ") - ... print(arg) - - To add overloaded implementations to the function, use the :func:`register` - attribute of the generic function. It is a decorator, taking a type - parameter and decorating a function implementing the operation for that - type:: - - >>> @fun.register(int) - ... def _(arg, verbose=False): - ... if verbose: - ... print("Strength in numbers, eh?", end=" ") - ... print(arg) - ... - >>> @fun.register(list) - ... def _(arg, verbose=False): - ... if verbose: - ... print("Enumerate this:") - ... for i, elem in enumerate(arg): - ... print(i, elem) - - To enable registering lambdas and pre-existing functions, the - :func:`register` attribute can be used in a functional form:: - - >>> def nothing(arg, verbose=False): - ... print("Nothing.") - ... - >>> fun.register(type(None), nothing) - - The :func:`register` attribute returns the undecorated function which - enables decorator stacking, pickling, as well as creating unit tests for - each variant independently:: - - >>> @fun.register(float) - ... @fun.register(Decimal) - ... def fun_num(arg, verbose=False): - ... if verbose: - ... print("Half of your number:", end=" ") - ... print(arg / 2) - ... - >>> fun_num is fun - False - - When called, the generic function dispatches on the type of the first - argument:: - - >>> fun("Hello, world.") - Hello, world. - >>> fun("test.", verbose=True) - Let me just say, test. - >>> fun(42, verbose=True) - Strength in numbers, eh? 42 - >>> fun(['spam', 'spam', 'eggs', 'spam'], verbose=True) - Enumerate this: - 0 spam - 1 spam - 2 eggs - 3 spam - >>> fun(None) - Nothing. - >>> fun(1.23) - 0.615 - - Where there is no registered implementation for a specific type, its - method resolution order is used to find a more generic implementation. - The original function decorated with ``@singledispatch`` is registered - for the base ``object`` type, which means it is used if no better - implementation is found. - - To check which implementation will the generic function choose for - a given type, use the ``dispatch()`` attribute:: - - >>> fun.dispatch(float) - - >>> fun.dispatch(dict) # note: default implementation - - - To access all registered implementations, use the read-only ``registry`` - attribute:: - - >>> fun.registry.keys() - dict_keys([, , , - , , - ]) - >>> fun.registry[float] - - >>> fun.registry[object] - - - .. versionadded:: 3.4 - - -.. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) - - Update a *wrapper* function to look like the *wrapped* function. The optional - arguments are tuples to specify which attributes of the original function are - assigned directly to the matching attributes on the wrapper function and which - attributes of the wrapper function are updated with the corresponding attributes - from the original function. The default values for these arguments are the - module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which - updates the wrapper function's ``__dict__``, i.e. the instance dictionary). - - To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function - automatically adds a ``__wrapped__`` attribute to the wrapper that refers to - the function being wrapped. - - The main intended use for this function is in :term:`decorator` functions which - wrap the decorated function and return the wrapper. If the wrapper function is - not updated, the metadata of the returned function will reflect the wrapper - definition rather than the original function definition, which is typically less - than helpful. - - :func:`update_wrapper` may be used with callables other than functions. Any - attributes named in *assigned* or *updated* that are missing from the object - being wrapped are ignored (i.e. this function will not attempt to set them - on the wrapper function). :exc:`AttributeError` is still raised if the - wrapper function itself is missing any attributes named in *updated*. - - .. versionadded:: 3.2 - Automatic addition of the ``__wrapped__`` attribute. - - .. versionadded:: 3.2 - Copying of the ``__annotations__`` attribute by default. - - .. versionchanged:: 3.2 - Missing attributes no longer trigger an :exc:`AttributeError`. - - .. versionchanged:: 3.4 - The ``__wrapped__`` attribute now always refers to the wrapped - function, even if that function defined a ``__wrapped__`` attribute. - (see :issue:`17482`) - - -.. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) - - This is a convenience function for invoking :func:`update_wrapper` as a - function decorator when defining a wrapper function. It is equivalent to - ``partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated)``. - For example:: - - >>> from functools import wraps - >>> def my_decorator(f): - ... @wraps(f) - ... def wrapper(*args, **kwds): - ... print('Calling decorated function') - ... return f(*args, **kwds) - ... return wrapper - ... - >>> @my_decorator - ... def example(): - ... """Docstring""" - ... print('Called example function') - ... - >>> example() - Calling decorated function - Called example function - >>> example.__name__ - 'example' - >>> example.__doc__ - 'Docstring' - - Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` - would have been lost. - - -.. _partial-objects: - -:class:`partial` Objects ------------------------- - -:class:`partial` objects are callable objects created by :func:`partial`. They -have three read-only attributes: - - -.. attribute:: partial.func - - A callable object or function. Calls to the :class:`partial` object will be - forwarded to :attr:`func` with new arguments and keywords. - - -.. attribute:: partial.args - - The leftmost positional arguments that will be prepended to the positional - arguments provided to a :class:`partial` object call. - - -.. attribute:: partial.keywords - - The keyword arguments that will be supplied when the :class:`partial` object is - called. - -:class:`partial` objects are like :class:`function` objects in that they are -callable, weak referencable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. diff --git a/third_party/python/Doc/library/gc.rst b/third_party/python/Doc/library/gc.rst deleted file mode 100644 index 87d682445..000000000 --- a/third_party/python/Doc/library/gc.rst +++ /dev/null @@ -1,270 +0,0 @@ -:mod:`gc` --- Garbage Collector interface -========================================= - -.. module:: gc - :synopsis: Interface to the cycle-detecting garbage collector. - -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - --------------- - -This module provides an interface to the optional garbage collector. It -provides the ability to disable the collector, tune the collection frequency, -and set debugging options. It also provides access to unreachable objects that -the collector found but cannot free. Since the collector supplements the -reference counting already used in Python, you can disable the collector if you -are sure your program does not create reference cycles. Automatic collection -can be disabled by calling ``gc.disable()``. To debug a leaking program call -``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes -``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in -gc.garbage for inspection. - -The :mod:`gc` module provides the following functions: - - -.. function:: enable() - - Enable automatic garbage collection. - - -.. function:: disable() - - Disable automatic garbage collection. - - -.. function:: isenabled() - - Returns true if automatic collection is enabled. - - -.. function:: collect(generation=2) - - With no arguments, run a full collection. The optional argument *generation* - may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. - - The free lists maintained for a number of built-in types are cleared - whenever a full collection or collection of the highest generation (2) - is run. Not all items in some free lists may be freed due to the - particular implementation, in particular :class:`float`. - - -.. function:: set_debug(flags) - - Set the garbage collection debugging flags. Debugging information will be - written to ``sys.stderr``. See below for a list of debugging flags which can be - combined using bit operations to control debugging. - - -.. function:: get_debug() - - Return the debugging flags currently set. - - -.. function:: get_objects() - - Returns a list of all objects tracked by the collector, excluding the list - returned. - - -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) - - Set the garbage collection thresholds (the collection frequency). Setting - *threshold0* to zero disables collection. - - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object - allocations and deallocations since the last collection. When the number of - allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. Similarly, *threshold2* - controls the number of collections of generation ``1`` before collecting - generation ``2``. - - -.. function:: get_count() - - Return the current collection counts as a tuple of ``(count0, count1, - count2)``. - - -.. function:: get_threshold() - - Return the current collection thresholds as a tuple of ``(threshold0, - threshold1, threshold2)``. - - -.. function:: get_referrers(*objs) - - Return the list of objects that directly refer to any of objs. This function - will only locate those containers which support garbage collection; extension - types which do refer to other objects but do not support garbage collection will - not be found. - - Note that objects which have already been dereferenced, but which live in cycles - and have not yet been collected by the garbage collector can be listed among the - resulting referrers. To get only currently live objects, call :func:`collect` - before calling :func:`get_referrers`. - - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. - - -.. function:: get_referents(*objs) - - Return a list of objects directly referred to by any of the arguments. The - referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects - that support garbage collection, and are only required to visit objects that may - be involved in a cycle. So, for example, if an integer is directly reachable - from an argument, that integer object may or may not appear in the result list. - - -.. function:: is_tracked(obj) - - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't - tracked and instances of non-atomic types (containers, user-defined - objects...) are. However, some type-specific optimizations can be present - in order to suppress the garbage collector footprint of simple instances - (e.g. dicts containing only atomic keys and values):: - - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked({}) - False - >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) - True - - .. versionadded:: 3.1 - - -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): - -.. data:: garbage - - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-NULL ``tp_del`` slot. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. - - .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a - :exc:`ResourceWarning` is emitted, which is silent by default. If - :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects - are printed. - - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :attr:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - - -The following constants are provided for use with :func:`set_debug`: - - -.. data:: DEBUG_STATS - - Print statistics during collection. This information can be useful when tuning - the collection frequency. - - -.. data:: DEBUG_COLLECTABLE - - Print information on collectable objects found. - - -.. data:: DEBUG_UNCOLLECTABLE - - Print information of uncollectable objects found (objects which are not - reachable but cannot be freed by the collector). These objects will be added - to the ``garbage`` list. - - .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. - -.. data:: DEBUG_SAVEALL - - When set, all unreachable objects found will be appended to *garbage* rather - than being freed. This can be useful for debugging a leaking program. - - -.. data:: DEBUG_LEAK - - The debugging flags necessary for the collector to print information about a - leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | - DEBUG_SAVEALL``). diff --git a/third_party/python/Doc/library/getopt.rst b/third_party/python/Doc/library/getopt.rst deleted file mode 100644 index 336deab28..000000000 --- a/third_party/python/Doc/library/getopt.rst +++ /dev/null @@ -1,164 +0,0 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= - -.. module:: getopt - :synopsis: Portable parser for command line options; support both short and - long option names. - -**Source code:** :source:`Lib/getopt.py` - -.. note:: - - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`getopt` function. Users who - are unfamiliar with the C :c:func:`getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. - --------------- - -This module helps scripts to parse the command line arguments in ``sys.argv``. -It supports the same conventions as the Unix :c:func:`getopt` function (including -the special meanings of arguments of the form '``-``' and '``--``'). Long -options similar to those supported by GNU software may be used as well via an -optional third argument. - -This module provides two functions and an -exception: - - -.. function:: getopt(args, shortopts, longopts=[]) - - Parses command line options and parameter list. *args* is the argument list to - be parsed, without the leading reference to the running program. Typically, this - means ``sys.argv[1:]``. *shortopts* is the string of option letters that the - script wants to recognize, with options that require an argument followed by a - colon (``':'``; i.e., the same format that Unix :c:func:`getopt` uses). - - .. note:: - - Unlike GNU :c:func:`getopt`, after a non-option argument, all further - arguments are considered also non-options. This is similar to the way - non-GNU Unix systems work. - - *longopts*, if specified, must be a list of strings with the names of the - long options which should be supported. The leading ``'--'`` characters - should not be included in the option name. Long options which require an - argument should be followed by an equal sign (``'='``). Optional arguments - are not supported. To accept only long options, *shortopts* should be an - empty string. Long options on the command line can be recognized so long as - they provide a prefix of the option name that matches exactly one of the - accepted options. For example, if *longopts* is ``['foo', 'frob']``, the - option ``--fo`` will match as ``--foo``, but ``--f`` will - not match uniquely, so :exc:`GetoptError` will be raised. - - The return value consists of two elements: the first is a list of ``(option, - value)`` pairs; the second is the list of program arguments left after the - option list was stripped (this is a trailing slice of *args*). Each - option-and-value pair returned has the option as its first element, prefixed - with a hyphen for short options (e.g., ``'-x'``) or two hyphens for long - options (e.g., ``'--long-option'``), and the option argument as its - second element, or an empty string if the option has no argument. The - options occur in the list in the same order in which they were found, thus - allowing multiple occurrences. Long and short options may be mixed. - - -.. function:: gnu_getopt(args, shortopts, longopts=[]) - - This function works like :func:`getopt`, except that GNU style scanning mode is - used by default. This means that option and non-option arguments may be - intermixed. The :func:`getopt` function stops processing options as soon as a - non-option argument is encountered. - - If the first character of the option string is ``'+'``, or if the environment - variable :envvar:`POSIXLY_CORRECT` is set, then option processing stops as - soon as a non-option argument is encountered. - - -.. exception:: GetoptError - - This is raised when an unrecognized option is found in the argument list or when - an option requiring an argument is given none. The argument to the exception is - a string indicating the cause of the error. For long options, an argument given - to an option which does not require one will also cause this exception to be - raised. The attributes :attr:`msg` and :attr:`opt` give the error message and - related option; if there is no specific option to which the exception relates, - :attr:`opt` is an empty string. - -.. XXX deprecated? -.. exception:: error - - Alias for :exc:`GetoptError`; for backward compatibility. - -An example using only Unix style options: - - >>> import getopt - >>> args = '-a -b -cfoo -d bar a1 a2'.split() - >>> args - ['-a', '-b', '-cfoo', '-d', 'bar', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'abc:d:') - >>> optlist - [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] - >>> args - ['a1', 'a2'] - -Using long option names is equally easy: - - >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' - >>> args = s.split() - >>> args - ['--condition=foo', '--testing', '--output-file', 'abc.def', '-x', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'x', [ - ... 'condition=', 'output-file=', 'testing']) - >>> optlist - [('--condition', 'foo'), ('--testing', ''), ('--output-file', 'abc.def'), ('-x', '')] - >>> args - ['a1', 'a2'] - -In a script, typical usage is something like this:: - - import getopt, sys - - def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "ho:v", ["help", "output="]) - except getopt.GetoptError as err: - # print help information and exit: - print(err) # will print something like "option -a not recognized" - usage() - sys.exit(2) - output = None - verbose = False - for o, a in opts: - if o == "-v": - verbose = True - elif o in ("-h", "--help"): - usage() - sys.exit() - elif o in ("-o", "--output"): - output = a - else: - assert False, "unhandled option" - # ... - - if __name__ == "__main__": - main() - -Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module:: - - import argparse - - if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-o', '--output') - parser.add_argument('-v', dest='verbose', action='store_true') - args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. - -.. seealso:: - - Module :mod:`argparse` - Alternative command line option and argument parsing library. - diff --git a/third_party/python/Doc/library/getpass.rst b/third_party/python/Doc/library/getpass.rst deleted file mode 100644 index 82b11919a..000000000 --- a/third_party/python/Doc/library/getpass.rst +++ /dev/null @@ -1,51 +0,0 @@ -:mod:`getpass` --- Portable password input -========================================== - -.. module:: getpass - :synopsis: Portable reading of passwords and retrieval of the userid. - -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Fred L. Drake, Jr. -.. Windows (& Mac?) support by Guido van Rossum. - -**Source code:** :source:`Lib/getpass.py` - --------------- - -The :mod:`getpass` module provides two functions: - - -.. function:: getpass(prompt='Password: ', stream=None) - - Prompt the user for a password without echoing. The user is prompted using - the string *prompt*, which defaults to ``'Password: '``. On Unix, the - prompt is written to the file-like object *stream* using the replace error - handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this - argument is ignored on Windows). - - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and - issuing a :exc:`GetPassWarning`. - - .. note:: - If you call getpass from within IDLE, the input may be done in the - terminal you launched IDLE from rather than the idle window itself. - -.. exception:: GetPassWarning - - A :exc:`UserWarning` subclass issued when password input may be echoed. - - -.. function:: getuser() - - Return the "login name" of the user. - - This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and - returns the value of the first one which is set to a non-empty string. If - none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. - - In general, this function should be preferred over :func:`os.getlogin()`. diff --git a/third_party/python/Doc/library/gettext.rst b/third_party/python/Doc/library/gettext.rst deleted file mode 100644 index c98c5e7de..000000000 --- a/third_party/python/Doc/library/gettext.rst +++ /dev/null @@ -1,660 +0,0 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= - -.. module:: gettext - :synopsis: Multilingual internationalization services. - -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw - -**Source code:** :source:`Lib/gettext.py` - --------------- - -The :mod:`gettext` module provides internationalization (I18N) and localization -(L10N) services for your Python modules and applications. It supports both the -GNU ``gettext`` message catalog API and a higher level, class-based API that may -be more appropriate for Python files. The interface described below allows you -to write your module and application messages in one natural language, and -provide a catalog of translated messages for running under different natural -languages. - -Some hints on localizing your Python modules and applications are also given. - - -GNU :program:`gettext` API --------------------------- - -The :mod:`gettext` module defines the following API, which is very similar to -the GNU :program:`gettext` API. If you use this API you will affect the -translation of your entire application globally. Often this is what you want if -your application is monolingual, with the choice of language dependent on the -locale of your user. If you are localizing a Python module, or if your -application needs to switch languages on the fly, you probably want to use the -class-based API instead. - - -.. function:: bindtextdomain(domain, localedir=None) - - Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using - the path (on Unix): :file:`localedir/language/LC_MESSAGES/domain.mo`, where - *languages* is searched for in the environment variables :envvar:`LANGUAGE`, - :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. - - If *localedir* is omitted or ``None``, then the current binding for *domain* is - returned. [#]_ - - -.. function:: bind_textdomain_codeset(domain, codeset=None) - - Bind the *domain* to *codeset*, changing the encoding of byte strings - returned by the :func:`lgettext`, :func:`ldgettext`, :func:`lngettext` - and :func:`ldngettext` functions. - If *codeset* is omitted, then the current binding is returned. - - -.. function:: textdomain(domain=None) - - Change or query the current global domain. If *domain* is ``None``, then the - current global domain is returned, otherwise the global domain is set to - *domain*, which is returned. - - -.. index:: single: _ (underscore); gettext -.. function:: gettext(message) - - Return the localized translation of *message*, based on the current global - domain, language, and locale directory. This function is usually aliased as - :func:`_` in the local namespace (see examples below). - - -.. function:: dgettext(domain, message) - - Like :func:`.gettext`, but look the message up in the specified *domain*. - - -.. function:: ngettext(singular, plural, n) - - Like :func:`.gettext`, but consider plural forms. If a translation is found, - apply the plural formula to *n*, and return the resulting message (some - languages have more than two plural forms). If no translation is found, return - *singular* if *n* is 1; return *plural* otherwise. - - The Plural formula is taken from the catalog header. It is a C or Python - expression that has a free variable *n*; the expression evaluates to the index - of the plural in the catalog. See - `the GNU gettext documentation `__ - for the precise syntax to be used in :file:`.po` files and the - formulas for a variety of languages. - - -.. function:: dngettext(domain, singular, plural, n) - - Like :func:`ngettext`, but look the message up in the specified *domain*. - - -.. function:: lgettext(message) -.. function:: ldgettext(domain, message) -.. function:: lngettext(singular, plural, n) -.. function:: ldngettext(domain, singular, plural, n) - - Equivalent to the corresponding functions without the ``l`` prefix - (:func:`.gettext`, :func:`dgettext`, :func:`ngettext` and :func:`dngettext`), - but the translation is returned as a byte string encoded in the preferred - system encoding if no other encoding was explicitly set with - :func:`bind_textdomain_codeset`. - - .. warning:: - - These functions should be avoided in Python 3, because they return - encoded bytes. It's much better to use alternatives which return - Unicode strings instead, since most Python applications will want to - manipulate human readable text as strings instead of bytes. Further, - it's possible that you may get unexpected Unicode-related exceptions - if there are encoding problems with the translated strings. It is - possible that the ``l*()`` functions will be deprecated in future Python - versions due to their inherent problems and limitations. - - -Note that GNU :program:`gettext` also defines a :func:`dcgettext` method, but -this was deemed not useful and so it is currently unimplemented. - -Here's an example of typical usage for this API:: - - import gettext - gettext.bindtextdomain('myapplication', '/path/to/my/language/directory') - gettext.textdomain('myapplication') - _ = gettext.gettext - # ... - print(_('This is a translatable string.')) - - -Class-based API ---------------- - -The class-based API of the :mod:`gettext` module gives you more flexibility and -greater convenience than the GNU :program:`gettext` API. It is the recommended -way of localizing your Python applications and modules. :mod:`!gettext` defines -a "translations" class which implements the parsing of GNU :file:`.mo` format -files, and has methods for returning strings. Instances of this "translations" -class can also install themselves in the built-in namespace as the function -:func:`_`. - - -.. function:: find(domain, localedir=None, languages=None, all=False) - - This function implements the standard :file:`.mo` file search algorithm. It - takes a *domain*, identical to what :func:`textdomain` takes. Optional - *localedir* is as in :func:`bindtextdomain` Optional *languages* is a list of - strings, where each string is a language code. - - If *localedir* is not given, then the default system locale directory is used. - [#]_ If *languages* is not given, then the following environment variables are - searched: :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and - :envvar:`LANG`. The first one returning a non-empty value is used for the - *languages* variable. The environment variables should contain a colon separated - list of languages, which will be split on the colon to produce the expected list - of language code strings. - - :func:`find` then expands and normalizes the languages, and then iterates - through them, searching for an existing file built of these components: - - :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo` - - The first such file name that exists is returned by :func:`find`. If no such - file is found, then ``None`` is returned. If *all* is given, it returns a list - of all file names, in the order in which they appear in the languages list or - the environment variables. - - -.. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False, codeset=None) - - Return a :class:`Translations` instance based on the *domain*, *localedir*, - and *languages*, which are first passed to :func:`find` to get a list of the - associated :file:`.mo` file paths. Instances with identical :file:`.mo` file - names are cached. The actual class instantiated is either *class_* if - provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. If provided, *codeset* will change - the charset used to encode translated strings in the - :meth:`~NullTranslations.lgettext` and :meth:`~NullTranslations.lngettext` - methods. - - If multiple files are found, later files are used as fallbacks for earlier ones. - To allow setting the fallback, :func:`copy.copy` is used to clone each - translation object from the cache; the actual instance data is still shared with - the cache. - - If no :file:`.mo` file is found, this function raises :exc:`OSError` if - *fallback* is false (which is the default), and returns a - :class:`NullTranslations` instance if *fallback* is true. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - - -.. function:: install(domain, localedir=None, codeset=None, names=None) - - This installs the function :func:`_` in Python's builtins namespace, based on - *domain*, *localedir*, and *codeset* which are passed to the function - :func:`translation`. - - For the *names* parameter, please see the description of the translation - object's :meth:`~NullTranslations.install` method. - - As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`_` - function, like this:: - - print(_('This string will be translated.')) - - For convenience, you want the :func:`_` function to be installed in Python's - builtins namespace, so it is easily accessible in all modules of your - application. - - -The :class:`NullTranslations` class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Translation classes are what actually implement the translation of original -source file message strings to translated message strings. The base class used -by all translation classes is :class:`NullTranslations`; this provides the basic -interface you can use to write your own specialized translation classes. Here -are the methods of :class:`!NullTranslations`: - - -.. class:: NullTranslations(fp=None) - - Takes an optional :term:`file object` *fp*, which is ignored by the base class. - Initializes "protected" instance variables *_info* and *_charset* which are set - by derived classes, as well as *_fallback*, which is set through - :meth:`add_fallback`. It then calls ``self._parse(fp)`` if *fp* is not - ``None``. - - .. method:: _parse(fp) - - No-op'd in the base class, this method takes file object *fp*, and reads - the data from the file, initializing its message catalog. If you have an - unsupported message catalog file format, you should override this method - to parse your format. - - - .. method:: add_fallback(fallback) - - Add *fallback* as the fallback object for the current translation object. - A translation object should consult the fallback if it cannot provide a - translation for a given message. - - - .. method:: gettext(message) - - If a fallback has been set, forward :meth:`!gettext` to the fallback. - Otherwise, return *message*. Overridden in derived classes. - - - .. method:: ngettext(singular, plural, n) - - If a fallback has been set, forward :meth:`!ngettext` to the fallback. - Otherwise, return *singular* if *n* is 1; return *plural* otherwise. - Overridden in derived classes. - - - .. method:: lgettext(message) - .. method:: lngettext(singular, plural, n) - - Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation - is returned as a byte string encoded in the preferred system encoding - if no encoding was explicitly set with :meth:`set_output_charset`. - Overridden in derived classes. - - .. warning:: - - These methods should be avoided in Python 3. See the warning for the - :func:`lgettext` function. - - - .. method:: info() - - Return the "protected" :attr:`_info` variable. - - - .. method:: charset() - - Return the encoding of the message catalog file. - - - .. method:: output_charset() - - Return the encoding used to return translated messages in :meth:`.lgettext` - and :meth:`.lngettext`. - - - .. method:: set_output_charset(charset) - - Change the encoding used to return translated messages. - - - .. method:: install(names=None) - - This method installs :meth:`.gettext` into the built-in namespace, - binding it to ``_``. - - If the *names* parameter is given, it must be a sequence containing the - names of functions you want to install in the builtins namespace in - addition to :func:`_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'lgettext'`` and ``'lngettext'``. - - Note that this is only one way, albeit the most convenient way, to make - the :func:`_` function available to your application. Because it affects - the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`_`. Instead, they should use - this code to make :func:`_` available to their module:: - - import gettext - t = gettext.translation('mymodule', ...) - _ = t.gettext - - This puts :func:`_` only in the module's global namespace and so only - affects calls within this module. - - -The :class:`GNUTranslations` class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :mod:`gettext` module provides one additional class derived from -:class:`NullTranslations`: :class:`GNUTranslations`. This class overrides -:meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files -in both big-endian and little-endian format. - -:class:`GNUTranslations` parses optional meta-data out of the translation -catalog. It is convention with GNU :program:`gettext` to include meta-data as -the translation for the empty string. This meta-data is in :rfc:`822`\ -style -``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the -key ``Content-Type`` is found, then the ``charset`` property is used to -initialize the "protected" :attr:`_charset` instance variable, defaulting to -``None`` if not found. If the charset encoding is specified, then all message -ids and message strings read from the catalog are converted to Unicode using -this encoding, else ASCII encoding is assumed. - -Since message ids are read as Unicode strings too, all :meth:`*gettext` methods -will assume message ids as Unicode strings, not byte strings. - -The entire set of key/value pairs are placed into a dictionary and set as the -"protected" :attr:`_info` instance variable. - -If the :file:`.mo` file's magic number is invalid, the major version number is -unexpected, or if other problems occur while reading the file, instantiating a -:class:`GNUTranslations` class can raise :exc:`OSError`. - -.. class:: GNUTranslations - - The following methods are overridden from the base class implementation: - - .. method:: gettext(message) - - Look up the *message* id in the catalog and return the corresponding message - string, as a Unicode string. If there is no entry in the catalog for the - *message* id, and a fallback has been set, the look up is forwarded to the - fallback's :meth:`~NullTranslations.gettext` method. Otherwise, the - *message* id is returned. - - - .. method:: ngettext(singular, plural, n) - - Do a plural-forms lookup of a message id. *singular* is used as the message id - for purposes of lookup in the catalog, while *n* is used to determine which - plural form to use. The returned message string is a Unicode string. - - If the message id is not found in the catalog, and a fallback is specified, - the request is forwarded to the fallback's :meth:`~NullTranslations.ngettext` - method. Otherwise, when *n* is 1 *singular* is returned, and *plural* is - returned in all other cases. - - Here is an example:: - - n = len(os.listdir('.')) - cat = GNUTranslations(somefile) - message = cat.ngettext( - 'There is %(num)d file in this directory', - 'There are %(num)d files in this directory', - n) % {'num': n} - - - .. method:: lgettext(message) - .. method:: lngettext(singular, plural, n) - - Equivalent to :meth:`.gettext` and :meth:`.ngettext`, but the translation - is returned as a byte string encoded in the preferred system encoding - if no encoding was explicitly set with - :meth:`~NullTranslations.set_output_charset`. - - .. warning:: - - These methods should be avoided in Python 3. See the warning for the - :func:`lgettext` function. - - -Solaris message catalog support -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Solaris operating system defines its own binary :file:`.mo` file format, but -since no documentation can be found on this format, it is not supported at this -time. - - -The Catalog constructor -^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: single: GNOME - -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this -version has a slightly different API. Its documented usage was:: - - import gettext - cat = gettext.Catalog(domain, localedir) - _ = cat.gettext - print(_('hello world')) - -For compatibility with this older module, the function :func:`Catalog` is an -alias for the :func:`translation` function described above. - -One difference between this module and Henstridge's: his catalog objects -supported access through a mapping API, but this appears to be unused and so is -not currently supported. - - -Internationalizing your programs and modules --------------------------------------------- - -Internationalization (I18N) refers to the operation by which a program is made -aware of multiple languages. Localization (L10N) refers to the adaptation of -your program, once internationalized, to the local language and cultural habits. -In order to provide multilingual messages for your Python programs, you need to -take the following steps: - -#. prepare your program or module by specially marking translatable strings - -#. run a suite of tools over your marked files to generate raw messages catalogs - -#. create language specific translations of the message catalogs - -#. use the :mod:`gettext` module so that message strings are properly translated - -In order to prepare your code for I18N, you need to look at all the strings in -your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_`. For example:: - - filename = 'mylog.txt' - message = _('writing a log message') - fp = open(filename, 'w') - fp.write(message) - fp.close() - -In this example, the string ``'writing a log message'`` is marked as a candidate -for translation, while the strings ``'mylog.txt'`` and ``'w'`` are not. - -There are a few tools to extract the strings meant for translation. -The original GNU :program:`gettext` only supported C or C++ source -code but its extended version :program:`xgettext` scans code written -in a number of languages, including Python, to find strings marked as -translatable. `Babel `__ is a Python -internationalization library that includes a :file:`pybabel` script to -extract and compile message catalogs. François Pinard's program -called :program:`xpot` does a similar job and is available as part of -his `po-utils package `__. - -(Python also includes pure-Python versions of these programs, called -:program:`pygettext.py` and :program:`msgfmt.py`; some Python distributions -will install them for you. :program:`pygettext.py` is similar to -:program:`xgettext`, but only understands Python source code and -cannot handle other programming languages such as C or C++. -:program:`pygettext.py` supports a command-line interface similar to -:program:`xgettext`; for details on its use, run ``pygettext.py ---help``. :program:`msgfmt.py` is binary compatible with GNU -:program:`msgfmt`. With these two programs, you may not need the GNU -:program:`gettext` package to internationalize your Python -applications.) - -:program:`xgettext`, :program:`pygettext`, and similar tools generate -:file:`.po` files that are message catalogs. They are structured -human-readable files that contain every marked string in the source -code, along with a placeholder for the translated versions of these -strings. - -Copies of these :file:`.po` files are then handed over to the -individual human translators who write translations for every -supported natural language. They send back the completed -language-specific versions as a :file:`.po` file that's -compiled into a machine-readable :file:`.mo` binary catalog file using -the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at -run-time. - -How you use the :mod:`gettext` module in your code depends on whether you are -internationalizing a single module or your entire application. The next two -sections will discuss each case. - - -Localizing your module -^^^^^^^^^^^^^^^^^^^^^^ - -If you are localizing your module, you must take care not to make global -changes, e.g. to the built-in namespace. You should not use the GNU ``gettext`` -API but instead the class-based API. - -Let's say your module is called "spam" and the module's various natural language -translation :file:`.mo` files reside in :file:`/usr/share/locale` in GNU -:program:`gettext` format. Here's what you would put at the top of your -module:: - - import gettext - t = gettext.translation('spam', '/usr/share/locale') - _ = t.gettext - - -Localizing your application -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are localizing your application, you can install the :func:`_` function -globally into the built-in namespace, usually in the main driver file of your -application. This will let all your application-specific files just use -``_('...')`` without having to explicitly install it in each file. - -In the simple case then, you need only add the following bit of code to the main -driver file of your application:: - - import gettext - gettext.install('myapplication') - -If you need to set the locale directory, you can pass it into the -:func:`install` function:: - - import gettext - gettext.install('myapplication', '/usr/share/locale') - - -Changing languages on the fly -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program needs to support many languages at the same time, you may want -to create multiple translation instances and then switch between them -explicitly, like so:: - - import gettext - - lang1 = gettext.translation('myapplication', languages=['en']) - lang2 = gettext.translation('myapplication', languages=['fr']) - lang3 = gettext.translation('myapplication', languages=['de']) - - # start by using language1 - lang1.install() - - # ... time goes by, user selects language 2 - lang2.install() - - # ... more time goes by, user selects language 3 - lang3.install() - - -Deferred translations -^^^^^^^^^^^^^^^^^^^^^ - -In most coding situations, strings are translated where they are coded. -Occasionally however, you need to mark strings for translation, but defer actual -translation until later. A classic example is:: - - animals = ['mollusk', - 'albatross', - 'rat', - 'penguin', - 'python', ] - # ... - for a in animals: - print(a) - -Here, you want to mark the strings in the ``animals`` list as being -translatable, but you don't actually want to translate them until they are -printed. - -Here is one way you can handle this situation:: - - def _(message): return message - - animals = [_('mollusk'), - _('albatross'), - _('rat'), - _('penguin'), - _('python'), ] - - del _ - - # ... - for a in animals: - print(_(a)) - -This works because the dummy definition of :func:`_` simply returns the string -unchanged. And this dummy definition will temporarily override any definition -of :func:`_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`_` in the local -namespace. - -Note that the second use of :func:`_` will not identify "a" as being -translatable to the :program:`gettext` program, because the parameter -is not a string literal. - -Another way to handle this is with the following example:: - - def N_(message): return message - - animals = [N_('mollusk'), - N_('albatross'), - N_('rat'), - N_('penguin'), - N_('python'), ] - - # ... - for a in animals: - print(_(a)) - -In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. -However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, -:program:`pygettext`, ``pybabel extract``, and :program:`xpot` all -support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. - - -Acknowledgements ----------------- - -The following people contributed code, feedback, design suggestions, previous -implementations, and valuable experience to the creation of this module: - -* Peter Funk - -* James Henstridge - -* Juan David Ibáñez Palomar - -* Marc-André Lemburg - -* Martin von Löwis - -* François Pinard - -* Barry Warsaw - -* Gustavo Niemeyer - -.. rubric:: Footnotes - -.. [#] The default locale directory is system dependent; for example, on RedHat Linux - it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. - The :mod:`gettext` module does not try to support these system dependent - defaults; instead its default is :file:`sys.prefix/share/locale`. For this - reason, it is always best to call :func:`bindtextdomain` with an explicit - absolute path at the start of your application. - -.. [#] See the footnote for :func:`bindtextdomain` above. diff --git a/third_party/python/Doc/library/glob.rst b/third_party/python/Doc/library/glob.rst deleted file mode 100644 index 0db10b5ef..000000000 --- a/third_party/python/Doc/library/glob.rst +++ /dev/null @@ -1,111 +0,0 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== - -.. module:: glob - :synopsis: Unix shell style pathname pattern expansion. - -**Source code:** :source:`Lib/glob.py` - -.. index:: single: filenames; pathname expansion - --------------- - -.. index:: - single: * (asterisk); in glob-style wildcards - single: ? (question mark); in glob-style wildcards - single: [] (square brackets); in glob-style wildcards - single: ! (exclamation); in glob-style wildcards - single: - (minus); in glob-style wildcards - single: . (dot); in glob-style wildcards - -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character -ranges expressed with ``[]`` will be correctly matched. This is done by using -the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and -not by actually invoking a subshell. Note that unlike :func:`fnmatch.fnmatch`, -:mod:`glob` treats filenames beginning with a dot (``.``) as special cases. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) - -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - -.. function:: glob(pathname, *, recursive=False) - - Return a possibly-empty list of path names that match *pathname*, which must be - a string containing a path specification. *pathname* can be either absolute - (like :file:`/usr/src/Python-1.5/Makefile`) or relative (like - :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken - symlinks are included in the results (as in the shell). - - .. index:: - single: **; in glob-style wildcards - - If *recursive* is true, the pattern "``**``" will match any files and zero or - more directories and subdirectories. If the pattern is followed by an - ``os.sep``, only directories and subdirectories match. - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. versionchanged:: 3.5 - Support for recursive globs using "``**``". - - -.. function:: iglob(pathname, *, recursive=False) - - Return an :term:`iterator` which yields the same values as :func:`glob` - without actually storing them all simultaneously. - - -.. function:: escape(pathname) - - Escape all special characters (``'?'``, ``'*'`` and ``'['``). - This is useful if you want to match an arbitrary literal string that may - have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows - ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. - - .. versionadded:: 3.4 - - -For example, consider a directory containing the following files: -:file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub` -which contains only the file :file:`3.txt`. :func:`glob` will produce -the following results. Notice how any leading components of the path are -preserved. :: - - >>> import glob - >>> glob.glob('./[0-9].*') - ['./1.gif', './2.txt'] - >>> glob.glob('*.gif') - ['1.gif', 'card.gif'] - >>> glob.glob('?.gif') - ['1.gif'] - >>> glob.glob('**/*.txt', recursive=True) - ['2.txt', 'sub/3.txt'] - >>> glob.glob('./**/', recursive=True) - ['./', './sub/'] - -If the directory contains files starting with ``.`` they won't be matched by -default. For example, consider a directory containing :file:`card.gif` and -:file:`.card.gif`:: - - >>> import glob - >>> glob.glob('*.gif') - ['card.gif'] - >>> glob.glob('.c*') - ['.card.gif'] - -.. seealso:: - - Module :mod:`fnmatch` - Shell-style filename (not path) expansion - diff --git a/third_party/python/Doc/library/grp.rst b/third_party/python/Doc/library/grp.rst deleted file mode 100644 index 74de3f952..000000000 --- a/third_party/python/Doc/library/grp.rst +++ /dev/null @@ -1,68 +0,0 @@ -:mod:`grp` --- The group database -================================= - -.. module:: grp - :platform: Unix - :synopsis: The group database (getgrnam() and friends). - --------------- - -This module provides access to the Unix group database. It is available on all -Unix versions. - -Group database entries are reported as a tuple-like object, whose attributes -correspond to the members of the ``group`` structure (Attribute field below, see -````): - -+-------+-----------+---------------------------------+ -| Index | Attribute | Meaning | -+=======+===========+=================================+ -| 0 | gr_name | the name of the group | -+-------+-----------+---------------------------------+ -| 1 | gr_passwd | the (encrypted) group password; | -| | | often empty | -+-------+-----------+---------------------------------+ -| 2 | gr_gid | the numerical group ID | -+-------+-----------+---------------------------------+ -| 3 | gr_mem | all the group member's user | -| | | names | -+-------+-----------+---------------------------------+ - -The gid is an integer, name and password are strings, and the member list is a -list of strings. (Note that most users are not explicitly listed as members of -the group they are in according to the password database. Check both databases -to get complete membership information. Also note that a ``gr_name`` that -starts with a ``+`` or ``-`` is likely to be a YP/NIS reference and may not be -accessible via :func:`getgrnam` or :func:`getgrgid`.) - -It defines the following items: - - -.. function:: getgrgid(gid) - - Return the group database entry for the given numeric group ID. :exc:`KeyError` - is raised if the entry asked for cannot be found. - - .. deprecated:: 3.6 - Since Python 3.6 the support of non-integer arguments like floats or - strings in :func:`getgrgid` is deprecated. - -.. function:: getgrnam(name) - - Return the group database entry for the given group name. :exc:`KeyError` is - raised if the entry asked for cannot be found. - - -.. function:: getgrall() - - Return a list of all available group entries, in arbitrary order. - - -.. seealso:: - - Module :mod:`pwd` - An interface to the user database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/third_party/python/Doc/library/gzip.rst b/third_party/python/Doc/library/gzip.rst deleted file mode 100644 index 9c6b72237..000000000 --- a/third_party/python/Doc/library/gzip.rst +++ /dev/null @@ -1,213 +0,0 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= - -.. module:: gzip - :synopsis: Interfaces for gzip compression and decompression using file objects. - -**Source code:** :source:`Lib/gzip.py` - --------------- - -This module provides a simple interface to compress and decompress files just -like the GNU programs :program:`gzip` and :program:`gunzip` would. - -The data compression is provided by the :mod:`zlib` module. - -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the -:func:`.open`, :func:`compress` and :func:`decompress` convenience functions. -The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, -automatically compressing or decompressing the data so that it looks like an -ordinary :term:`file object`. - -Note that additional file formats which can be decompressed by the -:program:`gzip` and :program:`gunzip` programs, such as those produced by -:program:`compress` and :program:`pack`, are not supported by this module. - -The module defines the following items: - - -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) - - Open a gzip-compressed file in binary or text mode, returning a :term:`file - object`. - - The *filename* argument can be an actual filename (a :class:`str` or - :class:`bytes` object), or an existing file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, - ``'w'``, ``'wb'``, ``'x'`` or ``'xb'`` for binary mode, or ``'rt'``, - ``'at'``, ``'wt'``, or ``'xt'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 0 to 9, as for the - :class:`GzipFile` constructor. - - For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`GzipFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.3 - Added support for *filename* being a file object, support for text mode, - and the *encoding*, *errors* and *newline* arguments. - - .. versionchanged:: 3.4 - Added support for the ``'x'``, ``'xb'`` and ``'xt'`` modes. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) - - Constructor for the :class:`GzipFile` class, which simulates most of the - methods of a :term:`file object`, with the exception of the :meth:`truncate` - method. At least one of *fileobj* and *filename* must be given a non-trivial - value. - - The new class instance is based on *fileobj*, which can be a regular file, an - :class:`io.BytesIO` object, or any other object which simulates a file. It - defaults to ``None``, in which case *filename* is opened to provide a file - object. - - When *fileobj* is not ``None``, the *filename* argument is only used to be - included in the :program:`gzip` file header, which may include the original - filename of the uncompressed file. It defaults to the filename of *fileobj*, if - discernible; otherwise, it defaults to the empty string, and in this case the - original filename is not included in the header. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``, - ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or - written. The default is the mode of *fileobj* if discernible; otherwise, the - default is ``'rb'``. - - Note that the file is always opened in binary mode. To open a compressed file - in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an - :class:`io.TextIOWrapper`). - - The *compresslevel* argument is an integer from ``0`` to ``9`` controlling - the level of compression; ``1`` is fastest and produces the least - compression, and ``9`` is slowest and produces the most compression. ``0`` - is no compression. The default is ``9``. - - The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. - - Calling a :class:`GzipFile` object's :meth:`close` method does not close - *fileobj*, since you might wish to append more material after the compressed - data. This also allows you to pass an :class:`io.BytesIO` object opened for - writing as *fileobj*, and retrieve the resulting memory buffer using the - :class:`io.BytesIO` object's :meth:`~io.BytesIO.getvalue` method. - - :class:`GzipFile` supports the :class:`io.BufferedIOBase` interface, - including iteration and the :keyword:`with` statement. Only the - :meth:`truncate` method isn't implemented. - - :class:`GzipFile` also provides the following method and attribute: - - .. method:: peek(n) - - Read *n* uncompressed bytes without advancing the file position. - At most one single read on the compressed stream is done to satisfy - the call. The number of bytes returned may be more or less than - requested. - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`GzipFile`, it may change the position of the underlying - file object (e.g. if the :class:`GzipFile` was constructed with the - *fileobj* parameter). - - .. versionadded:: 3.2 - - .. attribute:: mtime - - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. - - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. - - .. versionchanged:: 3.1 - Support for the :keyword:`with` statement was added, along with the - *mtime* constructor argument and :attr:`mtime` attribute. - - .. versionchanged:: 3.2 - Support for zero-padded and unseekable files was added. - - .. versionchanged:: 3.3 - The :meth:`io.BufferedIOBase.read1` method is now implemented. - - .. versionchanged:: 3.4 - Added support for the ``'x'`` and ``'xb'`` modes. - - .. versionchanged:: 3.5 - Added support for writing arbitrary - :term:`bytes-like objects `. - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: compress(data, compresslevel=9) - - Compress the *data*, returning a :class:`bytes` object containing - the compressed data. *compresslevel* has the same meaning as in - the :class:`GzipFile` constructor above. - - .. versionadded:: 3.2 - -.. function:: decompress(data) - - Decompress the *data*, returning a :class:`bytes` object containing the - uncompressed data. - - .. versionadded:: 3.2 - - -.. _gzip-usage-examples: - -Examples of usage ------------------ - -Example of how to read a compressed file:: - - import gzip - with gzip.open('/home/joe/file.txt.gz', 'rb') as f: - file_content = f.read() - -Example of how to create a compressed GZIP file:: - - import gzip - content = b"Lots of content here" - with gzip.open('/home/joe/file.txt.gz', 'wb') as f: - f.write(content) - -Example of how to GZIP compress an existing file:: - - import gzip - import shutil - with open('/home/joe/file.txt', 'rb') as f_in: - with gzip.open('/home/joe/file.txt.gz', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - -Example of how to GZIP compress a binary string:: - - import gzip - s_in = b"Lots of content here" - s_out = gzip.compress(s_in) - -.. seealso:: - - Module :mod:`zlib` - The basic data compression module needed to support the :program:`gzip` file - format. - diff --git a/third_party/python/Doc/library/hashlib-blake2-tree.png b/third_party/python/Doc/library/hashlib-blake2-tree.png deleted file mode 100644 index 010dcbafe..000000000 Binary files a/third_party/python/Doc/library/hashlib-blake2-tree.png and /dev/null differ diff --git a/third_party/python/Doc/library/hashlib.rst b/third_party/python/Doc/library/hashlib.rst deleted file mode 100644 index b4f517635..000000000 --- a/third_party/python/Doc/library/hashlib.rst +++ /dev/null @@ -1,739 +0,0 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== - -.. module:: hashlib - :synopsis: Secure hash and message digest algorithms. - -.. moduleauthor:: Gregory P. Smith -.. sectionauthor:: Gregory P. Smith - -**Source code:** :source:`Lib/hashlib.py` - -.. index:: - single: message digest, MD5 - single: secure hash algorithm, SHA1, SHA224, SHA256, SHA384, SHA512 - -.. testsetup:: - - import hashlib - - --------------- - -This module implements a common interface to many different secure hash and -message digest algorithms. Included are the FIPS secure hash algorithms SHA1, -SHA224, SHA256, SHA384, and SHA512 (defined in FIPS 180-2) as well as RSA's MD5 -algorithm (defined in Internet :rfc:`1321`). The terms "secure hash" and -"message digest" are interchangeable. Older algorithms were called message -digests. The modern term is secure hash. - -.. note:: - - If you want the adler32 or crc32 hash functions, they are available in - the :mod:`zlib` module. - -.. warning:: - - Some algorithms have known hash collision weaknesses, refer to the "See - also" section at the end. - - -.. _hash-algorithms: - -Hash algorithms ---------------- - -There is one constructor method named for each type of :dfn:`hash`. All return -a hash object with the same simple interface. For example: use :func:`sha256` to -create a SHA-256 hash object. You can now feed this object with :term:`bytes-like -objects ` (normally :class:`bytes`) using the :meth:`update` method. -At any point you can ask it for the :dfn:`digest` of the -concatenation of the data fed to it so far using the :meth:`digest` or -:meth:`hexdigest` methods. - -.. note:: - - For better multithreading performance, the Python :term:`GIL` is released for - data larger than 2047 bytes at object creation or on update. - -.. note:: - - Feeding string objects into :meth:`update` is not supported, as hashes work - on bytes, not on characters. - -.. index:: single: OpenSSL; (use in module hashlib) - -Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, -:func:`sha512`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it -may be missing if you are using a rare "FIPS compliant" build of Python. -Additional algorithms may also be available depending upon the OpenSSL -library that Python uses on your platform. On most platforms the -:func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256` are also available. - -.. versionadded:: 3.6 - SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, - :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256`. - -.. versionadded:: 3.6 - :func:`blake2b` and :func:`blake2s` were added. - -For example, to obtain the digest of the byte string ``b'Nobody inspects the -spammish repetition'``:: - - >>> import hashlib - >>> m = hashlib.sha256() - >>> m.update(b"Nobody inspects") - >>> m.update(b" the spammish repetition") - >>> m.digest() - b'\x03\x1e\xdd}Ae\x15\x93\xc5\xfe\\\x00o\xa5u+7\xfd\xdf\xf7\xbcN\x84:\xa6\xaf\x0c\x95\x0fK\x94\x06' - >>> m.digest_size - 32 - >>> m.block_size - 64 - -More condensed: - - >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() - 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' - -.. function:: new(name[, data]) - - Is a generic constructor that takes the string *name* of the desired - algorithm as its first parameter. It also exists to allow access to the - above listed hashes as well as any other algorithms that your OpenSSL - library may offer. The named constructors are much faster than :func:`new` - and should be preferred. - -Using :func:`new` with an algorithm provided by OpenSSL: - - >>> h = hashlib.new('ripemd160') - >>> h.update(b"Nobody inspects the spammish repetition") - >>> h.hexdigest() - 'cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc' - -Hashlib provides the following constant attributes: - -.. data:: algorithms_guaranteed - - A set containing the names of the hash algorithms guaranteed to be supported - by this module on all platforms. Note that 'md5' is in this list despite - some upstream vendors offering an odd "FIPS compliant" Python build that - excludes it. - - .. versionadded:: 3.2 - -.. data:: algorithms_available - - A set containing the names of the hash algorithms that are available in the - running Python interpreter. These names will be recognized when passed to - :func:`new`. :attr:`algorithms_guaranteed` will always be a subset. The - same algorithm may appear multiple times in this set under different names - (thanks to OpenSSL). - - .. versionadded:: 3.2 - -The following values are provided as constant attributes of the hash objects -returned by the constructors: - - -.. data:: hash.digest_size - - The size of the resulting hash in bytes. - -.. data:: hash.block_size - - The internal block size of the hash algorithm in bytes. - -A hash object has the following attributes: - -.. attribute:: hash.name - - The canonical name of this hash, always lowercase and always suitable as a - parameter to :func:`new` to create another hash of this type. - - .. versionchanged:: 3.4 - The name attribute has been present in CPython since its inception, but - until Python 3.4 was not formally specified, so may not exist on some - platforms. - -A hash object has the following methods: - - -.. method:: hash.update(data) - - Update the hash object with the :term:`bytes-like object`. - Repeated calls are equivalent to a single call with the - concatenation of all the arguments: ``m.update(a); m.update(b)`` is - equivalent to ``m.update(a+b)``. - - .. versionchanged:: 3.1 - The Python GIL is released to allow other threads to run while hash - updates on data larger than 2047 bytes is taking place when using hash - algorithms supplied by OpenSSL. - - -.. method:: hash.digest() - - Return the digest of the data passed to the :meth:`update` method so far. - This is a bytes object of size :attr:`digest_size` which may contain bytes in - the whole range from 0 to 255. - - -.. method:: hash.hexdigest() - - Like :meth:`digest` except the digest is returned as a string object of - double length, containing only hexadecimal digits. This may be used to - exchange the value safely in email or other non-binary environments. - - -.. method:: hash.copy() - - Return a copy ("clone") of the hash object. This can be used to efficiently - compute the digests of data sharing a common initial substring. - - -SHAKE variable length digests ------------------------------ - -The :func:`shake_128` and :func:`shake_256` algorithms provide variable -length digests with length_in_bits//2 up to 128 or 256 bits of security. -As such, their digest methods require a length. Maximum length is not limited -by the SHAKE algorithm. - -.. method:: shake.digest(length) - - Return the digest of the data passed to the :meth:`update` method so far. - This is a bytes object of size *length* which may contain bytes in - the whole range from 0 to 255. - - -.. method:: shake.hexdigest(length) - - Like :meth:`digest` except the digest is returned as a string object of - double length, containing only hexadecimal digits. This may be used to - exchange the value safely in email or other non-binary environments. - - -Key derivation --------------- - -Key derivation and key stretching algorithms are designed for secure password -hashing. Naive algorithms such as ``sha1(password)`` are not resistant against -brute-force attacks. A good password hashing function must be tunable, slow, and -include a `salt `_. - - -.. function:: pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None) - - The function provides PKCS#5 password-based key derivation function 2. It - uses HMAC as pseudorandom function. - - The string *hash_name* is the desired name of the hash digest algorithm for - HMAC, e.g. 'sha1' or 'sha256'. *password* and *salt* are interpreted as - buffers of bytes. Applications and libraries should limit *password* to - a sensible length (e.g. 1024). *salt* should be about 16 or more bytes from - a proper source, e.g. :func:`os.urandom`. - - The number of *iterations* should be chosen based on the hash algorithm and - computing power. As of 2013, at least 100,000 iterations of SHA-256 are - suggested. - - *dklen* is the length of the derived key. If *dklen* is ``None`` then the - digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. - - >>> import hashlib, binascii - >>> dk = hashlib.pbkdf2_hmac('sha256', b'password', b'salt', 100000) - >>> binascii.hexlify(dk) - b'0394a2ede332c9a13eb82e9b24631604c31df978b4e2f0fbd2c549944f9d79a5' - - .. versionadded:: 3.4 - - .. note:: - - A fast implementation of *pbkdf2_hmac* is available with OpenSSL. The - Python implementation uses an inline version of :mod:`hmac`. It is about - three times slower and doesn't release the GIL. - -.. function:: scrypt(password, *, salt, n, r, p, maxmem=0, dklen=64) - - The function provides scrypt password-based key derivation function as - defined in :rfc:`7914`. - - *password* and *salt* must be :term:`bytes-like objects - `. Applications and libraries should limit *password* - to a sensible length (e.g. 1024). *salt* should be about 16 or more - bytes from a proper source, e.g. :func:`os.urandom`. - - *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization - factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MB). - *dklen* is the length of the derived key. - - Availability: OpenSSL 1.1+ - - .. versionadded:: 3.6 - - -BLAKE2 ------- - -.. sectionauthor:: Dmitry Chestnykh - -.. index:: - single: blake2b, blake2s - -BLAKE2_ is a cryptographic hash function defined in :rfc:`7693` that comes in two -flavors: - -* **BLAKE2b**, optimized for 64-bit platforms and produces digests of any size - between 1 and 64 bytes, - -* **BLAKE2s**, optimized for 8- to 32-bit platforms and produces digests of any - size between 1 and 32 bytes. - -BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), -**salted hashing**, **personalization**, and **tree hashing**. - -Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. - - -Creating hash objects -^^^^^^^^^^^^^^^^^^^^^ - -New hash objects are created by calling constructor functions: - - -.. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \ - person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False) - -.. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \ - person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False) - - -These functions return the corresponding hash objects for calculating -BLAKE2b or BLAKE2s. They optionally take these general parameters: - -* *data*: initial chunk of data to hash, which must be - :term:`bytes-like object`. It can be passed only as positional argument. - -* *digest_size*: size of output digest in bytes. - -* *key*: key for keyed hashing (up to 64 bytes for BLAKE2b, up to 32 bytes for - BLAKE2s). - -* *salt*: salt for randomized hashing (up to 16 bytes for BLAKE2b, up to 8 - bytes for BLAKE2s). - -* *person*: personalization string (up to 16 bytes for BLAKE2b, up to 8 bytes - for BLAKE2s). - -The following table shows limits for general parameters (in bytes): - -======= =========== ======== ========= =========== -Hash digest_size len(key) len(salt) len(person) -======= =========== ======== ========= =========== -BLAKE2b 64 64 16 16 -BLAKE2s 32 32 8 8 -======= =========== ======== ========= =========== - -.. note:: - - BLAKE2 specification defines constant lengths for salt and personalization - parameters, however, for convenience, this implementation accepts byte - strings of any size up to the specified length. If the length of the - parameter is less than specified, it is padded with zeros, thus, for - example, ``b'salt'`` and ``b'salt\x00'`` is the same value. (This is not - the case for *key*.) - -These sizes are available as module `constants`_ described below. - -Constructor functions also accept the following tree hashing parameters: - -* *fanout*: fanout (0 to 255, 0 if unlimited, 1 in sequential mode). - -* *depth*: maximal depth of tree (1 to 255, 255 if unlimited, 1 in - sequential mode). - -* *leaf_size*: maximal byte length of leaf (0 to 2**32-1, 0 if unlimited or in - sequential mode). - -* *node_offset*: node offset (0 to 2**64-1 for BLAKE2b, 0 to 2**48-1 for - BLAKE2s, 0 for the first, leftmost, leaf, or in sequential mode). - -* *node_depth*: node depth (0 to 255, 0 for leaves, or in sequential mode). - -* *inner_size*: inner digest size (0 to 64 for BLAKE2b, 0 to 32 for - BLAKE2s, 0 in sequential mode). - -* *last_node*: boolean indicating whether the processed node is the last - one (`False` for sequential mode). - -.. figure:: hashlib-blake2-tree.png - :alt: Explanation of tree mode parameters. - -See section 2.10 in `BLAKE2 specification -`_ for comprehensive review of tree -hashing. - - -Constants -^^^^^^^^^ - -.. data:: blake2b.SALT_SIZE -.. data:: blake2s.SALT_SIZE - -Salt length (maximum length accepted by constructors). - - -.. data:: blake2b.PERSON_SIZE -.. data:: blake2s.PERSON_SIZE - -Personalization string length (maximum length accepted by constructors). - - -.. data:: blake2b.MAX_KEY_SIZE -.. data:: blake2s.MAX_KEY_SIZE - -Maximum key size. - - -.. data:: blake2b.MAX_DIGEST_SIZE -.. data:: blake2s.MAX_DIGEST_SIZE - -Maximum digest size that the hash function can output. - - -Examples -^^^^^^^^ - -Simple hashing -"""""""""""""" - -To calculate hash of some data, you should first construct a hash object by -calling the appropriate constructor function (:func:`blake2b` or -:func:`blake2s`), then update it with the data by calling :meth:`update` on the -object, and, finally, get the digest out of the object by calling -:meth:`digest` (or :meth:`hexdigest` for hex-encoded string). - - >>> from hashlib import blake2b - >>> h = blake2b() - >>> h.update(b'Hello world') - >>> h.hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - - -As a shortcut, you can pass the first chunk of data to update directly to the -constructor as the positional argument: - - >>> from hashlib import blake2b - >>> blake2b(b'Hello world').hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - -You can call :meth:`hash.update` as many times as you need to iteratively -update the hash: - - >>> from hashlib import blake2b - >>> items = [b'Hello', b' ', b'world'] - >>> h = blake2b() - >>> for item in items: - ... h.update(item) - >>> h.hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - - -Using different digest sizes -"""""""""""""""""""""""""""" - -BLAKE2 has configurable size of digests up to 64 bytes for BLAKE2b and up to 32 -bytes for BLAKE2s. For example, to replace SHA-1 with BLAKE2b without changing -the size of output, we can tell BLAKE2b to produce 20-byte digests: - - >>> from hashlib import blake2b - >>> h = blake2b(digest_size=20) - >>> h.update(b'Replacing SHA1 with the more secure function') - >>> h.hexdigest() - 'd24f26cf8de66472d58d4e1b1774b4c9158b1f4c' - >>> h.digest_size - 20 - >>> len(h.digest()) - 20 - -Hash objects with different digest sizes have completely different outputs -(shorter hashes are *not* prefixes of longer hashes); BLAKE2b and BLAKE2s -produce different outputs even if the output length is the same: - - >>> from hashlib import blake2b, blake2s - >>> blake2b(digest_size=10).hexdigest() - '6fa1d8fcfd719046d762' - >>> blake2b(digest_size=11).hexdigest() - 'eb6ec15daf9546254f0809' - >>> blake2s(digest_size=10).hexdigest() - '1bf21a98c78a1c376ae9' - >>> blake2s(digest_size=11).hexdigest() - '567004bf96e4a25773ebf4' - - -Keyed hashing -""""""""""""" - -Keyed hashing can be used for authentication as a faster and simpler -replacement for `Hash-based message authentication code -`_ (HMAC). -BLAKE2 can be securely used in prefix-MAC mode thanks to the -indifferentiability property inherited from BLAKE. - -This example shows how to get a (hex-encoded) 128-bit authentication code for -message ``b'message data'`` with key ``b'pseudorandom key'``:: - - >>> from hashlib import blake2b - >>> h = blake2b(key=b'pseudorandom key', digest_size=16) - >>> h.update(b'message data') - >>> h.hexdigest() - '3d363ff7401e02026f4a4687d4863ced' - - -As a practical example, a web application can symmetrically sign cookies sent -to users and later verify them to make sure they weren't tampered with:: - - >>> from hashlib import blake2b - >>> from hmac import compare_digest - >>> - >>> SECRET_KEY = b'pseudorandomly generated server secret key' - >>> AUTH_SIZE = 16 - >>> - >>> def sign(cookie): - ... h = blake2b(digest_size=AUTH_SIZE, key=SECRET_KEY) - ... h.update(cookie) - ... return h.hexdigest().encode('utf-8') - >>> - >>> def verify(cookie, sig): - ... good_sig = sign(cookie) - ... return compare_digest(good_sig, sig) - >>> - >>> cookie = b'user-alice' - >>> sig = sign(cookie) - >>> print("{0},{1}".format(cookie.decode('utf-8'), sig)) - user-alice,b'43b3c982cf697e0c5ab22172d1ca7421' - >>> verify(cookie, sig) - True - >>> verify(b'user-bob', sig) - False - >>> verify(cookie, b'0102030405060708090a0b0c0d0e0f00') - False - -Even though there's a native keyed hashing mode, BLAKE2 can, of course, be used -in HMAC construction with :mod:`hmac` module:: - - >>> import hmac, hashlib - >>> m = hmac.new(b'secret key', digestmod=hashlib.blake2s) - >>> m.update(b'message') - >>> m.hexdigest() - 'e3c8102868d28b5ff85fc35dda07329970d1a01e273c37481326fe0c861c8142' - - -Randomized hashing -"""""""""""""""""" - -By setting *salt* parameter users can introduce randomization to the hash -function. Randomized hashing is useful for protecting against collision attacks -on the hash function used in digital signatures. - - Randomized hashing is designed for situations where one party, the message - preparer, generates all or part of a message to be signed by a second - party, the message signer. If the message preparer is able to find - cryptographic hash function collisions (i.e., two messages producing the - same hash value), then they might prepare meaningful versions of the message - that would produce the same hash value and digital signature, but with - different results (e.g., transferring $1,000,000 to an account, rather than - $10). Cryptographic hash functions have been designed with collision - resistance as a major goal, but the current concentration on attacking - cryptographic hash functions may result in a given cryptographic hash - function providing less collision resistance than expected. Randomized - hashing offers the signer additional protection by reducing the likelihood - that a preparer can generate two or more messages that ultimately yield the - same hash value during the digital signature generation process --- even if - it is practical to find collisions for the hash function. However, the use - of randomized hashing may reduce the amount of security provided by a - digital signature when all portions of the message are prepared - by the signer. - - (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) - -In BLAKE2 the salt is processed as a one-time input to the hash function during -initialization, rather than as an input to each compression function. - -.. warning:: - - *Salted hashing* (or just hashing) with BLAKE2 or any other general-purpose - cryptographic hash function, such as SHA-256, is not suitable for hashing - passwords. See `BLAKE2 FAQ `_ for more - information. -.. - - >>> import os - >>> from hashlib import blake2b - >>> msg = b'some message' - >>> # Calculate the first hash with a random salt. - >>> salt1 = os.urandom(blake2b.SALT_SIZE) - >>> h1 = blake2b(salt=salt1) - >>> h1.update(msg) - >>> # Calculate the second hash with a different random salt. - >>> salt2 = os.urandom(blake2b.SALT_SIZE) - >>> h2 = blake2b(salt=salt2) - >>> h2.update(msg) - >>> # The digests are different. - >>> h1.digest() != h2.digest() - True - - -Personalization -""""""""""""""" - -Sometimes it is useful to force hash function to produce different digests for -the same input for different purposes. Quoting the authors of the Skein hash -function: - - We recommend that all application designers seriously consider doing this; - we have seen many protocols where a hash that is computed in one part of - the protocol can be used in an entirely different part because two hash - computations were done on similar or related data, and the attacker can - force the application to make the hash inputs the same. Personalizing each - hash function used in the protocol summarily stops this type of attack. - - (`The Skein Hash Function Family - `_, - p. 21) - -BLAKE2 can be personalized by passing bytes to the *person* argument:: - - >>> from hashlib import blake2b - >>> FILES_HASH_PERSON = b'MyApp Files Hash' - >>> BLOCK_HASH_PERSON = b'MyApp Block Hash' - >>> h = blake2b(digest_size=32, person=FILES_HASH_PERSON) - >>> h.update(b'the same content') - >>> h.hexdigest() - '20d9cd024d4fb086aae819a1432dd2466de12947831b75c5a30cf2676095d3b4' - >>> h = blake2b(digest_size=32, person=BLOCK_HASH_PERSON) - >>> h.update(b'the same content') - >>> h.hexdigest() - 'cf68fb5761b9c44e7878bfb2c4c9aea52264a80b75005e65619778de59f383a3' - -Personalization together with the keyed mode can also be used to derive different -keys from a single one. - - >>> from hashlib import blake2s - >>> from base64 import b64decode, b64encode - >>> orig_key = b64decode(b'Rm5EPJai72qcK3RGBpW3vPNfZy5OZothY+kHY6h21KM=') - >>> enc_key = blake2s(key=orig_key, person=b'kEncrypt').digest() - >>> mac_key = blake2s(key=orig_key, person=b'kMAC').digest() - >>> print(b64encode(enc_key).decode('utf-8')) - rbPb15S/Z9t+agffno5wuhB77VbRi6F9Iv2qIxU7WHw= - >>> print(b64encode(mac_key).decode('utf-8')) - G9GtHFE1YluXY1zWPlYk1e/nWfu0WSEb0KRcjhDeP/o= - -Tree mode -""""""""" - -Here's an example of hashing a minimal tree with two leaf nodes:: - - 10 - / \ - 00 01 - -This example uses 64-byte internal digests, and returns the 32-byte final -digest:: - - >>> from hashlib import blake2b - >>> - >>> FANOUT = 2 - >>> DEPTH = 2 - >>> LEAF_SIZE = 4096 - >>> INNER_SIZE = 64 - >>> - >>> buf = bytearray(6000) - >>> - >>> # Left leaf - ... h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=0, node_depth=0, last_node=False) - >>> # Right leaf - ... h01 = blake2b(buf[LEAF_SIZE:], fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=1, node_depth=0, last_node=True) - >>> # Root node - ... h10 = blake2b(digest_size=32, fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=0, node_depth=1, last_node=True) - >>> h10.update(h00.digest()) - >>> h10.update(h01.digest()) - >>> h10.hexdigest() - '3ad2a9b37c6070e374c7a8c508fe20ca86b6ed54e286e93a0318e95e881db5aa' - -Credits -^^^^^^^ - -BLAKE2_ was designed by *Jean-Philippe Aumasson*, *Samuel Neves*, *Zooko -Wilcox-O'Hearn*, and *Christian Winnerlein* based on SHA-3_ finalist BLAKE_ -created by *Jean-Philippe Aumasson*, *Luca Henzen*, *Willi Meier*, and -*Raphael C.-W. Phan*. - -It uses core algorithm from ChaCha_ cipher designed by *Daniel J. Bernstein*. - -The stdlib implementation is based on pyblake2_ module. It was written by -*Dmitry Chestnykh* based on C implementation written by *Samuel Neves*. The -documentation was copied from pyblake2_ and written by *Dmitry Chestnykh*. - -The C code was partly rewritten for Python by *Christian Heimes*. - -The following public domain dedication applies for both C hash function -implementation, extension code, and this documentation: - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along - with this software. If not, see - http://creativecommons.org/publicdomain/zero/1.0/. - -The following people have helped with development or contributed their changes -to the project and the public domain according to the Creative Commons Public -Domain Dedication 1.0 Universal: - -* *Alexandr Sokolovskiy* - -.. _BLAKE2: https://blake2.net -.. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code -.. _BLAKE: https://131002.net/blake/ -.. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition -.. _ChaCha: https://cr.yp.to/chacha.html -.. _pyblake2: https://pythonhosted.org/pyblake2/ - - - -.. seealso:: - - Module :mod:`hmac` - A module to generate message authentication codes using hashes. - - Module :mod:`base64` - Another way to encode binary hashes for non-binary environments. - - https://blake2.net - Official BLAKE2 website. - - http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf - The FIPS 180-2 publication on Secure Hash Algorithms. - - https://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms - Wikipedia article with information on which algorithms have known issues and - what that means regarding their use. - - https://www.ietf.org/rfc/rfc2898.txt - PKCS #5: Password-Based Cryptography Specification Version 2.0 diff --git a/third_party/python/Doc/library/heapq.rst b/third_party/python/Doc/library/heapq.rst deleted file mode 100644 index d04033b1f..000000000 --- a/third_party/python/Doc/library/heapq.rst +++ /dev/null @@ -1,309 +0,0 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== - -.. module:: heapq - :synopsis: Heap queue algorithm (a.k.a. priority queue). - -.. moduleauthor:: Kevin O'Connor -.. sectionauthor:: Guido van Rossum -.. sectionauthor:: François Pinard -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/heapq.py` - --------------- - -This module provides an implementation of the heap queue algorithm, also known -as the priority queue algorithm. - -Heaps are binary trees for which every parent node has a value less than or -equal to any of its children. This implementation uses arrays for which -``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]`` for all *k*, counting -elements from zero. For the sake of comparison, non-existing elements are -considered to be infinite. The interesting property of a heap is that its -smallest element is always the root, ``heap[0]``. - -The API below differs from textbook heap algorithms in two aspects: (a) We use -zero-based indexing. This makes the relationship between the index for a node -and the indexes for its children slightly less obvious, but is more suitable -since Python uses zero-based indexing. (b) Our pop method returns the smallest -item, not the largest (called a "min heap" in textbooks; a "max heap" is more -common in texts because of its suitability for in-place sorting). - -These two make it possible to view the heap as a regular Python list without -surprises: ``heap[0]`` is the smallest item, and ``heap.sort()`` maintains the -heap invariant! - -To create a heap, use a list initialized to ``[]``, or you can transform a -populated list into a heap via function :func:`heapify`. - -The following functions are provided: - - -.. function:: heappush(heap, item) - - Push the value *item* onto the *heap*, maintaining the heap invariant. - - -.. function:: heappop(heap) - - Pop and return the smallest item from the *heap*, maintaining the heap - invariant. If the heap is empty, :exc:`IndexError` is raised. To access the - smallest item without popping it, use ``heap[0]``. - - -.. function:: heappushpop(heap, item) - - Push *item* on the heap, then pop and return the smallest item from the - *heap*. The combined action runs more efficiently than :func:`heappush` - followed by a separate call to :func:`heappop`. - - -.. function:: heapify(x) - - Transform list *x* into a heap, in-place, in linear time. - - -.. function:: heapreplace(heap, item) - - Pop and return the smallest item from the *heap*, and also push the new *item*. - The heap size doesn't change. If the heap is empty, :exc:`IndexError` is raised. - - This one step operation is more efficient than a :func:`heappop` followed by - :func:`heappush` and can be more appropriate when using a fixed-size heap. - The pop/push combination always returns an element from the heap and replaces - it with *item*. - - The value returned may be larger than the *item* added. If that isn't - desired, consider using :func:`heappushpop` instead. Its push/pop - combination returns the smaller of the two values, leaving the larger value - on the heap. - - -The module also offers three general purpose functions based on heaps. - - -.. function:: merge(*iterables, key=None, reverse=False) - - Merge multiple sorted inputs into a single sorted output (for example, merge - timestamped entries from multiple log files). Returns an :term:`iterator` - over the sorted values. - - Similar to ``sorted(itertools.chain(*iterables))`` but returns an iterable, does - not pull the data into memory all at once, and assumes that each of the input - streams is already sorted (smallest to largest). - - Has two optional arguments which must be specified as keyword arguments. - - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). - - *reverse* is a boolean value. If set to ``True``, then the input elements - are merged as if each comparison were reversed. - - .. versionchanged:: 3.5 - Added the optional *key* and *reverse* parameters. - - -.. function:: nlargest(n, iterable, key=None) - - Return a list with the *n* largest elements from the dataset defined by - *iterable*. *key*, if provided, specifies a function of one argument that is - used to extract a comparison key from each element in *iterable* (for example, - ``key=str.lower``). Equivalent to: ``sorted(iterable, key=key, - reverse=True)[:n]``. - - -.. function:: nsmallest(n, iterable, key=None) - - Return a list with the *n* smallest elements from the dataset defined by - *iterable*. *key*, if provided, specifies a function of one argument that is - used to extract a comparison key from each element in *iterable* (for example, - ``key=str.lower``). Equivalent to: ``sorted(iterable, key=key)[:n]``. - - -The latter two functions perform best for smaller values of *n*. For larger -values, it is more efficient to use the :func:`sorted` function. Also, when -``n==1``, it is more efficient to use the built-in :func:`min` and :func:`max` -functions. If repeated usage of these functions is required, consider turning -the iterable into an actual heap. - - -Basic Examples --------------- - -A `heapsort `_ can be implemented by -pushing all values onto a heap and then popping off the smallest values one at a -time:: - - >>> def heapsort(iterable): - ... h = [] - ... for value in iterable: - ... heappush(h, value) - ... return [heappop(h) for i in range(len(h))] - ... - >>> heapsort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0]) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -This is similar to ``sorted(iterable)``, but unlike :func:`sorted`, this -implementation is not stable. - -Heap elements can be tuples. This is useful for assigning comparison values -(such as task priorities) alongside the main record being tracked:: - - >>> h = [] - >>> heappush(h, (5, 'write code')) - >>> heappush(h, (7, 'release product')) - >>> heappush(h, (1, 'write spec')) - >>> heappush(h, (3, 'create tests')) - >>> heappop(h) - (1, 'write spec') - - -Priority Queue Implementation Notes ------------------------------------ - -A `priority queue `_ is common use -for a heap, and it presents several implementation challenges: - -* Sort stability: how do you get two tasks with equal priorities to be returned - in the order they were originally added? - -* Tuple comparison breaks for (priority, task) pairs if the priorities are equal - and the tasks do not have a default comparison order. - -* If the priority of a task changes, how do you move it to a new position in - the heap? - -* Or if a pending task needs to be deleted, how do you find it and remove it - from the queue? - -A solution to the first two challenges is to store entries as 3-element list -including the priority, an entry count, and the task. The entry count serves as -a tie-breaker so that two tasks with the same priority are returned in the order -they were added. And since no two entry counts are the same, the tuple -comparison will never attempt to directly compare two tasks. - -The remaining challenges revolve around finding a pending task and making -changes to its priority or removing it entirely. Finding a task can be done -with a dictionary pointing to an entry in the queue. - -Removing the entry or changing its priority is more difficult because it would -break the heap structure invariants. So, a possible solution is to mark the -entry as removed and add a new entry with the revised priority:: - - pq = [] # list of entries arranged in a heap - entry_finder = {} # mapping of tasks to entries - REMOVED = '' # placeholder for a removed task - counter = itertools.count() # unique sequence count - - def add_task(task, priority=0): - 'Add a new task or update the priority of an existing task' - if task in entry_finder: - remove_task(task) - count = next(counter) - entry = [priority, count, task] - entry_finder[task] = entry - heappush(pq, entry) - - def remove_task(task): - 'Mark an existing task as REMOVED. Raise KeyError if not found.' - entry = entry_finder.pop(task) - entry[-1] = REMOVED - - def pop_task(): - 'Remove and return the lowest priority task. Raise KeyError if empty.' - while pq: - priority, count, task = heappop(pq) - if task is not REMOVED: - del entry_finder[task] - return task - raise KeyError('pop from an empty priority queue') - - -Theory ------- - -Heaps are arrays for which ``a[k] <= a[2*k+1]`` and ``a[k] <= a[2*k+2]`` for all -*k*, counting elements from 0. For the sake of comparison, non-existing -elements are considered to be infinite. The interesting property of a heap is -that ``a[0]`` is always its smallest element. - -The strange invariant above is meant to be an efficient memory representation -for a tournament. The numbers below are *k*, not ``a[k]``:: - - 0 - - 1 2 - - 3 4 5 6 - - 7 8 9 10 11 12 13 14 - - 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 - -In the tree above, each cell *k* is topping ``2*k+1`` and ``2*k+2``. In a usual -binary tournament we see in sports, each cell is the winner over the two cells -it tops, and we can trace the winner down the tree to see all opponents s/he -had. However, in many computer applications of such tournaments, we do not need -to trace the history of a winner. To be more memory efficient, when a winner is -promoted, we try to replace it by something else at a lower level, and the rule -becomes that a cell and the two cells it tops contain three different items, but -the top cell "wins" over the two topped cells. - -If this heap invariant is protected at all time, index 0 is clearly the overall -winner. The simplest algorithmic way to remove it and find the "next" winner is -to move some loser (let's say cell 30 in the diagram above) into the 0 position, -and then percolate this new 0 down the tree, exchanging values, until the -invariant is re-established. This is clearly logarithmic on the total number of -items in the tree. By iterating over all items, you get an O(n log n) sort. - -A nice feature of this sort is that you can efficiently insert new items while -the sort is going on, provided that the inserted items are not "better" than the -last 0'th element you extracted. This is especially useful in simulation -contexts, where the tree holds all incoming events, and the "win" condition -means the smallest scheduled time. When an event schedules other events for -execution, they are scheduled into the future, so they can easily go into the -heap. So, a heap is a good structure for implementing schedulers (this is what -I used for my MIDI sequencer :-). - -Various structures for implementing schedulers have been extensively studied, -and heaps are good for this, as they are reasonably speedy, the speed is almost -constant, and the worst case is not much different than the average case. -However, there are other representations which are more efficient overall, yet -the worst cases might be terrible. - -Heaps are also very useful in big disk sorts. You most probably all know that a -big sort implies producing "runs" (which are pre-sorted sequences, whose size is -usually related to the amount of CPU memory), followed by a merging passes for -these runs, which merging is often very cleverly organised [#]_. It is very -important that the initial sort produces the longest runs possible. Tournaments -are a good way to achieve that. If, using all the memory available to hold a -tournament, you replace and percolate items that happen to fit the current run, -you'll produce runs which are twice the size of the memory for random input, and -much better for input fuzzily ordered. - -Moreover, if you output the 0'th item on disk and get an input which may not fit -in the current tournament (because the value "wins" over the last output value), -it cannot fit in the heap, so the size of the heap decreases. The freed memory -could be cleverly reused immediately for progressively building a second heap, -which grows at exactly the same rate the first heap is melting. When the first -heap completely vanishes, you switch heaps and start a new run. Clever and -quite effective! - -In a word, heaps are useful memory structures to know. I use them in a few -applications, and I think it is good to keep a 'heap' module around. :-) - -.. rubric:: Footnotes - -.. [#] The disk balancing algorithms which are current, nowadays, are more annoying - than clever, and this is a consequence of the seeking capabilities of the disks. - On devices which cannot seek, like big tape drives, the story was quite - different, and one had to be very clever to ensure (far in advance) that each - tape movement will be the most effective possible (that is, will best - participate at "progressing" the merge). Some tapes were even able to read - backwards, and this was also used to avoid the rewinding time. Believe me, real - good tape sorts were quite spectacular to watch! From all times, sorting has - always been a Great Art! :-) - diff --git a/third_party/python/Doc/library/hmac.rst b/third_party/python/Doc/library/hmac.rst deleted file mode 100644 index adbf78a7b..000000000 --- a/third_party/python/Doc/library/hmac.rst +++ /dev/null @@ -1,123 +0,0 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== - -.. module:: hmac - :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation - -.. moduleauthor:: Gerhard Häring -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/hmac.py` - --------------- - -This module implements the HMAC algorithm as described by :rfc:`2104`. - - -.. function:: new(key, msg=None, digestmod=None) - - Return a new hmac object. *key* is a bytes or bytearray object giving the - secret key. If *msg* is present, the method call ``update(msg)`` is made. - *digestmod* is the digest name, digest constructor or module for the HMAC - object to use. It supports any name suitable to :func:`hashlib.new` and - defaults to the :data:`hashlib.md5` constructor. - - .. versionchanged:: 3.4 - Parameter *key* can be a bytes or bytearray object. - Parameter *msg* can be of any type supported by :mod:`hashlib`. - Parameter *digestmod* can be the name of a hash algorithm. - - .. deprecated:: 3.4 - MD5 as implicit default digest for *digestmod* is deprecated. - - -An HMAC object has the following methods: - -.. method:: HMAC.update(msg) - - Update the hmac object with *msg*. Repeated calls are equivalent to a - single call with the concatenation of all the arguments: - ``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``. - - .. versionchanged:: 3.4 - Parameter *msg* can be of any type supported by :mod:`hashlib`. - - -.. method:: HMAC.digest() - - Return the digest of the bytes passed to the :meth:`update` method so far. - This bytes object will be the same length as the *digest_size* of the digest - given to the constructor. It may contain non-ASCII bytes, including NUL - bytes. - - .. warning:: - - When comparing the output of :meth:`digest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - - -.. method:: HMAC.hexdigest() - - Like :meth:`digest` except the digest is returned as a string twice the - length containing only hexadecimal digits. This may be used to exchange the - value safely in email or other non-binary environments. - - .. warning:: - - When comparing the output of :meth:`hexdigest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - - -.. method:: HMAC.copy() - - Return a copy ("clone") of the hmac object. This can be used to efficiently - compute the digests of strings that share a common initial substring. - - -A hash object has the following attributes: - -.. attribute:: HMAC.digest_size - - The size of the resulting HMAC digest in bytes. - -.. attribute:: HMAC.block_size - - The internal block size of the hash algorithm in bytes. - - .. versionadded:: 3.4 - -.. attribute:: HMAC.name - - The canonical name of this HMAC, always lowercase, e.g. ``hmac-md5``. - - .. versionadded:: 3.4 - - -This module also provides the following helper function: - -.. function:: compare_digest(a, b) - - Return ``a == b``. This function uses an approach designed to prevent - timing analysis by avoiding content-based short circuiting behaviour, - making it appropriate for cryptography. *a* and *b* must both be of the - same type: either :class:`str` (ASCII only, as e.g. returned by - :meth:`HMAC.hexdigest`), or a :term:`bytes-like object`. - - .. note:: - - If *a* and *b* are of different lengths, or if an error occurs, - a timing attack could theoretically reveal information about the - types and lengths of *a* and *b*—but not their values. - - - .. versionadded:: 3.3 - - -.. seealso:: - - Module :mod:`hashlib` - The Python module providing secure hash functions. diff --git a/third_party/python/Doc/library/html.entities.rst b/third_party/python/Doc/library/html.entities.rst deleted file mode 100644 index 067e1b1e5..000000000 --- a/third_party/python/Doc/library/html.entities.rst +++ /dev/null @@ -1,47 +0,0 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= - -.. module:: html.entities - :synopsis: Definitions of HTML general entities. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/html/entities.py` - --------------- - -This module defines four dictionaries, :data:`html5`, -:data:`name2codepoint`, :data:`codepoint2name`, and :data:`entitydefs`. - - -.. data:: html5 - - A dictionary that maps HTML5 named character references [#]_ to the - equivalent Unicode character(s), e.g. ``html5['gt;'] == '>'``. - Note that the trailing semicolon is included in the name (e.g. ``'gt;'``), - however some of the names are accepted by the standard even without the - semicolon: in this case the name is present with and without the ``';'``. - See also :func:`html.unescape`. - - .. versionadded:: 3.3 - - -.. data:: entitydefs - - A dictionary mapping XHTML 1.0 entity definitions to their replacement text in - ISO Latin-1. - - -.. data:: name2codepoint - - A dictionary that maps HTML entity names to the Unicode code points. - - -.. data:: codepoint2name - - A dictionary that maps Unicode code points to HTML entity names. - - -.. rubric:: Footnotes - -.. [#] See https://www.w3.org/TR/html5/syntax.html#named-character-references diff --git a/third_party/python/Doc/library/html.parser.rst b/third_party/python/Doc/library/html.parser.rst deleted file mode 100644 index ac844a683..000000000 --- a/third_party/python/Doc/library/html.parser.rst +++ /dev/null @@ -1,340 +0,0 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== - -.. module:: html.parser - :synopsis: A simple parser that can handle HTML and XHTML. - -**Source code:** :source:`Lib/html/parser.py` - -.. index:: - single: HTML - single: XHTML - --------------- - -This module defines a class :class:`HTMLParser` which serves as the basis for -parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. - -.. class:: HTMLParser(*, convert_charrefs=True) - - Create a parser instance able to parse invalid markup. - - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are - automatically converted to the corresponding Unicode characters. - - An :class:`.HTMLParser` instance is fed HTML data and calls handler methods - when start tags, end tags, text, comments, and other markup elements are - encountered. The user should subclass :class:`.HTMLParser` and override its - methods to implement the desired behavior. - - This parser does not check that end tags match start tags or call the end-tag - handler for elements which are closed implicitly by closing an outer element. - - .. versionchanged:: 3.4 - *convert_charrefs* keyword argument added. - - .. versionchanged:: 3.5 - The default value for argument *convert_charrefs* is now ``True``. - - -Example HTML Parser Application -------------------------------- - -As a basic example, below is a simple HTML parser that uses the -:class:`HTMLParser` class to print out start tags, end tags, and data -as they are encountered:: - - from html.parser import HTMLParser - - class MyHTMLParser(HTMLParser): - def handle_starttag(self, tag, attrs): - print("Encountered a start tag:", tag) - - def handle_endtag(self, tag): - print("Encountered an end tag :", tag) - - def handle_data(self, data): - print("Encountered some data :", data) - - parser = MyHTMLParser() - parser.feed('Test' - '

    Parse me!

    ') - -The output will then be: - -.. code-block:: none - - Encountered a start tag: html - Encountered a start tag: head - Encountered a start tag: title - Encountered some data : Test - Encountered an end tag : title - Encountered an end tag : head - Encountered a start tag: body - Encountered a start tag: h1 - Encountered some data : Parse me! - Encountered an end tag : h1 - Encountered an end tag : body - Encountered an end tag : html - - -:class:`.HTMLParser` Methods ----------------------------- - -:class:`HTMLParser` instances have the following methods: - - -.. method:: HTMLParser.feed(data) - - Feed some text to the parser. It is processed insofar as it consists of - complete elements; incomplete data is buffered until more data is fed or - :meth:`close` is called. *data* must be :class:`str`. - - -.. method:: HTMLParser.close() - - Force processing of all buffered data as if it were followed by an end-of-file - mark. This method may be redefined by a derived class to define additional - processing at the end of the input, but the redefined version should always call - the :class:`HTMLParser` base class method :meth:`close`. - - -.. method:: HTMLParser.reset() - - Reset the instance. Loses all unprocessed data. This is called implicitly at - instantiation time. - - -.. method:: HTMLParser.getpos() - - Return current line number and offset. - - -.. method:: HTMLParser.get_starttag_text() - - Return the text of the most recently opened start tag. This should not normally - be needed for structured processing, but may be useful in dealing with HTML "as - deployed" or for re-generating input with minimal changes (whitespace between - attributes can be preserved, etc.). - - -The following methods are called when data or markup elements are encountered -and they are meant to be overridden in a subclass. The base class -implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): - - -.. method:: HTMLParser.handle_starttag(tag, attrs) - - This method is called to handle the start of a tag (e.g. ``    ``). - - The *tag* argument is the name of the tag converted to lower case. - - -.. method:: HTMLParser.handle_startendtag(tag, attrs) - - Similar to :meth:`handle_starttag`, but called when the parser encounters an - XHTML-style empty tag (````). This method may be overridden by - subclasses which require this particular lexical information; the default - implementation simply calls :meth:`handle_starttag` and :meth:`handle_endtag`. - - -.. method:: HTMLParser.handle_data(data) - - This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). - - -.. method:: HTMLParser.handle_entityref(name) - - This method is called to process a named character reference of the form - ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. - - -.. method:: HTMLParser.handle_charref(name) - - This method is called to process decimal and hexadecimal numeric character - references of the form ``&#NNN;`` and ``&#xNNN;``. For example, the decimal - equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. - - -.. method:: HTMLParser.handle_comment(data) - - This method is called when a comment is encountered (e.g. ````). - - For example, the comment ```` will cause this method to be - called with the argument ``' comment '``. - - The content of Internet Explorer conditional comments (condcoms) will also be - sent to this method, so, for ````, - this method will receive ``'[if IE 9]>IE9-specific content``). - - The *decl* parameter will be the entire contents of the declaration inside - the ```` markup (e.g. ``'DOCTYPE html'``). - - -.. method:: HTMLParser.handle_pi(data) - - Method called when a processing instruction is encountered. The *data* - parameter will contain the entire processing instruction. For example, for the - processing instruction ````, this method would be called as - ``handle_pi("proc color='red'")``. It is intended to be overridden by a derived - class; the base class implementation does nothing. - - .. note:: - - The :class:`HTMLParser` class uses the SGML syntactic rules for processing - instructions. An XHTML processing instruction using the trailing ``'?'`` will - cause the ``'?'`` to be included in *data*. - - -.. method:: HTMLParser.unknown_decl(data) - - This method is called when an unrecognized declaration is read by the parser. - - The *data* parameter will be the entire contents of the declaration inside - the ```` markup. It is sometimes useful to be overridden by a - derived class. The base class implementation does nothing. - - -.. _htmlparser-examples: - -Examples --------- - -The following class implements a parser that will be used to illustrate more -examples:: - - from html.parser import HTMLParser - from html.entities import name2codepoint - - class MyHTMLParser(HTMLParser): - def handle_starttag(self, tag, attrs): - print("Start tag:", tag) - for attr in attrs: - print(" attr:", attr) - - def handle_endtag(self, tag): - print("End tag :", tag) - - def handle_data(self, data): - print("Data :", data) - - def handle_comment(self, data): - print("Comment :", data) - - def handle_entityref(self, name): - c = chr(name2codepoint[name]) - print("Named ent:", c) - - def handle_charref(self, name): - if name.startswith('x'): - c = chr(int(name[1:], 16)) - else: - c = chr(int(name)) - print("Num ent :", c) - - def handle_decl(self, data): - print("Decl :", data) - - parser = MyHTMLParser() - -Parsing a doctype:: - - >>> parser.feed('') - Decl : DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" - -Parsing an element with a few attributes and a title:: - - >>> parser.feed('The Python logo') - Start tag: img - attr: ('src', 'python-logo.png') - attr: ('alt', 'The Python logo') - >>> - >>> parser.feed('

    Python

    ') - Start tag: h1 - Data : Python - End tag : h1 - -The content of ``script`` and ``style`` elements is returned as is, without -further parsing:: - - >>> parser.feed('') - Start tag: style - attr: ('type', 'text/css') - Data : #python { color: green } - End tag : style - - >>> parser.feed('') - Start tag: script - attr: ('type', 'text/javascript') - Data : alert("hello!"); - End tag : script - -Parsing comments:: - - >>> parser.feed('' - ... '') - Comment : a comment - Comment : [if IE 9]>IE-specific content'``):: - - >>> parser.feed('>>>') - Named ent: > - Num ent : > - Num ent : > - -Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but -:meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: - - >>> for chunk in ['buff', 'ered ', 'text']: - ... parser.feed(chunk) - ... - Start tag: span - Data : buff - Data : ered - Data : text - End tag : span - -Parsing invalid HTML (e.g. unquoted attributes) also works:: - - >>> parser.feed('

    tag soup

    ') - Start tag: p - Start tag: a - attr: ('class', 'link') - attr: ('href', '#main') - Data : tag soup - End tag : p - End tag : a diff --git a/third_party/python/Doc/library/html.rst b/third_party/python/Doc/library/html.rst deleted file mode 100644 index c2b01e14e..000000000 --- a/third_party/python/Doc/library/html.rst +++ /dev/null @@ -1,39 +0,0 @@ -:mod:`html` --- HyperText Markup Language support -================================================= - -.. module:: html - :synopsis: Helpers for manipulating HTML. - -**Source code:** :source:`Lib/html/__init__.py` - --------------- - -This module defines utilities to manipulate HTML. - -.. function:: escape(s, quote=True) - - Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe - sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. - - .. versionadded:: 3.2 - - -.. function:: unescape(s) - - Convert all named and numeric character references (e.g. ``>``, - ``>``, ``>``) in the string *s* to the corresponding Unicode - characters. This function uses the rules defined by the HTML 5 standard - for both valid and invalid character references, and the :data:`list of - HTML 5 named character references `. - - .. versionadded:: 3.4 - --------------- - -Submodules in the ``html`` package are: - -* :mod:`html.parser` -- HTML/XHTML parser with lenient parsing mode -* :mod:`html.entities` -- HTML entity definitions diff --git a/third_party/python/Doc/library/http.client.rst b/third_party/python/Doc/library/http.client.rst deleted file mode 100644 index 2f59ecebb..000000000 --- a/third_party/python/Doc/library/http.client.rst +++ /dev/null @@ -1,551 +0,0 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== - -.. module:: http.client - :synopsis: HTTP and HTTPS protocol client (requires sockets). - -**Source code:** :source:`Lib/http/client.py` - -.. index:: - pair: HTTP; protocol - single: HTTP; http.client (standard module) - -.. index:: module: urllib.request - --------------- - -This module defines classes which implement the client side of the HTTP and -HTTPS protocols. It is normally not used directly --- the module -:mod:`urllib.request` uses it to handle URLs that use HTTP and HTTPS. - -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level HTTP client interface. - -.. note:: - - HTTPS support is only available if Python was compiled with SSL support - (through the :mod:`ssl` module). - -The module provides the following classes: - - -.. class:: HTTPConnection(host, port=None[, timeout], source_address=None) - - An :class:`HTTPConnection` instance represents one transaction with an HTTP - server. It should be instantiated passing it a host and optional port - number. If no port number is passed, the port is extracted from the host - string if it has the form ``host:port``, else the default HTTP port (80) is - used. If the optional *timeout* parameter is given, blocking - operations (like connection attempts) will timeout after that many seconds - (if it is not given, the global default timeout setting is used). - The optional *source_address* parameter may be a tuple of a (host, port) - to use as the source address the HTTP connection is made from. - - For example, the following calls all create instances that connect to the server - at the same host and port:: - - >>> h1 = http.client.HTTPConnection('www.python.org') - >>> h2 = http.client.HTTPConnection('www.python.org:80') - >>> h3 = http.client.HTTPConnection('www.python.org', 80) - >>> h4 = http.client.HTTPConnection('www.python.org', 80, timeout=10) - - .. versionchanged:: 3.2 - *source_address* was added. - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - not longer supported. - - -.. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, timeout], \ - source_address=None, *, context=None, \ - check_hostname=None) - - A subclass of :class:`HTTPConnection` that uses SSL for communication with - secure servers. Default port is ``443``. If *context* is specified, it - must be a :class:`ssl.SSLContext` instance describing the various SSL - options. - - Please read :ref:`ssl-security` for more information on best practices. - - .. versionchanged:: 3.2 - *source_address*, *context* and *check_hostname* were added. - - .. versionchanged:: 3.2 - This class now supports HTTPS virtual hosts if possible (that is, - if :data:`ssl.HAS_SNI` is true). - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - no longer supported. - - .. versionchanged:: 3.4.3 - This class now performs all the necessary certificate and hostname checks - by default. To revert to the previous, unverified, behavior - :func:`ssl._create_unverified_context` can be passed to the *context* - parameter. - - .. deprecated:: 3.6 - - *key_file* and *cert_file* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should - be used instead. - - -.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) - - Class whose instances are returned upon successful connection. Not - instantiated directly by user. - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9 style "Simple Responses" are - no longer supported. - - -The following exceptions are raised as appropriate: - - -.. exception:: HTTPException - - The base class of the other exceptions in this module. It is a subclass of - :exc:`Exception`. - - -.. exception:: NotConnected - - A subclass of :exc:`HTTPException`. - - -.. exception:: InvalidURL - - A subclass of :exc:`HTTPException`, raised if a port is given and is either - non-numeric or empty. - - -.. exception:: UnknownProtocol - - A subclass of :exc:`HTTPException`. - - -.. exception:: UnknownTransferEncoding - - A subclass of :exc:`HTTPException`. - - -.. exception:: UnimplementedFileMode - - A subclass of :exc:`HTTPException`. - - -.. exception:: IncompleteRead - - A subclass of :exc:`HTTPException`. - - -.. exception:: ImproperConnectionState - - A subclass of :exc:`HTTPException`. - - -.. exception:: CannotSendRequest - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: CannotSendHeader - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: ResponseNotReady - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: BadStatusLine - - A subclass of :exc:`HTTPException`. Raised if a server responds with a HTTP - status code that we don't understand. - - -.. exception:: LineTooLong - - A subclass of :exc:`HTTPException`. Raised if an excessively long line - is received in the HTTP protocol from the server. - - -.. exception:: RemoteDisconnected - - A subclass of :exc:`ConnectionResetError` and :exc:`BadStatusLine`. Raised - by :meth:`HTTPConnection.getresponse` when the attempt to read the response - results in no data read from the connection, indicating that the remote end - has closed the connection. - - .. versionadded:: 3.5 - Previously, :exc:`BadStatusLine`\ ``('')`` was raised. - - -The constants defined in this module are: - -.. data:: HTTP_PORT - - The default port for the HTTP protocol (always ``80``). - -.. data:: HTTPS_PORT - - The default port for the HTTPS protocol (always ``443``). - -.. data:: responses - - This dictionary maps the HTTP 1.1 status codes to the W3C names. - - Example: ``http.client.responses[http.client.NOT_FOUND]`` is ``'Not Found'``. - -See :ref:`http-status-codes` for a list of HTTP status codes that are -available in this module as constants. - - -.. _httpconnection-objects: - -HTTPConnection Objects ----------------------- - -:class:`HTTPConnection` instances have the following methods: - - -.. method:: HTTPConnection.request(method, url, body=None, headers={}, *, \ - encode_chunked=False) - - This will send a request to the server using the HTTP request - method *method* and the selector *url*. - - If *body* is specified, the specified data is sent after the headers are - finished. It may be a :class:`str`, a :term:`bytes-like object`, an - open :term:`file object`, or an iterable of :class:`bytes`. If *body* - is a string, it is encoded as ISO-8859-1, the default for HTTP. If it - is a bytes-like object, the bytes are sent as is. If it is a :term:`file - object`, the contents of the file is sent; this file object should - support at least the ``read()`` method. If the file object is an - instance of :class:`io.TextIOBase`, the data returned by the ``read()`` - method will be encoded as ISO-8859-1, otherwise the data returned by - ``read()`` is sent as is. If *body* is an iterable, the elements of the - iterable are sent as is until the iterable is exhausted. - - The *headers* argument should be a mapping of extra HTTP headers to send - with the request. - - If *headers* contains neither Content-Length nor Transfer-Encoding, - but there is a request body, one of those - header fields will be added automatically. If - *body* is ``None``, the Content-Length header is set to ``0`` for - methods that expect a body (``PUT``, ``POST``, and ``PATCH``). If - *body* is a string or a bytes-like object that is not also a - :term:`file `, the Content-Length header is - set to its length. Any other type of *body* (files - and iterables in general) will be chunk-encoded, and the - Transfer-Encoding header will automatically be set instead of - Content-Length. - - The *encode_chunked* argument is only relevant if Transfer-Encoding is - specified in *headers*. If *encode_chunked* is ``False``, the - HTTPConnection object assumes that all encoding is handled by the - calling code. If it is ``True``, the body will be chunk-encoded. - - .. note:: - Chunked transfer encoding has been added to the HTTP protocol - version 1.1. Unless the HTTP server is known to handle HTTP 1.1, - the caller must either specify the Content-Length, or must pass a - :class:`str` or bytes-like object that is not also a file as the - body representation. - - .. versionadded:: 3.2 - *body* can now be an iterable. - - .. versionchanged:: 3.6 - If neither Content-Length nor Transfer-Encoding are set in - *headers*, file and iterable *body* objects are now chunk-encoded. - The *encode_chunked* argument was added. - No attempt is made to determine the Content-Length for file - objects. - -.. method:: HTTPConnection.getresponse() - - Should be called after a request is sent to get the response from the server. - Returns an :class:`HTTPResponse` instance. - - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - - .. versionchanged:: 3.5 - If a :exc:`ConnectionError` or subclass is raised, the - :class:`HTTPConnection` object will be ready to reconnect when - a new request is sent. - - -.. method:: HTTPConnection.set_debuglevel(level) - - Set the debugging level. The default debug level is ``0``, meaning no - debugging output is printed. Any value greater than ``0`` will cause all - currently defined debug output to be printed to stdout. The ``debuglevel`` - is passed to any new :class:`HTTPResponse` objects that are created. - - .. versionadded:: 3.1 - - -.. method:: HTTPConnection.set_tunnel(host, port=None, headers=None) - - Set the host and the port for HTTP Connect Tunnelling. This allows running - the connection through a proxy server. - - The host and port arguments specify the endpoint of the tunneled connection - (i.e. the address included in the CONNECT request, *not* the address of the - proxy server). - - The headers argument should be a mapping of extra HTTP headers to send with - the CONNECT request. - - For example, to tunnel through a HTTPS proxy server running locally on port - 8080, we would pass the address of the proxy to the :class:`HTTPSConnection` - constructor, and the address of the host that we eventually want to reach to - the :meth:`~HTTPConnection.set_tunnel` method:: - - >>> import http.client - >>> conn = http.client.HTTPSConnection("localhost", 8080) - >>> conn.set_tunnel("www.python.org") - >>> conn.request("HEAD","/index.html") - - .. versionadded:: 3.2 - - -.. method:: HTTPConnection.connect() - - Connect to the server specified when the object was created. By default, - this is called automatically when making a request if the client does not - already have a connection. - - -.. method:: HTTPConnection.close() - - Close the connection to the server. - -As an alternative to using the :meth:`request` method described above, you can -also send your request step by step, by using the four functions below. - - -.. method:: HTTPConnection.putrequest(method, url, skip_host=False, \ - skip_accept_encoding=False) - - This should be the first call after the connection to the server has been - made. It sends a line to the server consisting of the *method* string, - the *url* string, and the HTTP version (``HTTP/1.1``). To disable automatic - sending of ``Host:`` or ``Accept-Encoding:`` headers (for example to accept - additional content encodings), specify *skip_host* or *skip_accept_encoding* - with non-False values. - - -.. method:: HTTPConnection.putheader(header, argument[, ...]) - - Send an :rfc:`822`\ -style header to the server. It sends a line to the server - consisting of the header, a colon and a space, and the first argument. If more - arguments are given, continuation lines are sent, each consisting of a tab and - an argument. - - -.. method:: HTTPConnection.endheaders(message_body=None, *, encode_chunked=False) - - Send a blank line to the server, signalling the end of the headers. The - optional *message_body* argument can be used to pass a message body - associated with the request. - - If *encode_chunked* is ``True``, the result of each iteration of - *message_body* will be chunk-encoded as specified in :rfc:`7230`, - Section 3.3.1. How the data is encoded is dependent on the type of - *message_body*. If *message_body* implements the :ref:`buffer interface - ` the encoding will result in a single chunk. - If *message_body* is a :class:`collections.Iterable`, each iteration - of *message_body* will result in a chunk. If *message_body* is a - :term:`file object`, each call to ``.read()`` will result in a chunk. - The method automatically signals the end of the chunk-encoded data - immediately after *message_body*. - - .. note:: Due to the chunked encoding specification, empty chunks - yielded by an iterator body will be ignored by the chunk-encoder. - This is to avoid premature termination of the read of the request by - the target server due to malformed encoding. - - .. versionadded:: 3.6 - Chunked encoding support. The *encode_chunked* parameter was - added. - - -.. method:: HTTPConnection.send(data) - - Send data to the server. This should be used directly only after the - :meth:`endheaders` method has been called and before :meth:`getresponse` is - called. - - -.. _httpresponse-objects: - -HTTPResponse Objects --------------------- - -An :class:`HTTPResponse` instance wraps the HTTP response from the -server. It provides access to the request headers and the entity -body. The response is an iterable object and can be used in a with -statement. - -.. versionchanged:: 3.5 - The :class:`io.BufferedIOBase` interface is now implemented and - all of its reader operations are supported. - - -.. method:: HTTPResponse.read([amt]) - - Reads and returns the response body, or up to the next *amt* bytes. - -.. method:: HTTPResponse.readinto(b) - - Reads up to the next len(b) bytes of the response body into the buffer *b*. - Returns the number of bytes read. - - .. versionadded:: 3.3 - -.. method:: HTTPResponse.getheader(name, default=None) - - Return the value of the header *name*, or *default* if there is no header - matching *name*. If there is more than one header with the name *name*, - return all of the values joined by ', '. If 'default' is any iterable other - than a single string, its elements are similarly returned joined by commas. - -.. method:: HTTPResponse.getheaders() - - Return a list of (header, value) tuples. - -.. method:: HTTPResponse.fileno() - - Return the ``fileno`` of the underlying socket. - -.. attribute:: HTTPResponse.msg - - A :class:`http.client.HTTPMessage` instance containing the response - headers. :class:`http.client.HTTPMessage` is a subclass of - :class:`email.message.Message`. - -.. attribute:: HTTPResponse.version - - HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. - -.. attribute:: HTTPResponse.status - - Status code returned by server. - -.. attribute:: HTTPResponse.reason - - Reason phrase returned by server. - -.. attribute:: HTTPResponse.debuglevel - - A debugging hook. If :attr:`debuglevel` is greater than zero, messages - will be printed to stdout as the response is read and parsed. - -.. attribute:: HTTPResponse.closed - - Is ``True`` if the stream is closed. - -Examples --------- - -Here is an example session that uses the ``GET`` method:: - - >>> import http.client - >>> conn = http.client.HTTPSConnection("www.python.org") - >>> conn.request("GET", "/") - >>> r1 = conn.getresponse() - >>> print(r1.status, r1.reason) - 200 OK - >>> data1 = r1.read() # This will return entire content. - >>> # The following example demonstrates reading data in chunks. - >>> conn.request("GET", "/") - >>> r1 = conn.getresponse() - >>> while not r1.closed: - ... print(r1.read(200)) # 200 bytes - b'\n 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** - -============================ ============================ ================================================= ============================================================= -Iterator Arguments Results Example -============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` -============================ ============================ ================================================= ============================================================= - -**Combinatoric iterators:** - -============================================== ==================== ============================================================= -Iterator Arguments Results -============================================== ==================== ============================================================= -:func:`product` p, q, ... [repeat=1] cartesian product, equivalent to a nested for-loop -:func:`permutations` p[, r] r-length tuples, all possible orderings, no repeated elements -:func:`combinations` p, r r-length tuples, in sorted order, no repeated elements -:func:`combinations_with_replacement` p, r r-length tuples, in sorted order, with repeated elements -``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` -``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` -``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` -============================================== ==================== ============================================================= - - -.. _itertools-functions: - -Itertool functions ------------------- - -The following module functions all construct and return iterators. Some provide -streams of infinite length, so they should only be accessed by functions or -loops that truncate the stream. - -.. function:: accumulate(iterable[, func]) - - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) If the input iterable is empty, the - output iterable will also be empty. - - Roughly equivalent to:: - - def accumulate(iterable, func=operator.add): - 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 - it = iter(iterable) - try: - total = next(it) - except StopIteration: - return - yield total - for element in it: - total = func(total, element) - yield total - - There are a number of uses for the *func* argument. It can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments. First-order - `recurrence relations `_ - can be modeled by supplying the initial value in the iterable and using only - the accumulated total in *func* argument:: - - >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] - >>> list(accumulate(data, max)) # running maximum - [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] - - # Amortize a 5% loan of 1000 with 4 annual payments of 90 - >>> cashflows = [1000, -90, -90, -90, -90] - >>> list(accumulate(cashflows, lambda bal, pmt: bal*1.05 + pmt)) - [1000, 960.0, 918.0, 873.9000000000001, 827.5950000000001] - - # Chaotic recurrence relation https://en.wikipedia.org/wiki/Logistic_map - >>> logistic_map = lambda x, _: r * x * (1 - x) - >>> r = 3.8 - >>> x0 = 0.4 - >>> inputs = repeat(x0, 36) # only the initial value is used - >>> [format(x, '.2f') for x in accumulate(inputs, logistic_map)] - ['0.40', '0.91', '0.30', '0.81', '0.60', '0.92', '0.29', '0.79', '0.63', - '0.88', '0.39', '0.90', '0.33', '0.84', '0.52', '0.95', '0.18', '0.57', - '0.93', '0.25', '0.71', '0.79', '0.63', '0.88', '0.39', '0.91', '0.32', - '0.83', '0.54', '0.95', '0.20', '0.60', '0.91', '0.30', '0.80', '0.60'] - - See :func:`functools.reduce` for a similar function that returns only the - final accumulated value. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the optional *func* parameter. - -.. function:: chain(*iterables) - - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: - - def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F - for it in iterables: - for element in it: - yield element - - -.. classmethod:: chain.from_iterable(iterable) - - Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Roughly equivalent to:: - - def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F - for it in iterables: - for element in it: - yield element - - -.. function:: combinations(iterable, r) - - Return *r* length subsequences of elements from the input *iterable*. - - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeat - values in each combination. - - Roughly equivalent to:: - - def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 - pool = tuple(iterable) - n = len(pool) - if r > n: - return - indices = list(range(r)) - yield tuple(pool[i] for i in indices) - while True: - for i in reversed(range(r)): - if indices[i] != i + n - r: - break - else: - return - indices[i] += 1 - for j in range(i+1, r): - indices[j] = indices[j-1] + 1 - yield tuple(pool[i] for i in indices) - - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. - -.. function:: combinations_with_replacement(iterable, r) - - Return *r* length subsequences of elements from the input *iterable* - allowing individual elements to be repeated more than once. - - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations - will also be unique. - - Roughly equivalent to:: - - def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC - pool = tuple(iterable) - n = len(pool) - if not n and r: - return - indices = [0] * r - yield tuple(pool[i] for i in indices) - while True: - for i in reversed(range(r)): - if indices[i] != n - 1: - break - else: - return - indices[i:] = [indices[i] + 1] * (r - i) - yield tuple(pool[i] for i in indices) - - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - - .. versionadded:: 3.1 - - -.. function:: compress(data, selectors) - - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. - Roughly equivalent to:: - - def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F - return (d for d, s in zip(data, selectors) if s) - - .. versionadded:: 3.1 - - -.. function:: count(start=0, step=1) - - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: - - def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) -> 2.5 3.0 3.5 ... - n = start - while True: - yield n - n += step - - When counting with floating point numbers, better accuracy can sometimes be - achieved by substituting multiplicative code such as: ``(start + step * i - for i in count())``. - - .. versionchanged:: 3.1 - Added *step* argument and allowed non-integer arguments. - -.. function:: cycle(iterable) - - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: - - def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... - saved = [] - for element in iterable: - yield element - saved.append(element) - while saved: - for element in saved: - yield element - - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). - - -.. function:: dropwhile(predicate, iterable) - - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: - - def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 - iterable = iter(iterable) - for x in iterable: - if not predicate(x): - yield x - break - for x in iterable: - yield x - -.. function:: filterfalse(predicate, iterable) - - Make an iterator that filters elements from iterable returning only those for - which the predicate is ``False``. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: - - def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 - if predicate is None: - predicate = bool - for x in iterable: - if not predicate(x): - yield x - - -.. function:: groupby(iterable, key=None) - - Make an iterator that returns consecutive keys and groups from the *iterable*. - The *key* is a function computing a key value for each element. If not - specified or is ``None``, *key* defaults to an identity function and returns - the element unchanged. Generally, the iterable needs to already be sorted on - the same key function. - - The operation of :func:`groupby` is similar to the ``uniq`` filter in Unix. It - generates a break or new group every time the value of the key function changes - (which is why it is usually necessary to have sorted the data using the same key - function). That behavior differs from SQL's GROUP BY which aggregates common - elements regardless of their input order. - - The returned group is itself an iterator that shares the underlying iterable - with :func:`groupby`. Because the source is shared, when the :func:`groupby` - object is advanced, the previous group is no longer visible. So, if that data - is needed later, it should be stored as a list:: - - groups = [] - uniquekeys = [] - data = sorted(data, key=keyfunc) - for k, g in groupby(data, keyfunc): - groups.append(list(g)) # Store group iterator as a list - uniquekeys.append(k) - - :func:`groupby` is roughly equivalent to:: - - class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - def __iter__(self): - return self - def __next__(self): - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey)) - def _grouper(self, tgtkey): - while self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: - return - self.currkey = self.keyfunc(self.currvalue) - - -.. function:: islice(iterable, stop) - islice(iterable, start, stop[, step]) - - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. Unlike regular slicing, :func:`islice` does not support - negative values for *start*, *stop*, or *step*. Can be used to extract related - fields from data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). Roughly equivalent to:: - - def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G - s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass - - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. - - -.. function:: permutations(iterable, r=None) - - Return successive *r* length permutations of elements in the *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length - of the *iterable* and all possible full-length permutations - are generated. - - Permutations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the permutation tuples will be produced - in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeat - values in each permutation. - - Roughly equivalent to:: - - def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - if r > n: - return - indices = list(range(n)) - cycles = list(range(n, n-r, -1)) - yield tuple(pool[i] for i in indices[:r]) - while n: - for i in reversed(range(r)): - cycles[i] -= 1 - if cycles[i] == 0: - indices[i:] = indices[i+1:] + indices[i:i+1] - cycles[i] = n - i - else: - j = cycles[i] - indices[i], indices[-j] = indices[-j], indices[i] - yield tuple(pool[i] for i in indices[:r]) - break - else: - return - - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. - -.. function:: product(*iterables, repeat=1) - - Cartesian product of input iterables. - - Roughly equivalent to nested for-loops in a generator expression. For example, - ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. - - The nested loops cycle like an odometer with the rightmost element advancing - on every iteration. This pattern creates a lexicographic ordering so that if - the input's iterables are sorted, the product tuples are emitted in sorted - order. - - To compute the product of an iterable with itself, specify the number of - repetitions with the optional *repeat* keyword argument. For example, - ``product(A, repeat=4)`` means the same as ``product(A, A, A, A)``. - - This function is roughly equivalent to the following code, except that the - actual implementation does not build up intermediate results in memory:: - - def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat - result = [[]] - for pool in pools: - result = [x+[y] for x in result for y in pool] - for prod in result: - yield tuple(prod) - - -.. function:: repeat(object[, times]) - - Make an iterator that returns *object* over and over again. Runs indefinitely - unless the *times* argument is specified. Used as argument to :func:`map` for - invariant parameters to the called function. Also used with :func:`zip` to - create an invariant part of a tuple record. - - Roughly equivalent to:: - - def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 - if times is None: - while True: - yield object - else: - for i in range(times): - yield object - - A common use for *repeat* is to supply a stream of constant values to *map* - or *zip*:: - - >>> list(map(pow, range(10), repeat(2))) - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - -.. function:: starmap(function, iterable) - - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (the data has been "pre-zipped"). The - difference between :func:`map` and :func:`starmap` parallels the distinction - between ``function(a,b)`` and ``function(*c)``. Roughly equivalent to:: - - def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 - for args in iterable: - yield function(*args) - - -.. function:: takewhile(predicate, iterable) - - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: - - def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 - for x in iterable: - if predicate(x): - yield x - else: - break - - -.. function:: tee(iterable, n=2) - - Return *n* independent iterators from a single iterable. - - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue). - - Roughly equivalent to:: - - def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) - - Once :func:`tee` has made a split, the original *iterable* should not be - used anywhere else; otherwise, the *iterable* could get advanced without - the tee objects being informed. - - This itertool may require significant auxiliary storage (depending on how - much temporary data needs to be stored). In general, if one iterator uses - most or all of the data before another iterator starts, it is faster to use - :func:`list` instead of :func:`tee`. - - -.. function:: zip_longest(*iterables, fillvalue=None) - - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: - - class ZipExhausted(Exception): - pass - - def zip_longest(*args, **kwds): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- - fillvalue = kwds.get('fillvalue') - counter = len(args) - 1 - def sentinel(): - nonlocal counter - if not counter: - raise ZipExhausted - counter -= 1 - yield fillvalue - fillers = repeat(fillvalue) - iterators = [chain(it, sentinel(), fillers) for it in args] - try: - while iterators: - yield tuple(map(next, iterators)) - except ZipExhausted: - pass - - If one of the iterables is potentially infinite, then the :func:`zip_longest` - function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. - - -.. _itertools-recipes: - -Itertools Recipes ------------------ - -This section shows recipes for creating an extended toolset using the existing -itertools as building blocks. - -The extended tools offer the same high performance as the underlying toolset. -The superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. - -.. testcode:: - - def take(n, iterable): - "Return first n items of the iterable as a list" - return list(islice(iterable, n)) - - def prepend(value, iterator): - "Prepend a single value in front of an iterator" - # prepend(1, [2, 3, 4]) -> 1 2 3 4 - return chain([value], iterator) - - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - - def tail(n, iterable): - "Return an iterator over the last n items" - # tail(3, 'ABCDEFG') --> E F G - return iter(collections.deque(iterable, maxlen=n)) - - def consume(iterator, n=None): - "Advance the iterator n-steps ahead. If n is None, consume entirely." - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - collections.deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - def nth(iterable, n, default=None): - "Returns the nth item or a default value" - return next(islice(iterable, n, None), default) - - def all_equal(iterable): - "Returns True if all the elements are equal to each other" - g = groupby(iterable) - return next(g, True) and not next(g, False) - - def quantify(iterable, pred=bool): - "Count how many times the predicate is true" - return sum(map(pred, iterable)) - - def padnone(iterable): - """Returns the sequence elements and then returns None indefinitely. - - Useful for emulating the behavior of the built-in map() function. - """ - return chain(iterable, repeat(None)) - - def ncycles(iterable, n): - "Returns the sequence elements n times" - return chain.from_iterable(repeat(tuple(iterable), n)) - - def dotproduct(vec1, vec2): - return sum(map(operator.mul, vec1, vec2)) - - def flatten(listOfLists): - "Flatten one level of nesting" - return chain.from_iterable(listOfLists) - - def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) - - def partition(pred, iterable): - 'Use a predicate to partition entries into false entries and true entries' - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 - t1, t2 = tee(iterable) - return filterfalse(pred, t1), filter(pred, t2) - - def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBCcAD', str.lower) --> A B C A D - return map(next, map(itemgetter(1), groupby(iterable, key))) - - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass - - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item - for which pred(item) is true. - - """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) - - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat - return tuple(random.choice(pool) for pool in pools) - - def random_permutation(iterable, r=None): - "Random selection from itertools.permutations(iterable, r)" - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(random.sample(pool, r)) - - def random_combination(iterable, r): - "Random selection from itertools.combinations(iterable, r)" - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.sample(range(n), r)) - return tuple(pool[i] for i in indices) - - def random_combination_with_replacement(iterable, r): - "Random selection from itertools.combinations_with_replacement(iterable, r)" - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.randrange(n) for i in range(r)) - return tuple(pool[i] for i in indices) - - def nth_combination(iterable, r, index): - 'Equivalent to list(combinations(iterable, r))[index]' - pool = tuple(iterable) - n = len(pool) - if r < 0 or r > n: - raise ValueError - c = 1 - k = min(r, n-r) - for i in range(1, k+1): - c = c * (n - k + i) // i - if index < 0: - index += c - if index < 0 or index >= c: - raise IndexError - result = [] - while r: - c, n, r = c*r//n, n-1, r-1 - while index >= c: - index -= c - c, n = c*(n-r)//n, n-1 - result.append(pool[-1-n]) - return tuple(result) - -Note, many of the above recipes can be optimized by replacing global lookups -with local variables defined as default values. For example, the -*dotproduct* recipe can be written as:: - - def dotproduct(vec1, vec2, sum=sum, map=map, mul=operator.mul): - return sum(map(mul, vec1, vec2)) diff --git a/third_party/python/Doc/library/json.rst b/third_party/python/Doc/library/json.rst deleted file mode 100644 index 98ca86e4b..000000000 --- a/third_party/python/Doc/library/json.rst +++ /dev/null @@ -1,735 +0,0 @@ -:mod:`json` --- JSON encoder and decoder -======================================== - -.. module:: json - :synopsis: Encode and decode the JSON format. - -.. moduleauthor:: Bob Ippolito -.. sectionauthor:: Bob Ippolito - -**Source code:** :source:`Lib/json/__init__.py` - --------------- - -`JSON (JavaScript Object Notation) `_, specified by -:rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, -is a lightweight data interchange format inspired by -`JavaScript `_ object literal syntax -(although it is not a strict subset of JavaScript [#rfc-errata]_ ). - -:mod:`json` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. - -Encoding basic Python object hierarchies:: - - >>> import json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps('\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from io import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import json - >>> json.dumps([1, 2, 3, {'4': 5, '6': 7}], separators=(',', ':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import json - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') - ['foo', {'bar': ['baz', None, 1.0, 2]}] - >>> json.loads('"\\"foo\\bar"') - '"foo\x08ar' - >>> from io import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io) - ['streaming API'] - -Specializing JSON object decoding:: - - >>> import json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> import decimal - >>> json.loads('1.1', parse_float=decimal.Decimal) - Decimal('1.1') - -Extending :class:`JSONEncoder`:: - - >>> import json - >>> class ComplexEncoder(json.JSONEncoder): - ... def default(self, obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) - ... - >>> json.dumps(2 + 1j, cls=ComplexEncoder) - '[2.0, 1.0]' - >>> ComplexEncoder().encode(2 + 1j) - '[2.0, 1.0]' - >>> list(ComplexEncoder().iterencode(2 + 1j)) - ['[2.0', ', 1.0', ']'] - - -Using :mod:`json.tool` from the shell to validate and pretty-print: - -.. code-block:: shell-session - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - -See :ref:`json-commandline` for detailed documentation. - -.. note:: - - JSON is a subset of `YAML `_ 1.2. The JSON produced by - this module's default settings (in particular, the default *separators* - value) is also a subset of YAML 1.0 and 1.1. This module can thus also be - used as a YAML serializer. - - -Basic Usage ------------ - -.. function:: dump(obj, fp, *, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) - - Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting - :term:`file-like object`) using this :ref:`conversion table - `. - - If *skipkeys* is true (default: ``False``), then dict keys that are not - of a basic type (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, - ``None``) will be skipped instead of raising a :exc:`TypeError`. - - The :mod:`json` module always produces :class:`str` objects, not - :class:`bytes` objects. Therefore, ``fp.write()`` must support :class:`str` - input. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. - - If *check_circular* is false (default: ``True``), then the circular - reference check for container types will be skipped and a circular reference - will result in an :exc:`OverflowError` (or worse). - - If *allow_nan* is false (default: ``True``), then it will be a - :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, - ``inf``, ``-inf``) in strict compliance of the JSON specification. - If *allow_nan* is true, their JavaScript equivalents (``NaN``, - ``Infinity``, ``-Infinity``) will be used. - - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. - - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. - - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - If *sort_keys* is true (default: ``False``), then the output of - dictionaries will be sorted by key. - - To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the - :meth:`default` method to serialize additional types), specify it with the - *cls* kwarg; otherwise :class:`JSONEncoder` is used. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. note:: - - Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, - so trying to serialize multiple objects with repeated calls to - :func:`dump` using the same *fp* will result in an invalid JSON file. - -.. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) - - Serialize *obj* to a JSON formatted :class:`str` using this :ref:`conversion - table `. The arguments have the same meaning as in - :func:`dump`. - - .. note:: - - Keys in key/value pairs of JSON are always of the type :class:`str`. When - a dictionary is converted into JSON, all the keys of the dictionary are - coerced to strings. As a result of this, if a dictionary is converted - into JSON and then back into a dictionary, the dictionary may not equal - the original one. That is, ``loads(dumps(x)) != x`` if x has non-string - keys. - -.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - - Deserialize *fp* (a ``.read()``-supporting :term:`text file` or - :term:`binary file` containing a JSON document) to a Python object using - this :ref:`conversion table `. - - *object_hook* is an optional function that will be called with the result of - any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). - - *object_pairs_hook* is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders that - rely on the order that the key and value pairs are decoded (for example, - :func:`collections.OrderedDict` will remember the order of insertion). If - *object_hook* is also defined, the *object_pairs_hook* takes priority. - - .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - .. versionchanged:: 3.1 - *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - - To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls`` - kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments - will be passed to the constructor of the class. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. versionchanged:: 3.6 - *fp* can now be a :term:`binary file`. The input encoding should be - UTF-8, UTF-16 or UTF-32. - -.. function:: loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - - Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` - instance containing a JSON document) to a Python object using this - :ref:`conversion table `. - - The other arguments have the same meaning as in :func:`load`, except - *encoding* which is ignored and deprecated. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - *s* can now be of type :class:`bytes` or :class:`bytearray`. The - input encoding should be UTF-8, UTF-16 or UTF-32. - - -Encoders and Decoders ---------------------- - -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) - - Simple JSON decoder. - - Performs the following translations in decoding by default: - - .. _json-to-py-table: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str | - +---------------+-------------------+ - | number (int) | int | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their - corresponding ``float`` values, which is outside the JSON spec. - - *object_hook*, if specified, will be called with the result of every JSON - object decoded and its return value will be used in place of the given - :class:`dict`. This can be used to provide custom deserializations (e.g. to - support JSON-RPC class hinting). - - *object_pairs_hook*, if specified will be called with the result of every - JSON object decoded with an ordered list of pairs. The return value of - *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders that rely on the order - that the key and value pairs are decoded (for example, - :func:`collections.OrderedDict` will remember the order of insertion). If - *object_hook* is also defined, the *object_pairs_hook* takes priority. - - .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - If *strict* is false (``True`` is the default), then control characters - will be allowed inside strings. Control characters in this context are - those with character codes in the 0--31 range, including ``'\t'`` (tab), - ``'\n'``, ``'\r'`` and ``'\0'``. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All parameters are now :ref:`keyword-only `. - - .. method:: decode(s) - - Return the Python representation of *s* (a :class:`str` instance - containing a JSON document). - - :exc:`JSONDecodeError` will be raised if the given JSON document is not - valid. - - .. method:: raw_decode(s) - - Decode a JSON document from *s* (a :class:`str` beginning with a - JSON document) and return a 2-tuple of the Python representation - and the index in *s* where the document ended. - - This can be used to decode a JSON document from a string that may have - extraneous data at the end. - - -.. class:: JSONEncoder(*, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None) - - Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - .. _py-to-json-table: - - +----------------------------------------+---------------+ - | Python | JSON | - +========================================+===============+ - | dict | object | - +----------------------------------------+---------------+ - | list, tuple | array | - +----------------------------------------+---------------+ - | str | string | - +----------------------------------------+---------------+ - | int, float, int- & float-derived Enums | number | - +----------------------------------------+---------------+ - | True | true | - +----------------------------------------+---------------+ - | False | false | - +----------------------------------------+---------------+ - | None | null | - +----------------------------------------+---------------+ - - .. versionchanged:: 3.4 - Added support for int- and float-derived Enum classes. - - To extend this to recognize other objects, subclass and implement a - :meth:`default` method with another method that returns a serializable object - for ``o`` if possible, otherwise it should call the superclass implementation - (to raise :exc:`TypeError`). - - If *skipkeys* is false (the default), then it is a :exc:`TypeError` to - attempt encoding of keys that are not :class:`str`, :class:`int`, - :class:`float` or ``None``. If *skipkeys* is true, such items are simply - skipped. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. - - If *check_circular* is true (the default), then lists, dicts, and custom - encoded objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an :exc:`OverflowError`). - Otherwise, no such check takes place. - - If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and - ``-Infinity`` will be encoded as such. This behavior is not JSON - specification compliant, but is consistent with most JavaScript based - encoders and decoders. Otherwise, it will be a :exc:`ValueError` to encode - such floats. - - If *sort_keys* is true (default: ``False``), then the output of dictionaries - will be sorted by key; this is useful for regression tests to ensure that - JSON serializations can be compared on a day-to-day basis. - - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. - - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. - - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - .. versionchanged:: 3.6 - All parameters are now :ref:`keyword-only `. - - - .. method:: default(o) - - Implement this method in a subclass such that it returns a serializable - object for *o*, or calls the base implementation (to raise a - :exc:`TypeError`). - - For example, to support arbitrary iterators, you could implement default - like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) - - - .. method:: encode(o) - - Return a JSON string representation of a Python data structure, *o*. For - example:: - - >>> json.JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - - .. method:: iterencode(o) - - Encode the given object, *o*, and yield each string representation as - available. For example:: - - for chunk in json.JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - -Exceptions ----------- - -.. exception:: JSONDecodeError(msg, doc, pos) - - Subclass of :exc:`ValueError` with the following additional attributes: - - .. attribute:: msg - - The unformatted error message. - - .. attribute:: doc - - The JSON document being parsed. - - .. attribute:: pos - - The start index of *doc* where parsing failed. - - .. attribute:: lineno - - The line corresponding to *pos*. - - .. attribute:: colno - - The column corresponding to *pos*. - - .. versionadded:: 3.5 - - -Standard Compliance and Interoperability ----------------------------------------- - -The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. -This section details this module's level of compliance with the RFC. -For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and -parameters other than those explicitly mentioned, are not considered. - -This module does not comply with the RFC in a strict fashion, implementing some -extensions that are valid JavaScript but not valid JSON. In particular: - -- Infinite and NaN number values are accepted and output; -- Repeated names within an object are accepted, and only the value of the last - name-value pair is used. - -Since the RFC permits RFC-compliant parsers to accept input texts that are not -RFC-compliant, this module's deserializer is technically RFC-compliant under -default settings. - -Character Encodings -^^^^^^^^^^^^^^^^^^^ - -The RFC requires that JSON be represented using either UTF-8, UTF-16, or -UTF-32, with UTF-8 being the recommended default for maximum interoperability. - -As permitted, though not required, by the RFC, this module's serializer sets -*ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. - -Other than the *ensure_ascii* parameter, this module is defined strictly in -terms of conversion between Python objects and -:class:`Unicode strings `, and thus does not otherwise directly address -the issue of character encodings. - -The RFC prohibits adding a byte order mark (BOM) to the start of a JSON text, -and this module's serializer does not add a BOM to its output. -The RFC permits, but does not require, JSON deserializers to ignore an initial -BOM in their input. This module's deserializer raises a :exc:`ValueError` -when an initial BOM is present. - -The RFC does not explicitly forbid JSON strings which contain byte sequences -that don't correspond to valid Unicode characters (e.g. unpaired UTF-16 -surrogates), but it does note that they may cause interoperability problems. -By default, this module accepts and outputs (when present in the original -:class:`str`) code points for such sequences. - - -Infinite and NaN Number Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC does not permit the representation of infinite or NaN number values. -Despite that, by default, this module accepts and outputs ``Infinity``, -``-Infinity``, and ``NaN`` as if they were valid JSON number literal values:: - - >>> # Neither of these calls raises an exception, but the results are not valid JSON - >>> json.dumps(float('-inf')) - '-Infinity' - >>> json.dumps(float('nan')) - 'NaN' - >>> # Same when deserializing - >>> json.loads('-Infinity') - -inf - >>> json.loads('NaN') - nan - -In the serializer, the *allow_nan* parameter can be used to alter this -behavior. In the deserializer, the *parse_constant* parameter can be used to -alter this behavior. - - -Repeated Names Within an Object -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC specifies that the names within a JSON object should be unique, but -does not mandate how repeated names in JSON objects should be handled. By -default, this module does not raise an exception; instead, it ignores all but -the last name-value pair for a given name:: - - >>> weird_json = '{"x": 1, "x": 2, "x": 3}' - >>> json.loads(weird_json) - {'x': 3} - -The *object_pairs_hook* parameter can be used to alter this behavior. - - -Top-level Non-Object, Non-Array Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The old version of JSON specified by the obsolete :rfc:`4627` required that -the top-level value of a JSON text must be either a JSON object or array -(Python :class:`dict` or :class:`list`), and could not be a JSON null, -boolean, number, or string value. :rfc:`7159` removed that restriction, and -this module does not and has never implemented that restriction in either its -serializer or its deserializer. - -Regardless, for maximum interoperability, you may wish to voluntarily adhere -to the restriction yourself. - - -Implementation Limitations -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some JSON deserializer implementations may set limits on: - -* the size of accepted JSON texts -* the maximum level of nesting of JSON objects and arrays -* the range and precision of JSON numbers -* the content and maximum length of JSON strings - -This module does not impose any such limits beyond those of the relevant -Python datatypes themselves or the Python interpreter itself. - -When serializing to JSON, beware any such limitations in applications that may -consume your JSON. In particular, it is common for JSON numbers to be -deserialized into IEEE 754 double precision numbers and thus subject to that -representation's range and precision limitations. This is especially relevant -when serializing Python :class:`int` values of extremely large magnitude, or -when serializing instances of "exotic" numerical types such as -:class:`decimal.Decimal`. - - -.. _json-commandline: -.. program:: json.tool - -Command Line Interface ----------------------- - -.. module:: json.tool - :synopsis: A command line to validate and pretty-print JSON. - -**Source code:** :source:`Lib/json/tool.py` - --------------- - -The :mod:`json.tool` module provides a simple command line interface to validate -and pretty-print JSON objects. - -If the optional ``infile`` and ``outfile`` arguments are not -specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively: - -.. code-block:: shell-session - - $ echo '{"json": "obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - -.. versionchanged:: 3.5 - The output is now in the same order as the input. Use the - :option:`--sort-keys` option to sort the output of dictionaries - alphabetically by key. - - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. cmdoption:: infile - - The JSON file to be validated or pretty-printed: - - .. code-block:: shell-session - - $ python -m json.tool mp_films.json - [ - { - "title": "And Now for Something Completely Different", - "year": 1971 - }, - { - "title": "Monty Python and the Holy Grail", - "year": 1975 - } - ] - - If *infile* is not specified, read from :attr:`sys.stdin`. - -.. cmdoption:: outfile - - Write the output of the *infile* to the given *outfile*. Otherwise, write it - to :attr:`sys.stdout`. - -.. cmdoption:: --sort-keys - - Sort the output of dictionaries alphabetically by key. - - .. versionadded:: 3.5 - -.. cmdoption:: -h, --help - - Show the help message. - - -.. rubric:: Footnotes - -.. [#rfc-errata] As noted in `the errata for RFC 7159 - `_, - JSON permits literal U+2028 (LINE SEPARATOR) and - U+2029 (PARAGRAPH SEPARATOR) characters in strings, whereas JavaScript - (as of ECMAScript Edition 5.1) does not. diff --git a/third_party/python/Doc/library/keyword.rst b/third_party/python/Doc/library/keyword.rst deleted file mode 100644 index 173db2354..000000000 --- a/third_party/python/Doc/library/keyword.rst +++ /dev/null @@ -1,23 +0,0 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== - -.. module:: keyword - :synopsis: Test whether a string is a keyword in Python. - -**Source code:** :source:`Lib/keyword.py` - --------------- - -This module allows a Python program to determine if a string is a keyword. - - -.. function:: iskeyword(s) - - Return true if *s* is a Python keyword. - - -.. data:: kwlist - - Sequence containing all the keywords defined for the interpreter. If any - keywords are defined to only be active when particular :mod:`__future__` - statements are in effect, these will be included as well. diff --git a/third_party/python/Doc/library/language.rst b/third_party/python/Doc/library/language.rst deleted file mode 100644 index 1eac32e45..000000000 --- a/third_party/python/Doc/library/language.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. _language: - -************************ -Python Language Services -************************ - -Python provides a number of modules to assist in working with the Python -language. These modules support tokenizing, parsing, syntax analysis, bytecode -disassembly, and various other facilities. - -These modules include: - - -.. toctree:: - - parser.rst - ast.rst - symtable.rst - symbol.rst - token.rst - keyword.rst - tokenize.rst - tabnanny.rst - pyclbr.rst - py_compile.rst - compileall.rst - dis.rst - pickletools.rst diff --git a/third_party/python/Doc/library/linecache.rst b/third_party/python/Doc/library/linecache.rst deleted file mode 100644 index 34fcac57c..000000000 --- a/third_party/python/Doc/library/linecache.rst +++ /dev/null @@ -1,64 +0,0 @@ -:mod:`linecache` --- Random access to text lines -================================================ - -.. module:: linecache - :synopsis: This module provides random access to individual lines from text files. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/linecache.py` - --------------- - -The :mod:`linecache` module allows one to get any line from a Python source file, while -attempting to optimize internally, using a cache, the common case where many -lines are read from a single file. This is used by the :mod:`traceback` module -to retrieve source lines for inclusion in the formatted traceback. - -The :func:`tokenize.open` function is used to open files. This -function uses :func:`tokenize.detect_encoding` to get the encoding of the -file; in the absence of an encoding token, the file encoding defaults to UTF-8. - -The :mod:`linecache` module defines the following functions: - - -.. function:: getline(filename, lineno, module_globals=None) - - Get line *lineno* from file named *filename*. This function will never raise an - exception --- it will return ``''`` on errors (the terminating newline character - will be included for lines that are found). - - .. index:: triple: module; search; path - - If a file named *filename* is not found, the function will look for it in the - module search path, ``sys.path``, after first checking for a :pep:`302` - ``__loader__`` in *module_globals*, in case the module was imported from a - zipfile or other non-filesystem import source. - - -.. function:: clearcache() - - Clear the cache. Use this function if you no longer need lines from files - previously read using :func:`getline`. - - -.. function:: checkcache(filename=None) - - Check the cache for validity. Use this function if files in the cache may have - changed on disk, and you require the updated version. If *filename* is omitted, - it will check all the entries in the cache. - -.. function:: lazycache(filename, module_globals) - - Capture enough detail about a non-file-based module to permit getting its - lines later via :func:`getline` even if *module_globals* is ``None`` in the later - call. This avoids doing I/O until a line is actually needed, without having - to carry the module globals around indefinitely. - - .. versionadded:: 3.5 - -Example:: - - >>> import linecache - >>> linecache.getline(linecache.__file__, 8) - 'import sys\n' diff --git a/third_party/python/Doc/library/locale.rst b/third_party/python/Doc/library/locale.rst deleted file mode 100644 index 2addc0ea6..000000000 --- a/third_party/python/Doc/library/locale.rst +++ /dev/null @@ -1,572 +0,0 @@ -:mod:`locale` --- Internationalization services -=============================================== - -.. module:: locale - :synopsis: Internationalization services. - -.. moduleauthor:: Martin von Löwis -.. sectionauthor:: Martin von Löwis - -**Source code:** :source:`Lib/locale.py` - --------------- - -The :mod:`locale` module opens access to the POSIX locale database and -functionality. The POSIX locale mechanism allows programmers to deal with -certain cultural issues in an application, without requiring the programmer to -know all the specifics of each country where the software is executed. - -.. index:: module: _locale - -The :mod:`locale` module is implemented on top of the :mod:`_locale` module, -which in turn uses an ANSI C locale implementation if available. - -The :mod:`locale` module defines the following exception and functions: - - -.. exception:: Error - - Exception raised when the locale passed to :func:`setlocale` is not - recognized. - - -.. function:: setlocale(category, locale=None) - - If *locale* is given and not ``None``, :func:`setlocale` modifies the locale - setting for the *category*. The available categories are listed in the data - description below. *locale* may be a string, or an iterable of two strings - (language code and encoding). If it's an iterable, it's converted to a locale - name using the locale aliasing engine. An empty string specifies the user's - default settings. If the modification of the locale fails, the exception - :exc:`Error` is raised. If successful, the new locale setting is returned. - - If *locale* is omitted or ``None``, the current setting for *category* is - returned. - - :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: - - import locale - locale.setlocale(locale.LC_ALL, '') - - This sets the locale for all categories to the user's default setting (typically - specified in the :envvar:`LANG` environment variable). If the locale is not - changed thereafter, using multithreading should not cause problems. - - -.. function:: localeconv() - - Returns the database of the local conventions as a dictionary. This dictionary - has the following strings as keys: - - .. tabularcolumns:: |l|l|L| - - +----------------------+-------------------------------------+--------------------------------+ - | Category | Key | Meaning | - +======================+=====================================+================================+ - | :const:`LC_NUMERIC` | ``'decimal_point'`` | Decimal point character. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'grouping'`` | Sequence of numbers specifying | - | | | which relative positions the | - | | | ``'thousands_sep'`` is | - | | | expected. If the sequence is | - | | | terminated with | - | | | :const:`CHAR_MAX`, no further | - | | | grouping is performed. If the | - | | | sequence terminates with a | - | | | ``0``, the last group size is | - | | | repeatedly used. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'thousands_sep'`` | Character used between groups. | - +----------------------+-------------------------------------+--------------------------------+ - | :const:`LC_MONETARY` | ``'int_curr_symbol'`` | International currency symbol. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'currency_symbol'`` | Local currency symbol. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_cs_precedes/n_cs_precedes'`` | Whether the currency symbol | - | | | precedes the value (for | - | | | positive resp. negative | - | | | values). | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_sep_by_space/n_sep_by_space'`` | Whether the currency symbol is | - | | | separated from the value by a | - | | | space (for positive resp. | - | | | negative values). | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_decimal_point'`` | Decimal point used for | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'frac_digits'`` | Number of fractional digits | - | | | used in local formatting of | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'int_frac_digits'`` | Number of fractional digits | - | | | used in international | - | | | formatting of monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_thousands_sep'`` | Group separator used for | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_grouping'`` | Equivalent to ``'grouping'``, | - | | | used for monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'positive_sign'`` | Symbol used to annotate a | - | | | positive monetary value. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'negative_sign'`` | Symbol used to annotate a | - | | | negative monetary value. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_sign_posn/n_sign_posn'`` | The position of the sign (for | - | | | positive resp. negative | - | | | values), see below. | - +----------------------+-------------------------------------+--------------------------------+ - - All numeric values can be set to :const:`CHAR_MAX` to indicate that there is no - value specified in this locale. - - The possible values for ``'p_sign_posn'`` and ``'n_sign_posn'`` are given below. - - +--------------+-----------------------------------------+ - | Value | Explanation | - +==============+=========================================+ - | ``0`` | Currency and value are surrounded by | - | | parentheses. | - +--------------+-----------------------------------------+ - | ``1`` | The sign should precede the value and | - | | currency symbol. | - +--------------+-----------------------------------------+ - | ``2`` | The sign should follow the value and | - | | currency symbol. | - +--------------+-----------------------------------------+ - | ``3`` | The sign should immediately precede the | - | | value. | - +--------------+-----------------------------------------+ - | ``4`` | The sign should immediately follow the | - | | value. | - +--------------+-----------------------------------------+ - | ``CHAR_MAX`` | Nothing is specified in this locale. | - +--------------+-----------------------------------------+ - - The function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` - locale or the ``LC_MONETARY`` locale if locales are different and numeric or - monetary strings are non-ASCII. This temporary change affects other threads. - - .. versionchanged:: 3.6.5 - The function now sets temporarily the ``LC_CTYPE`` locale to the - ``LC_NUMERIC`` locale in some cases. - - -.. function:: nl_langinfo(option) - - Return some locale-specific information as a string. This function is not - available on all systems, and the set of possible options might also vary - across platforms. The possible argument values are numbers, for which - symbolic constants are available in the locale module. - - The :func:`nl_langinfo` function accepts one of the following keys. Most - descriptions are taken from the corresponding description in the GNU C - library. - - .. data:: CODESET - - Get a string with the name of the character encoding used in the - selected locale. - - .. data:: D_T_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent date and time in a locale-specific way. - - .. data:: D_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent a date in a locale-specific way. - - .. data:: T_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent a time in a locale-specific way. - - .. data:: T_FMT_AMPM - - Get a format string for :func:`time.strftime` to represent time in the am/pm - format. - - .. data:: DAY_1 ... DAY_7 - - Get the name of the n-th day of the week. - - .. note:: - - This follows the US convention of :const:`DAY_1` being Sunday, not the - international convention (ISO 8601) that Monday is the first day of the - week. - - .. data:: ABDAY_1 ... ABDAY_7 - - Get the abbreviated name of the n-th day of the week. - - .. data:: MON_1 ... MON_12 - - Get the name of the n-th month. - - .. data:: ABMON_1 ... ABMON_12 - - Get the abbreviated name of the n-th month. - - .. data:: RADIXCHAR - - Get the radix character (decimal dot, decimal comma, etc.). - - .. data:: THOUSEP - - Get the separator character for thousands (groups of three digits). - - .. data:: YESEXPR - - Get a regular expression that can be used with the regex function to - recognize a positive response to a yes/no question. - - .. note:: - - The expression is in the syntax suitable for the :c:func:`regex` function - from the C library, which might differ from the syntax used in :mod:`re`. - - .. data:: NOEXPR - - Get a regular expression that can be used with the regex(3) function to - recognize a negative response to a yes/no question. - - .. data:: CRNCYSTR - - Get the currency symbol, preceded by "-" if the symbol should appear before - the value, "+" if the symbol should appear after the value, or "." if the - symbol should replace the radix character. - - .. data:: ERA - - Get a string that represents the era used in the current locale. - - Most locales do not define this value. An example of a locale which does - define this value is the Japanese one. In Japan, the traditional - representation of dates includes the name of the era corresponding to the - then-emperor's reign. - - Normally it should not be necessary to use this value directly. Specifying - the ``E`` modifier in their format strings causes the :func:`time.strftime` - function to use this information. The format of the returned string is not - specified, and therefore you should not assume knowledge of it on different - systems. - - .. data:: ERA_D_T_FMT - - Get a format string for :func:`time.strftime` to represent date and time in a - locale-specific era-based way. - - .. data:: ERA_D_FMT - - Get a format string for :func:`time.strftime` to represent a date in a - locale-specific era-based way. - - .. data:: ERA_T_FMT - - Get a format string for :func:`time.strftime` to represent a time in a - locale-specific era-based way. - - .. data:: ALT_DIGITS - - Get a representation of up to 100 values used to represent the values - 0 to 99. - - -.. function:: getdefaultlocale([envvars]) - - Tries to determine the default locale settings and returns them as a tuple of - the form ``(language code, encoding)``. - - According to POSIX, a program which has not called ``setlocale(LC_ALL, '')`` - runs using the portable ``'C'`` locale. Calling ``setlocale(LC_ALL, '')`` lets - it use the default locale as defined by the :envvar:`LANG` variable. Since we - do not want to interfere with the current locale setting we thus emulate the - behavior in the way described above. - - To maintain compatibility with other platforms, not only the :envvar:`LANG` - variable is tested, but a list of variables given as envvars parameter. The - first found to be defined will be used. *envvars* defaults to the search - path used in GNU gettext; it must always contain the variable name - ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, - ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be - determined. - - -.. function:: getlocale(category=LC_CTYPE) - - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be - determined. - - -.. function:: getpreferredencoding(do_setlocale=True) - - Return the encoding used for text data, according to user preferences. User - preferences are expressed differently on different systems, and might not be - available programmatically on some systems, so this function only returns a - guess. - - On some systems, it is necessary to invoke :func:`setlocale` to obtain the user - preferences, so this function is not thread-safe. If invoking setlocale is not - necessary or desired, *do_setlocale* should be set to ``False``. - - -.. function:: normalize(localename) - - Returns a normalized locale code for the given locale name. The returned locale - code is formatted for use with :func:`setlocale`. If normalization fails, the - original name is returned unchanged. - - If the given encoding is not known, the function defaults to the default - encoding for the locale code just like :func:`setlocale`. - - -.. function:: resetlocale(category=LC_ALL) - - Sets the locale for *category* to the default setting. - - The default setting is determined by calling :func:`getdefaultlocale`. - *category* defaults to :const:`LC_ALL`. - - -.. function:: strcoll(string1, string2) - - Compares two strings according to the current :const:`LC_COLLATE` setting. As - any other compare function, returns a negative, or a positive value, or ``0``, - depending on whether *string1* collates before or after *string2* or is equal to - it. - - -.. function:: strxfrm(string) - - Transforms a string to one that can be used in locale-aware - comparisons. For example, ``strxfrm(s1) < strxfrm(s2)`` is - equivalent to ``strcoll(s1, s2) < 0``. This function can be used - when the same string is compared repeatedly, e.g. when collating a - sequence of strings. - - -.. function:: format(format, val, grouping=False, monetary=False) - - Formats a number *val* according to the current :const:`LC_NUMERIC` setting. - The format follows the conventions of the ``%`` operator. For floating point - values, the decimal point is modified if appropriate. If *grouping* is true, - also takes the grouping into account. - - If *monetary* is true, the conversion uses monetary thousands separator and - grouping strings. - - Please note that this function will only work for exactly one %char specifier. - For whole format strings, use :func:`format_string`. - - -.. function:: format_string(format, val, grouping=False) - - Processes formatting specifiers as in ``format % val``, but takes the current - locale settings into account. - - -.. function:: currency(val, symbol=True, grouping=False, international=False) - - Formats a number *val* according to the current :const:`LC_MONETARY` settings. - - The returned string includes the currency symbol if *symbol* is true, which is - the default. If *grouping* is true (which is not the default), grouping is done - with the value. If *international* is true (which is not the default), the - international currency symbol is used. - - Note that this function will not work with the 'C' locale, so you have to set a - locale via :func:`setlocale` first. - - -.. function:: str(float) - - Formats a floating point number using the same format as the built-in function - ``str(float)``, but takes the decimal point into account. - - -.. function:: delocalize(string) - - Converts a string into a normalized number string, following the - :const:`LC_NUMERIC` settings. - - .. versionadded:: 3.5 - - -.. function:: atof(string) - - Converts a string to a floating point number, following the :const:`LC_NUMERIC` - settings. - - -.. function:: atoi(string) - - Converts a string to an integer, following the :const:`LC_NUMERIC` conventions. - - -.. data:: LC_CTYPE - - .. index:: module: string - - Locale category for the character type functions. Depending on the settings of - this category, the functions of module :mod:`string` dealing with case change - their behaviour. - - -.. data:: LC_COLLATE - - Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. - - -.. data:: LC_TIME - - Locale category for the formatting of time. The function :func:`time.strftime` - follows these conventions. - - -.. data:: LC_MONETARY - - Locale category for formatting of monetary values. The available options are - available from the :func:`localeconv` function. - - -.. data:: LC_MESSAGES - - Locale category for message display. Python currently does not support - application specific locale-aware messages. Messages displayed by the operating - system, like those returned by :func:`os.strerror` might be affected by this - category. - - -.. data:: LC_NUMERIC - - Locale category for formatting numbers. The functions :func:`.format`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are - affected by that category. All other numeric formatting operations are not - affected. - - -.. data:: LC_ALL - - Combination of all locale settings. If this flag is used when the locale is - changed, setting the locale for all categories is attempted. If that fails for - any category, no category is changed at all. When the locale is retrieved using - this flag, a string indicating the setting for all categories is returned. This - string can be later used to restore the settings. - - -.. data:: CHAR_MAX - - This is a symbolic constant used for different values returned by - :func:`localeconv`. - - -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - -Background, details, hints, tips and caveats --------------------------------------------- - -The C standard defines the locale as a program-wide property that may be -relatively expensive to change. On top of that, some implementation are broken -in such a way that frequent locale changes may cause core dumps. This makes the -locale somewhat painful to use correctly. - -Initially, when a program is started, the locale is the ``C`` locale, no matter -what the user's preferred locale is. There is one exception: the -:data:`LC_CTYPE` category is changed at startup to set the current locale -encoding to the user's preferred locale encoding. The program must explicitly -say that it wants the user's preferred locale settings for other categories by -calling ``setlocale(LC_ALL, '')``. - -It is generally a bad idea to call :func:`setlocale` in some library routine, -since as a side effect it affects the entire program. Saving and restoring it -is almost as bad: it is expensive and affects other threads that happen to run -before the settings have been restored. - -If, when coding a module for general use, you need a locale independent version -of an operation that is affected by the locale (such as -certain formats used with :func:`time.strftime`), you will have to find a way to -do it without using the standard library routine. Even better is convincing -yourself that using locale settings is okay. Only as a last resort should you -document that your module is not compatible with non-\ ``C`` locale settings. - -The only way to perform numeric operations according to the locale is to use the -special functions defined by this module: :func:`atof`, :func:`atoi`, -:func:`.format`, :func:`.str`. - -There is no way to perform case conversions and character classifications -according to the locale. For (Unicode) text strings these are done according -to the character value only, while for byte strings, the conversions and -classifications are done according to the ASCII value of the byte, and bytes -whose high bit is set (i.e., non-ASCII bytes) are never converted or considered -part of a character class such as letter or whitespace. - - -.. _embedding-locale: - -For extension writers and programs that embed Python ----------------------------------------------------- - -Extension modules should never call :func:`setlocale`, except to find out what -the current locale is. But since the return value can only be used portably to -restore it, that is not very useful (except perhaps to find out whether or not -the locale is ``C``). - -When Python code uses the :mod:`locale` module to change the locale, this also -affects the embedding application. If the embedding application doesn't want -this to happen, it should remove the :mod:`_locale` extension module (which does -all the work) from the table of built-in modules in the :file:`config.c` file, -and make sure that the :mod:`_locale` module is not accessible as a shared -library. - - -.. _locale-gettext: - -Access to message catalogs --------------------------- - -.. function:: gettext(msg) -.. function:: dgettext(domain, msg) -.. function:: dcgettext(domain, msg, category) -.. function:: textdomain(domain) -.. function:: bindtextdomain(domain, dir) - -The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`!gettext`, -:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, -and :func:`!bind_textdomain_codeset`. These are similar to the same functions in -the :mod:`gettext` module, but use the C library's binary format for message -catalogs, and the C library's search algorithms for locating message catalogs. - -Python applications should normally find no need to invoke these functions, and -should use :mod:`gettext` instead. A known exception to this rule are -applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be -necessary to bind the text domain, so that the libraries can properly locate -their message catalogs. - diff --git a/third_party/python/Doc/library/logging.config.rst b/third_party/python/Doc/library/logging.config.rst deleted file mode 100644 index bf3f37568..000000000 --- a/third_party/python/Doc/library/logging.config.rst +++ /dev/null @@ -1,811 +0,0 @@ -:mod:`logging.config` --- Logging configuration -=============================================== - -.. module:: logging.config - :synopsis: Configuration of the logging module. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/config.py` - -.. sidebar:: Important - - This page contains only reference information. For tutorials, - please see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This section describes the API for configuring the logging module. - -.. _logging-config-api: - -Configuration functions -^^^^^^^^^^^^^^^^^^^^^^^ - -The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the -logging module using these functions or by making calls to the main API (defined -in :mod:`logging` itself) and defining handlers which are declared either in -:mod:`logging` or :mod:`logging.handlers`. - -.. function:: dictConfig(config) - - Takes the logging configuration from a dictionary. The contents of - this dictionary are described in :ref:`logging-config-dictschema` - below. - - If an error is encountered during configuration, this function will - raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` - or :exc:`ImportError` with a suitably descriptive message. The - following is a (possibly incomplete) list of conditions which will - raise an error: - - * A ``level`` which is not a string or which is a string not - corresponding to an actual logging level. - * A ``propagate`` value which is not a boolean. - * An id which does not have a corresponding destination. - * A non-existent handler id found during an incremental call. - * An invalid logger name. - * Inability to resolve to an internal or external object. - - Parsing is performed by the :class:`DictConfigurator` class, whose - constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module - has a callable attribute :attr:`dictConfigClass` - which is initially set to :class:`DictConfigurator`. - You can replace the value of :attr:`dictConfigClass` with a - suitable implementation of your own. - - :func:`dictConfig` calls :attr:`dictConfigClass` passing - the specified dictionary, and then calls the :meth:`configure` method on - the returned object to put the configuration into effect:: - - def dictConfig(config): - dictConfigClass(config).configure() - - For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then - set up custom prefixes which would be usable in the subsequent - :meth:`configure` call. :attr:`dictConfigClass` would be bound to - this new subclass, and then :func:`dictConfig` could be called exactly as - in the default, uncustomized state. - - .. versionadded:: 3.2 - -.. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True) - - Reads the logging configuration from a :mod:`configparser`\-format file. The - format of the file should be as described in - :ref:`logging-config-fileformat`. - This function can be called several times from an application, allowing an - end user to select from various pre-canned configurations (if the developer - provides a mechanism to present the choices and load the chosen - configuration). - - :param fname: A filename, or a file-like object, or an instance derived - from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is - instantiated, and the configuration read by it from the - object passed in ``fname``. If that has a :meth:`readline` - method, it is assumed to be a file-like object and read using - :meth:`~configparser.ConfigParser.read_file`; otherwise, - it is assumed to be a filename and passed to - :meth:`~configparser.ConfigParser.read`. - - - :param defaults: Defaults to be passed to the ConfigParser can be specified - in this argument. - - :param disable_existing_loggers: If specified as ``False``, loggers which - exist when this call is made are left - enabled. The default is ``True`` because this - enables old behaviour in a - backward-compatible way. This behaviour is to - disable any existing loggers unless they or - their ancestors are explicitly named in the - logging configuration. - - .. versionchanged:: 3.4 - An instance of a subclass of :class:`~configparser.RawConfigParser` is - now accepted as a value for ``fname``. This facilitates: - - * Use of a configuration file where logging configuration is just part - of the overall application configuration. - * Use of a configuration read from a file, and then modified by the using - application (e.g. based on command-line parameters or other aspects - of the runtime environment) before being passed to ``fileConfig``. - -.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) - - Starts up a socket server on the specified port, and listens for new - configurations. If no port is specified, the module's default - :const:`DEFAULT_LOGGING_CONFIG_PORT` is used. Logging configurations will be - sent as a file suitable for processing by :func:`dictConfig` or - :func:`fileConfig`. Returns a :class:`~threading.Thread` instance on which - you can call :meth:`~threading.Thread.start` to start the server, and which - you can :meth:`~threading.Thread.join` when appropriate. To stop the server, - call :func:`stopListening`. - - The ``verify`` argument, if specified, should be a callable which should - verify whether bytes received across the socket are valid and should be - processed. This could be done by encrypting and/or signing what is sent - across the socket, such that the ``verify`` callable can perform - signature verification and/or decryption. The ``verify`` callable is called - with a single argument - the bytes received across the socket - and should - return the bytes to be processed, or ``None`` to indicate that the bytes should - be discarded. The returned bytes could be the same as the passed in bytes - (e.g. when only verification is done), or they could be completely different - (perhaps if decryption were performed). - - To send a configuration to the socket, read in the configuration file and - send it to the socket as a sequence of bytes preceded by a four-byte length - string packed in binary using ``struct.pack('>L', n)``. - - .. note:: - - Because portions of the configuration are passed through - :func:`eval`, use of this function may open its users to a security risk. - While the function only binds to a socket on ``localhost``, and so does - not accept connections from remote machines, there are scenarios where - untrusted code could be run under the account of the process which calls - :func:`listen`. Specifically, if the process calling :func:`listen` runs - on a multi-user machine where users cannot trust each other, then a - malicious user could arrange to run essentially arbitrary code in a - victim user's process, simply by connecting to the victim's - :func:`listen` socket and sending a configuration which runs whatever - code the attacker wants to have executed in the victim's process. This is - especially easy to do if the default port is used, but not hard even if a - different port is used). To avoid the risk of this happening, use the - ``verify`` argument to :func:`listen` to prevent unrecognised - configurations from being applied. - - .. versionchanged:: 3.4 - The ``verify`` argument was added. - - .. note:: - - If you want to send configurations to the listener which don't - disable existing loggers, you will need to use a JSON format for - the configuration, which will use :func:`dictConfig` for configuration. - This method allows you to specify ``disable_existing_loggers`` as - ``False`` in the configuration you send. - - -.. function:: stopListening() - - Stops the listening server which was created with a call to :func:`listen`. - This is typically called before calling :meth:`join` on the return value from - :func:`listen`. - - -.. _logging-config-dictschema: - -Configuration dictionary schema -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Describing a logging configuration requires listing the various -objects to create and the connections between them; for example, you -may create a handler named 'console' and then say that the logger -named 'startup' will send its messages to the 'console' handler. -These objects aren't limited to those provided by the :mod:`logging` -module because you might write your own formatter or handler class. -The parameters to these classes may also need to include external -objects such as ``sys.stderr``. The syntax for describing these -objects and connections is defined in :ref:`logging-config-dict-connections` -below. - -Dictionary Schema Details -""""""""""""""""""""""""" - -The dictionary passed to :func:`dictConfig` must contain the following -keys: - -* *version* - to be set to an integer value representing the schema - version. The only valid value at present is 1, but having this key - allows the schema to evolve while still preserving backwards - compatibility. - -All other keys are optional, but if present they will be interpreted -as described below. In all cases below where a 'configuring dict' is -mentioned, it will be checked for the special ``'()'`` key to see if a -custom instantiation is required. If so, the mechanism described in -:ref:`logging-config-dict-userdef` below is used to create an instance; -otherwise, the context is used to determine what to instantiate. - -* *formatters* - the corresponding value will be a dict in which each - key is a formatter id and each value is a dict describing how to - configure the corresponding :class:`~logging.Formatter` instance. - - The configuring dict is searched for keys ``format`` and ``datefmt`` - (with defaults of ``None``) and these are used to construct a - :class:`~logging.Formatter` instance. - -* *filters* - the corresponding value will be a dict in which each key - is a filter id and each value is a dict describing how to configure - the corresponding Filter instance. - - The configuring dict is searched for the key ``name`` (defaulting to the - empty string) and this is used to construct a :class:`logging.Filter` - instance. - -* *handlers* - the corresponding value will be a dict in which each - key is a handler id and each value is a dict describing how to - configure the corresponding Handler instance. - - The configuring dict is searched for the following keys: - - * ``class`` (mandatory). This is the fully qualified name of the - handler class. - - * ``level`` (optional). The level of the handler. - - * ``formatter`` (optional). The id of the formatter for this - handler. - - * ``filters`` (optional). A list of ids of the filters for this - handler. - - All *other* keys are passed through as keyword arguments to the - handler's constructor. For example, given the snippet: - - .. code-block:: yaml - - handlers: - console: - class : logging.StreamHandler - formatter: brief - level : INFO - filters: [allow_foo] - stream : ext://sys.stdout - file: - class : logging.handlers.RotatingFileHandler - formatter: precise - filename: logconfig.log - maxBytes: 1024 - backupCount: 3 - - the handler with id ``console`` is instantiated as a - :class:`logging.StreamHandler`, using ``sys.stdout`` as the underlying - stream. The handler with id ``file`` is instantiated as a - :class:`logging.handlers.RotatingFileHandler` with the keyword arguments - ``filename='logconfig.log', maxBytes=1024, backupCount=3``. - -* *loggers* - the corresponding value will be a dict in which each key - is a logger name and each value is a dict describing how to - configure the corresponding Logger instance. - - The configuring dict is searched for the following keys: - - * ``level`` (optional). The level of the logger. - - * ``propagate`` (optional). The propagation setting of the logger. - - * ``filters`` (optional). A list of ids of the filters for this - logger. - - * ``handlers`` (optional). A list of ids of the handlers for this - logger. - - The specified loggers will be configured according to the level, - propagation, filters and handlers specified. - -* *root* - this will be the configuration for the root logger. - Processing of the configuration will be as for any logger, except - that the ``propagate`` setting will not be applicable. - -* *incremental* - whether the configuration is to be interpreted as - incremental to the existing configuration. This value defaults to - ``False``, which means that the specified configuration replaces the - existing configuration with the same semantics as used by the - existing :func:`fileConfig` API. - - If the specified value is ``True``, the configuration is processed - as described in the section on :ref:`logging-config-dict-incremental`. - -* *disable_existing_loggers* - whether any existing loggers are to be - disabled. This setting mirrors the parameter of the same name in - :func:`fileConfig`. If absent, this parameter defaults to ``True``. - This value is ignored if *incremental* is ``True``. - -.. _logging-config-dict-incremental: - -Incremental Configuration -""""""""""""""""""""""""" - -It is difficult to provide complete flexibility for incremental -configuration. For example, because objects such as filters -and formatters are anonymous, once a configuration is set up, it is -not possible to refer to such anonymous objects when augmenting a -configuration. - -Furthermore, there is not a compelling case for arbitrarily altering -the object graph of loggers, handlers, filters, formatters at -run-time, once a configuration is set up; the verbosity of loggers and -handlers can be controlled just by setting levels (and, in the case of -loggers, propagation flags). Changing the object graph arbitrarily in -a safe way is problematic in a multi-threaded environment; while not -impossible, the benefits are not worth the complexity it adds to the -implementation. - -Thus, when the ``incremental`` key of a configuration dict is present -and is ``True``, the system will completely ignore any ``formatters`` and -``filters`` entries, and process only the ``level`` -settings in the ``handlers`` entries, and the ``level`` and -``propagate`` settings in the ``loggers`` and ``root`` entries. - -Using a value in the configuration dict lets configurations to be sent -over the wire as pickled dicts to a socket listener. Thus, the logging -verbosity of a long-running application can be altered over time with -no need to stop and restart the application. - -.. _logging-config-dict-connections: - -Object connections -"""""""""""""""""" - -The schema describes a set of logging objects - loggers, -handlers, formatters, filters - which are connected to each other in -an object graph. Thus, the schema needs to represent connections -between the objects. For example, say that, once configured, a -particular logger has attached to it a particular handler. For the -purposes of this discussion, we can say that the logger represents the -source, and the handler the destination, of a connection between the -two. Of course in the configured objects this is represented by the -logger holding a reference to the handler. In the configuration dict, -this is done by giving each destination object an id which identifies -it unambiguously, and then using the id in the source object's -configuration to indicate that a connection exists between the source -and the destination object with that id. - -So, for example, consider the following YAML snippet: - -.. code-block:: yaml - - formatters: - brief: - # configuration for formatter with id 'brief' goes here - precise: - # configuration for formatter with id 'precise' goes here - handlers: - h1: #This is an id - # configuration of handler with id 'h1' goes here - formatter: brief - h2: #This is another id - # configuration of handler with id 'h2' goes here - formatter: precise - loggers: - foo.bar.baz: - # other configuration for logger 'foo.bar.baz' - handlers: [h1, h2] - -(Note: YAML used here because it's a little more readable than the -equivalent Python source form for the dictionary.) - -The ids for loggers are the logger names which would be used -programmatically to obtain a reference to those loggers, e.g. -``foo.bar.baz``. The ids for Formatters and Filters can be any string -value (such as ``brief``, ``precise`` above) and they are transient, -in that they are only meaningful for processing the configuration -dictionary and used to determine connections between objects, and are -not persisted anywhere when the configuration call is complete. - -The above snippet indicates that logger named ``foo.bar.baz`` should -have two handlers attached to it, which are described by the handler -ids ``h1`` and ``h2``. The formatter for ``h1`` is that described by id -``brief``, and the formatter for ``h2`` is that described by id -``precise``. - - -.. _logging-config-dict-userdef: - -User-defined objects -"""""""""""""""""""" - -The schema supports user-defined objects for handlers, filters and -formatters. (Loggers do not need to have different types for -different instances, so there is no support in this configuration -schema for user-defined logger classes.) - -Objects to be configured are described by dictionaries -which detail their configuration. In some places, the logging system -will be able to infer from the context how an object is to be -instantiated, but when a user-defined object is to be instantiated, -the system will not know how to do this. In order to provide complete -flexibility for user-defined object instantiation, the user needs -to provide a 'factory' - a callable which is called with a -configuration dictionary and which returns the instantiated object. -This is signalled by an absolute import path to the factory being -made available under the special key ``'()'``. Here's a concrete -example: - -.. code-block:: yaml - - formatters: - brief: - format: '%(message)s' - default: - format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - custom: - (): my.package.customFormatterFactory - bar: baz - spam: 99.9 - answer: 42 - -The above YAML snippet defines three formatters. The first, with id -``brief``, is a standard :class:`logging.Formatter` instance with the -specified format string. The second, with id ``default``, has a -longer format and also defines the time format explicitly, and will -result in a :class:`logging.Formatter` initialized with those two format -strings. Shown in Python source form, the ``brief`` and ``default`` -formatters have configuration sub-dictionaries:: - - { - 'format' : '%(message)s' - } - -and:: - - { - 'format' : '%(asctime)s %(levelname)-8s %(name)-15s %(message)s', - 'datefmt' : '%Y-%m-%d %H:%M:%S' - } - -respectively, and as these dictionaries do not contain the special key -``'()'``, the instantiation is inferred from the context: as a result, -standard :class:`logging.Formatter` instances are created. The -configuration sub-dictionary for the third formatter, with id -``custom``, is:: - - { - '()' : 'my.package.customFormatterFactory', - 'bar' : 'baz', - 'spam' : 99.9, - 'answer' : 42 - } - -and this contains the special key ``'()'``, which means that -user-defined instantiation is wanted. In this case, the specified -factory callable will be used. If it is an actual callable it will be -used directly - otherwise, if you specify a string (as in the example) -the actual callable will be located using normal import mechanisms. -The callable will be called with the **remaining** items in the -configuration sub-dictionary as keyword arguments. In the above -example, the formatter with id ``custom`` will be assumed to be -returned by the call:: - - my.package.customFormatterFactory(bar='baz', spam=99.9, answer=42) - -The key ``'()'`` has been used as the special key because it is not a -valid keyword parameter name, and so will not clash with the names of -the keyword arguments used in the call. The ``'()'`` also serves as a -mnemonic that the corresponding value is a callable. - - -.. _logging-config-dict-externalobj: - -Access to external objects -"""""""""""""""""""""""""" - -There are times where a configuration needs to refer to objects -external to the configuration, for example ``sys.stderr``. If the -configuration dict is constructed using Python code, this is -straightforward, but a problem arises when the configuration is -provided via a text file (e.g. JSON, YAML). In a text file, there is -no standard way to distinguish ``sys.stderr`` from the literal string -``'sys.stderr'``. To facilitate this distinction, the configuration -system looks for certain special prefixes in string values and -treat them specially. For example, if the literal string -``'ext://sys.stderr'`` is provided as a value in the configuration, -then the ``ext://`` will be stripped off and the remainder of the -value processed using normal import mechanisms. - -The handling of such prefixes is done in a way analogous to protocol -handling: there is a generic mechanism to look for prefixes which -match the regular expression ``^(?P[a-z]+)://(?P.*)$`` -whereby, if the ``prefix`` is recognised, the ``suffix`` is processed -in a prefix-dependent manner and the result of the processing replaces -the string value. If the prefix is not recognised, then the string -value will be left as-is. - - -.. _logging-config-dict-internalobj: - -Access to internal objects -"""""""""""""""""""""""""" - -As well as external objects, there is sometimes also a need to refer -to objects in the configuration. This will be done implicitly by the -configuration system for things that it knows about. For example, the -string value ``'DEBUG'`` for a ``level`` in a logger or handler will -automatically be converted to the value ``logging.DEBUG``, and the -``handlers``, ``filters`` and ``formatter`` entries will take an -object id and resolve to the appropriate destination object. - -However, a more generic mechanism is needed for user-defined -objects which are not known to the :mod:`logging` module. For -example, consider :class:`logging.handlers.MemoryHandler`, which takes -a ``target`` argument which is another handler to delegate to. Since -the system already knows about this class, then in the configuration, -the given ``target`` just needs to be the object id of the relevant -target handler, and the system will resolve to the handler from the -id. If, however, a user defines a ``my.package.MyHandler`` which has -an ``alternate`` handler, the configuration system would not know that -the ``alternate`` referred to a handler. To cater for this, a generic -resolution system allows the user to specify: - -.. code-block:: yaml - - handlers: - file: - # configuration of file handler goes here - - custom: - (): my.package.MyHandler - alternate: cfg://handlers.file - -The literal string ``'cfg://handlers.file'`` will be resolved in an -analogous way to strings with the ``ext://`` prefix, but looking -in the configuration itself rather than the import namespace. The -mechanism allows access by dot or by index, in a similar way to -that provided by ``str.format``. Thus, given the following snippet: - -.. code-block:: yaml - - handlers: - email: - class: logging.handlers.SMTPHandler - mailhost: localhost - fromaddr: my_app@domain.tld - toaddrs: - - support_team@domain.tld - - dev_team@domain.tld - subject: Houston, we have a problem. - -in the configuration, the string ``'cfg://handlers'`` would resolve to -the dict with key ``handlers``, the string ``'cfg://handlers.email`` -would resolve to the dict with key ``email`` in the ``handlers`` dict, -and so on. The string ``'cfg://handlers.email.toaddrs[1]`` would -resolve to ``'dev_team.domain.tld'`` and the string -``'cfg://handlers.email.toaddrs[0]'`` would resolve to the value -``'support_team@domain.tld'``. The ``subject`` value could be accessed -using either ``'cfg://handlers.email.subject'`` or, equivalently, -``'cfg://handlers.email[subject]'``. The latter form only needs to be -used if the key contains spaces or non-alphanumeric characters. If an -index value consists only of decimal digits, access will be attempted -using the corresponding integer value, falling back to the string -value if needed. - -Given a string ``cfg://handlers.myhandler.mykey.123``, this will -resolve to ``config_dict['handlers']['myhandler']['mykey']['123']``. -If the string is specified as ``cfg://handlers.myhandler.mykey[123]``, -the system will attempt to retrieve the value from -``config_dict['handlers']['myhandler']['mykey'][123]``, and fall back -to ``config_dict['handlers']['myhandler']['mykey']['123']`` if that -fails. - - -.. _logging-import-resolution: - -Import resolution and custom importers -"""""""""""""""""""""""""""""""""""""" - -Import resolution, by default, uses the builtin :func:`__import__` function -to do its importing. You may want to replace this with your own importing -mechanism: if so, you can replace the :attr:`importer` attribute of the -:class:`DictConfigurator` or its superclass, the -:class:`BaseConfigurator` class. However, you need to be -careful because of the way functions are accessed from classes via -descriptors. If you are using a Python callable to do your imports, and you -want to define it at class level rather than instance level, you need to wrap -it with :func:`staticmethod`. For example:: - - from importlib import import_module - from logging.config import BaseConfigurator - - BaseConfigurator.importer = staticmethod(import_module) - -You don't need to wrap with :func:`staticmethod` if you're setting the import -callable on a configurator *instance*. - - -.. _logging-config-fileformat: - -Configuration file format -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The configuration file format understood by :func:`fileConfig` is based on -:mod:`configparser` functionality. The file must contain sections called -``[loggers]``, ``[handlers]`` and ``[formatters]`` which identify by name the -entities of each type which are defined in the file. For each such entity, there -is a separate section which identifies how that entity is configured. Thus, for -a logger named ``log01`` in the ``[loggers]`` section, the relevant -configuration details are held in a section ``[logger_log01]``. Similarly, a -handler called ``hand01`` in the ``[handlers]`` section will have its -configuration held in a section called ``[handler_hand01]``, while a formatter -called ``form01`` in the ``[formatters]`` section will have its configuration -specified in a section called ``[formatter_form01]``. The root logger -configuration must be specified in a section called ``[logger_root]``. - -.. note:: - - The :func:`fileConfig` API is older than the :func:`dictConfig` API and does - not provide functionality to cover certain aspects of logging. For example, - you cannot configure :class:`~logging.Filter` objects, which provide for - filtering of messages beyond simple integer levels, using :func:`fileConfig`. - If you need to have instances of :class:`~logging.Filter` in your logging - configuration, you will need to use :func:`dictConfig`. Note that future - enhancements to configuration functionality will be added to - :func:`dictConfig`, so it's worth considering transitioning to this newer - API when it's convenient to do so. - -Examples of these sections in the file are given below. - -.. code-block:: ini - - [loggers] - keys=root,log02,log03,log04,log05,log06,log07 - - [handlers] - keys=hand01,hand02,hand03,hand04,hand05,hand06,hand07,hand08,hand09 - - [formatters] - keys=form01,form02,form03,form04,form05,form06,form07,form08,form09 - -The root logger must specify a level and a list of handlers. An example of a -root logger section is given below. - -.. code-block:: ini - - [logger_root] - level=NOTSET - handlers=hand01 - -The ``level`` entry can be one of ``DEBUG, INFO, WARNING, ERROR, CRITICAL`` or -``NOTSET``. For the root logger only, ``NOTSET`` means that all messages will be -logged. Level values are :func:`eval`\ uated in the context of the ``logging`` -package's namespace. - -The ``handlers`` entry is a comma-separated list of handler names, which must -appear in the ``[handlers]`` section. These names must appear in the -``[handlers]`` section and have corresponding sections in the configuration -file. - -For loggers other than the root logger, some additional information is required. -This is illustrated by the following example. - -.. code-block:: ini - - [logger_parser] - level=DEBUG - handlers=hand01 - propagate=1 - qualname=compiler.parser - -The ``level`` and ``handlers`` entries are interpreted as for the root logger, -except that if a non-root logger's level is specified as ``NOTSET``, the system -consults loggers higher up the hierarchy to determine the effective level of the -logger. The ``propagate`` entry is set to 1 to indicate that messages must -propagate to handlers higher up the logger hierarchy from this logger, or 0 to -indicate that messages are **not** propagated to handlers up the hierarchy. The -``qualname`` entry is the hierarchical channel name of the logger, that is to -say the name used by the application to get the logger. - -Sections which specify handler configuration are exemplified by the following. - -.. code-block:: ini - - [handler_hand01] - class=StreamHandler - level=NOTSET - formatter=form01 - args=(sys.stdout,) - -The ``class`` entry indicates the handler's class (as determined by :func:`eval` -in the ``logging`` package's namespace). The ``level`` is interpreted as for -loggers, and ``NOTSET`` is taken to mean 'log everything'. - -The ``formatter`` entry indicates the key name of the formatter for this -handler. If blank, a default formatter (``logging._defaultFormatter``) is used. -If a name is specified, it must appear in the ``[formatters]`` section and have -a corresponding section in the configuration file. - -The ``args`` entry, when :func:`eval`\ uated in the context of the ``logging`` -package's namespace, is the list of arguments to the constructor for the handler -class. Refer to the constructors for the relevant handlers, or to the examples -below, to see how typical entries are constructed. - -.. code-block:: ini - - [handler_hand02] - class=FileHandler - level=DEBUG - formatter=form02 - args=('python.log', 'w') - - [handler_hand03] - class=handlers.SocketHandler - level=INFO - formatter=form03 - args=('localhost', handlers.DEFAULT_TCP_LOGGING_PORT) - - [handler_hand04] - class=handlers.DatagramHandler - level=WARN - formatter=form04 - args=('localhost', handlers.DEFAULT_UDP_LOGGING_PORT) - - [handler_hand05] - class=handlers.SysLogHandler - level=ERROR - formatter=form05 - args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) - - [handler_hand06] - class=handlers.NTEventLogHandler - level=CRITICAL - formatter=form06 - args=('Python Application', '', 'Application') - - [handler_hand07] - class=handlers.SMTPHandler - level=WARN - formatter=form07 - args=('localhost', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') - - [handler_hand08] - class=handlers.MemoryHandler - level=NOTSET - formatter=form08 - target= - args=(10, ERROR) - - [handler_hand09] - class=handlers.HTTPHandler - level=NOTSET - formatter=form09 - args=('localhost:9022', '/log', 'GET') - -Sections which specify formatter configuration are typified by the following. - -.. code-block:: ini - - [formatter_form01] - format=F1 %(asctime)s %(levelname)s %(message)s - datefmt= - class=logging.Formatter - -The ``format`` entry is the overall format string, and the ``datefmt`` entry is -the :func:`strftime`\ -compatible date/time format string. If empty, the -package substitutes something which is almost equivalent to specifying the date -format string ``'%Y-%m-%d %H:%M:%S'``. This format also specifies milliseconds, -which are appended to the result of using the above format string, with a comma -separator. An example time in this format is ``2003-01-23 00:29:50,411``. - -The ``class`` entry is optional. It indicates the name of the formatter's class -(as a dotted module and class name.) This option is useful for instantiating a -:class:`~logging.Formatter` subclass. Subclasses of -:class:`~logging.Formatter` can present exception tracebacks in an expanded or -condensed format. - -.. note:: - - Due to the use of :func:`eval` as described above, there are - potential security risks which result from using the :func:`listen` to send - and receive configurations via sockets. The risks are limited to where - multiple users with no mutual trust run code on the same machine; see the - :func:`listen` documentation for more information. - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. diff --git a/third_party/python/Doc/library/logging.handlers.rst b/third_party/python/Doc/library/logging.handlers.rst deleted file mode 100644 index a31ee3192..000000000 --- a/third_party/python/Doc/library/logging.handlers.rst +++ /dev/null @@ -1,1073 +0,0 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ - -.. module:: logging.handlers - :synopsis: Handlers for the logging module. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/handlers.py` - -.. sidebar:: Important - - This page contains only reference information. For tutorials, - please see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -.. currentmodule:: logging - -The following useful handlers are provided in the package. Note that three of -the handlers (:class:`StreamHandler`, :class:`FileHandler` and -:class:`NullHandler`) are actually defined in the :mod:`logging` module itself, -but have been documented here along with the other handlers. - -.. _stream-handler: - -StreamHandler -^^^^^^^^^^^^^ - -The :class:`StreamHandler` class, located in the core :mod:`logging` package, -sends logging output to streams such as *sys.stdout*, *sys.stderr* or any -file-like object (or, more precisely, any object which supports :meth:`write` -and :meth:`flush` methods). - - -.. class:: StreamHandler(stream=None) - - Returns a new instance of the :class:`StreamHandler` class. If *stream* is - specified, the instance will use it for logging output; otherwise, *sys.stderr* - will be used. - - - .. method:: emit(record) - - If a formatter is specified, it is used to format the record. The record - is then written to the stream with a terminator. If exception information - is present, it is formatted using :func:`traceback.print_exception` and - appended to the stream. - - - .. method:: flush() - - Flushes the stream by calling its :meth:`flush` method. Note that the - :meth:`close` method is inherited from :class:`~logging.Handler` and so - does no output, so an explicit :meth:`flush` call may be needed at times. - -.. versionchanged:: 3.2 - The ``StreamHandler`` class now has a ``terminator`` attribute, default - value ``'\n'``, which is used as the terminator when writing a formatted - record to a stream. If you don't want this newline termination, you can - set the handler instance's ``terminator`` attribute to the empty string. - In earlier versions, the terminator was hardcoded as ``'\n'``. - -.. _file-handler: - -FileHandler -^^^^^^^^^^^ - -The :class:`FileHandler` class, located in the core :mod:`logging` package, -sends logging output to a disk file. It inherits the output functionality from -:class:`StreamHandler`. - - -.. class:: FileHandler(filename, mode='a', encoding=None, delay=False) - - Returns a new instance of the :class:`FileHandler` class. The specified file is - opened and used as the stream for logging. If *mode* is not specified, - :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. method:: close() - - Closes the file. - - - .. method:: emit(record) - - Outputs the record to the file. - - -.. _null-handler: - -NullHandler -^^^^^^^^^^^ - -.. versionadded:: 3.1 - -The :class:`NullHandler` class, located in the core :mod:`logging` package, -does not do any formatting or output. It is essentially a 'no-op' handler -for use by library developers. - -.. class:: NullHandler() - - Returns a new instance of the :class:`NullHandler` class. - - .. method:: emit(record) - - This method does nothing. - - .. method:: handle(record) - - This method does nothing. - - .. method:: createLock() - - This method returns ``None`` for the lock, since there is no - underlying I/O to which access needs to be serialized. - - -See :ref:`library-config` for more information on how to use -:class:`NullHandler`. - -.. _watched-file-handler: - -WatchedFileHandler -^^^^^^^^^^^^^^^^^^ - -.. currentmodule:: logging.handlers - -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` -module, is a :class:`FileHandler` which watches the file it is logging to. If -the file changes, it is closed and reopened using the file name. - -A file change can happen because of usage of programs such as *newsyslog* and -*logrotate* which perform log file rotation. This handler, intended for use -under Unix/Linux, watches the file to see if it has changed since the last emit. -(A file is deemed to have changed if its device or inode have changed.) If the -file has changed, the old file stream is closed, and the file opened to get a -new stream. - -This handler is not appropriate for use under Windows, because under Windows -open log files cannot be moved or renamed - logging opens the files with -exclusive locks - and so there is no need for such a handler. Furthermore, -*ST_INO* is not supported under Windows; :func:`~os.stat` always returns zero -for this value. - - -.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False) - - Returns a new instance of the :class:`WatchedFileHandler` class. The specified - file is opened and used as the stream for logging. If *mode* is not specified, - :const:`'a'` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. method:: reopenIfNeeded() - - Checks to see if the file has changed. If it has, the existing stream is - flushed and closed and the file opened again, typically as a precursor to - outputting the record to the file. - - .. versionadded:: 3.6 - - - .. method:: emit(record) - - Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to - reopen the file if it has changed. - -.. _base-rotating-handler: - -BaseRotatingHandler -^^^^^^^^^^^^^^^^^^^ - -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` -module, is the base class for the rotating file handlers, -:class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should -not need to instantiate this class, but it has attributes and methods you may -need to override. - -.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False) - - The parameters are as for :class:`FileHandler`. The attributes are: - - .. attribute:: namer - - If this attribute is set to a callable, the :meth:`rotation_filename` - method delegates to this callable. The parameters passed to the callable - are those passed to :meth:`rotation_filename`. - - .. note:: The namer function is called quite a few times during rollover, - so it should be as simple and as fast as possible. It should also - return the same output every time for a given input, otherwise the - rollover behaviour may not work as expected. - - .. versionadded:: 3.3 - - - .. attribute:: BaseRotatingHandler.rotator - - If this attribute is set to a callable, the :meth:`rotate` method - delegates to this callable. The parameters passed to the callable are - those passed to :meth:`rotate`. - - .. versionadded:: 3.3 - - .. method:: BaseRotatingHandler.rotation_filename(default_name) - - Modify the filename of a log file when rotating. - - This is provided so that a custom filename can be provided. - - The default implementation calls the 'namer' attribute of the handler, - if it's callable, passing the default name to it. If the attribute isn't - callable (the default is ``None``), the name is returned unchanged. - - :param default_name: The default name for the log file. - - .. versionadded:: 3.3 - - - .. method:: BaseRotatingHandler.rotate(source, dest) - - When rotating, rotate the current log. - - The default implementation calls the 'rotator' attribute of the handler, - if it's callable, passing the source and dest arguments to it. If the - attribute isn't callable (the default is ``None``), the source is simply - renamed to the destination. - - :param source: The source filename. This is normally the base - filename, e.g. 'test.log'. - :param dest: The destination filename. This is normally - what the source is rotated to, e.g. 'test.log.1'. - - .. versionadded:: 3.3 - -The reason the attributes exist is to save you having to subclass - you can use -the same callables for instances of :class:`RotatingFileHandler` and -:class:`TimedRotatingFileHandler`. If either the namer or rotator callable -raises an exception, this will be handled in the same way as any other -exception during an :meth:`emit` call, i.e. via the :meth:`handleError` method -of the handler. - -If you need to make more significant changes to rotation processing, you can -override the methods. - -For an example, see :ref:`cookbook-rotator-namer`. - - -.. _rotating-file-handler: - -RotatingFileHandler -^^^^^^^^^^^^^^^^^^^ - -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` -module, supports rotation of disk log files. - - -.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False) - - Returns a new instance of the :class:`RotatingFileHandler` class. The specified - file is opened and used as the stream for logging. If *mode* is not specified, - ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. - - You can use the *maxBytes* and *backupCount* values to allow the file to - :dfn:`rollover` at a predetermined size. When the size is about to be exceeded, - the file is closed and a new file is silently opened for output. Rollover occurs - whenever the current log file is nearly *maxBytes* in length; but if either of - *maxBytes* or *backupCount* is zero, rollover never occurs, so you generally want - to set *backupCount* to at least 1, and have a non-zero *maxBytes*. - When *backupCount* is non-zero, the system will save old log files by appending - the extensions '.1', '.2' etc., to the filename. For example, with a *backupCount* - of 5 and a base file name of :file:`app.log`, you would get :file:`app.log`, - :file:`app.log.1`, :file:`app.log.2`, up to :file:`app.log.5`. The file being - written to is always :file:`app.log`. When this file is filled, it is closed - and renamed to :file:`app.log.1`, and if files :file:`app.log.1`, - :file:`app.log.2`, etc. exist, then they are renamed to :file:`app.log.2`, - :file:`app.log.3` etc. respectively. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. method:: doRollover() - - Does a rollover, as described above. - - - .. method:: emit(record) - - Outputs the record to the file, catering for rollover as described - previously. - -.. _timed-rotating-file-handler: - -TimedRotatingFileHandler -^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain -timed intervals. - - -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None) - - Returns a new instance of the :class:`TimedRotatingFileHandler` class. The - specified file is opened and used as the stream for logging. On rotating it also - sets the filename suffix. Rotating happens based on the product of *when* and - *interval*. - - You can use the *when* to specify the type of *interval*. The list of possible - values is below. Note that they are not case sensitive. - - +----------------+----------------------------+-------------------------+ - | Value | Type of interval | If/how *atTime* is used | - +================+============================+=========================+ - | ``'S'`` | Seconds | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'M'`` | Minutes | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'H'`` | Hours | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'D'`` | Days | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'W0'-'W6'`` | Weekday (0=Monday) | Used to compute initial | - | | | rollover time | - +----------------+----------------------------+-------------------------+ - | ``'midnight'`` | Roll over at midnight, if | Used to compute initial | - | | *atTime* not specified, | rollover time | - | | else at time *atTime* | | - +----------------+----------------------------+-------------------------+ - - When using weekday-based rotation, specify 'W0' for Monday, 'W1' for - Tuesday, and so on up to 'W6' for Sunday. In this case, the value passed for - *interval* isn't used. - - The system will save old log files by appending extensions to the filename. - The extensions are date-and-time based, using the strftime format - ``%Y-%m-%d_%H-%M-%S`` or a leading portion thereof, depending on the - rollover interval. - - When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else - the current time, is used to compute when the next rotation will occur. - - If the *utc* argument is true, times in UTC will be used; otherwise - local time is used. - - If *backupCount* is nonzero, at most *backupCount* files - will be kept, and if more would be created when rollover occurs, the oldest - one is deleted. The deletion logic uses the interval to determine which - files to delete, so changing the interval may leave old files lying around. - - If *delay* is true, then file opening is deferred until the first call to - :meth:`emit`. - - If *atTime* is not ``None``, it must be a ``datetime.time`` instance which - specifies the time of day when rollover occurs, for the cases where rollover - is set to happen "at midnight" or "on a particular weekday". Note that in - these cases, the *atTime* value is effectively used to compute the *initial* - rollover, and subsequent rollovers would be calculated via the normal - interval calculation. - - .. note:: Calculation of the initial rollover time is done when the handler - is initialised. Calculation of subsequent rollover times is done only - when rollover occurs, and rollover occurs only when emitting output. If - this is not kept in mind, it might lead to some confusion. For example, - if an interval of "every minute" is set, that does not mean you will - always see log files with times (in the filename) separated by a minute; - if, during application execution, logging output is generated more - frequently than once a minute, *then* you can expect to see log files - with times separated by a minute. If, on the other hand, logging messages - are only output once every five minutes (say), then there will be gaps in - the file times corresponding to the minutes where no output (and hence no - rollover) occurred. - - .. versionchanged:: 3.4 - *atTime* parameter was added. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. method:: doRollover() - - Does a rollover, as described above. - - .. method:: emit(record) - - Outputs the record to the file, catering for rollover as described above. - - -.. _socket-handler: - -SocketHandler -^^^^^^^^^^^^^ - -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, -sends logging output to a network socket. The base class uses a TCP socket. - - -.. class:: SocketHandler(host, port) - - Returns a new instance of the :class:`SocketHandler` class intended to - communicate with a remote machine whose address is given by *host* and *port*. - - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. - - .. method:: close() - - Closes the socket. - - - .. method:: emit() - - Pickles the record's attribute dictionary and writes it to the socket in - binary format. If there is an error with the socket, silently drops the - packet. If the connection was previously lost, re-establishes the - connection. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. - - - .. method:: handleError() - - Handles an error which has occurred during :meth:`emit`. The most likely - cause is a lost connection. Closes the socket so that we can retry on the - next event. - - - .. method:: makeSocket() - - This is a factory method which allows subclasses to define the precise - type of socket they want. The default implementation creates a TCP socket - (:const:`socket.SOCK_STREAM`). - - - .. method:: makePickle(record) - - Pickles the record's attribute dictionary in binary format with a length - prefix, and returns it ready for transmission across the socket. - - Note that pickles aren't completely secure. If you are concerned about - security, you may want to override this method to implement a more secure - mechanism. For example, you can sign pickles using HMAC and then verify - them on the receiving end, or alternatively you can disable unpickling of - global objects on the receiving end. - - - .. method:: send(packet) - - Send a pickled string *packet* to the socket. This function allows for - partial sends which can happen when the network is busy. - - - .. method:: createSocket() - - Tries to create a socket; on failure, uses an exponential back-off - algorithm. On initial failure, the handler will drop the message it was - trying to send. When subsequent messages are handled by the same - instance, it will not try connecting until some time has passed. The - default parameters are such that the initial delay is one second, and if - after that delay the connection still can't be made, the handler will - double the delay each time up to a maximum of 30 seconds. - - This behaviour is controlled by the following handler attributes: - - * ``retryStart`` (initial delay, defaulting to 1.0 seconds). - * ``retryFactor`` (multiplier, defaulting to 2.0). - * ``retryMax`` (maximum delay, defaulting to 30.0 seconds). - - This means that if the remote listener starts up *after* the handler has - been used, you could lose messages (since the handler won't even attempt - a connection until the delay has elapsed, but just silently drop messages - during the delay period). - - -.. _datagram-handler: - -DatagramHandler -^^^^^^^^^^^^^^^ - -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` -module, inherits from :class:`SocketHandler` to support sending logging messages -over UDP sockets. - - -.. class:: DatagramHandler(host, port) - - Returns a new instance of the :class:`DatagramHandler` class intended to - communicate with a remote machine whose address is given by *host* and *port*. - - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a UDP socket is created. - - .. method:: emit() - - Pickles the record's attribute dictionary and writes it to the socket in - binary format. If there is an error with the socket, silently drops the - packet. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. - - - .. method:: makeSocket() - - The factory method of :class:`SocketHandler` is here overridden to create - a UDP socket (:const:`socket.SOCK_DGRAM`). - - - .. method:: send(s) - - Send a pickled string to a socket. - - -.. _syslog-handler: - -SysLogHandler -^^^^^^^^^^^^^ - -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a remote or local Unix syslog. - - -.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM) - - Returns a new instance of the :class:`SysLogHandler` class intended to - communicate with a remote Unix machine whose address is given by *address* in - the form of a ``(host, port)`` tuple. If *address* is not specified, - ``('localhost', 514)`` is used. The address is used to open a socket. An - alternative to providing a ``(host, port)`` tuple is providing an address as a - string, for example '/dev/log'. In this case, a Unix domain socket is used to - send the message to the syslog. If *facility* is not specified, - :const:`LOG_USER` is used. The type of socket opened depends on the - *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus - opens a UDP socket. To open a TCP socket (for use with the newer syslog - daemons such as rsyslog), specify a value of :const:`socket.SOCK_STREAM`. - - Note that if your server is not listening on UDP port 514, - :class:`SysLogHandler` may appear not to work. In that case, check what - address you should be using for a domain socket - it's system dependent. - For example, on Linux it's usually '/dev/log' but on OS/X it's - '/var/run/syslog'. You'll need to check your platform and use the - appropriate address (you may need to do this check at runtime if your - application needs to run on several platforms). On Windows, you pretty - much have to use the UDP option. - - .. versionchanged:: 3.2 - *socktype* was added. - - - .. method:: close() - - Closes the socket to the remote host. - - - .. method:: emit(record) - - The record is formatted, and then sent to the syslog server. If exception - information is present, it is *not* sent to the server. - - .. versionchanged:: 3.2.1 - (See: :issue:`12168`.) In earlier versions, the message sent to the - syslog daemons was always terminated with a NUL byte, because early - versions of these daemons expected a NUL terminated message - even - though it's not in the relevant specification (:rfc:`5424`). More recent - versions of these daemons don't expect the NUL byte but strip it off - if it's there, and even more recent daemons (which adhere more closely - to RFC 5424) pass the NUL byte on as part of the message. - - To enable easier handling of syslog messages in the face of all these - differing daemon behaviours, the appending of the NUL byte has been - made configurable, through the use of a class-level attribute, - ``append_nul``. This defaults to ``True`` (preserving the existing - behaviour) but can be set to ``False`` on a ``SysLogHandler`` instance - in order for that instance to *not* append the NUL terminator. - - .. versionchanged:: 3.3 - (See: :issue:`12419`.) In earlier versions, there was no facility for - an "ident" or "tag" prefix to identify the source of the message. This - can now be specified using a class-level attribute, defaulting to - ``""`` to preserve existing behaviour, but which can be overridden on - a ``SysLogHandler`` instance in order for that instance to prepend - the ident to every message handled. Note that the provided ident must - be text, not bytes, and is prepended to the message exactly as is. - - .. method:: encodePriority(facility, priority) - - Encodes the facility and priority into an integer. You can pass in strings - or integers - if strings are passed, internal mapping dictionaries are - used to convert them to integers. - - The symbolic ``LOG_`` values are defined in :class:`SysLogHandler` and - mirror the values defined in the ``sys/syslog.h`` header file. - - **Priorities** - - +--------------------------+---------------+ - | Name (string) | Symbolic value| - +==========================+===============+ - | ``alert`` | LOG_ALERT | - +--------------------------+---------------+ - | ``crit`` or ``critical`` | LOG_CRIT | - +--------------------------+---------------+ - | ``debug`` | LOG_DEBUG | - +--------------------------+---------------+ - | ``emerg`` or ``panic`` | LOG_EMERG | - +--------------------------+---------------+ - | ``err`` or ``error`` | LOG_ERR | - +--------------------------+---------------+ - | ``info`` | LOG_INFO | - +--------------------------+---------------+ - | ``notice`` | LOG_NOTICE | - +--------------------------+---------------+ - | ``warn`` or ``warning`` | LOG_WARNING | - +--------------------------+---------------+ - - **Facilities** - - +---------------+---------------+ - | Name (string) | Symbolic value| - +===============+===============+ - | ``auth`` | LOG_AUTH | - +---------------+---------------+ - | ``authpriv`` | LOG_AUTHPRIV | - +---------------+---------------+ - | ``cron`` | LOG_CRON | - +---------------+---------------+ - | ``daemon`` | LOG_DAEMON | - +---------------+---------------+ - | ``ftp`` | LOG_FTP | - +---------------+---------------+ - | ``kern`` | LOG_KERN | - +---------------+---------------+ - | ``lpr`` | LOG_LPR | - +---------------+---------------+ - | ``mail`` | LOG_MAIL | - +---------------+---------------+ - | ``news`` | LOG_NEWS | - +---------------+---------------+ - | ``syslog`` | LOG_SYSLOG | - +---------------+---------------+ - | ``user`` | LOG_USER | - +---------------+---------------+ - | ``uucp`` | LOG_UUCP | - +---------------+---------------+ - | ``local0`` | LOG_LOCAL0 | - +---------------+---------------+ - | ``local1`` | LOG_LOCAL1 | - +---------------+---------------+ - | ``local2`` | LOG_LOCAL2 | - +---------------+---------------+ - | ``local3`` | LOG_LOCAL3 | - +---------------+---------------+ - | ``local4`` | LOG_LOCAL4 | - +---------------+---------------+ - | ``local5`` | LOG_LOCAL5 | - +---------------+---------------+ - | ``local6`` | LOG_LOCAL6 | - +---------------+---------------+ - | ``local7`` | LOG_LOCAL7 | - +---------------+---------------+ - - .. method:: mapPriority(levelname) - - Maps a logging level name to a syslog priority name. - You may need to override this if you are using custom levels, or - if the default algorithm is not suitable for your needs. The - default algorithm maps ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR`` and - ``CRITICAL`` to the equivalent syslog names, and all other level - names to 'warning'. - -.. _nt-eventlog-handler: - -NTEventLogHandler -^^^^^^^^^^^^^^^^^ - -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` -module, supports sending logging messages to a local Windows NT, Windows 2000 or -Windows XP event log. Before you can use it, you need Mark Hammond's Win32 -extensions for Python installed. - - -.. class:: NTEventLogHandler(appname, dllname=None, logtype='Application') - - Returns a new instance of the :class:`NTEventLogHandler` class. The *appname* is - used to define the application name as it appears in the event log. An - appropriate registry entry is created using this name. The *dllname* should give - the fully qualified pathname of a .dll or .exe which contains message - definitions to hold in the log (if not specified, ``'win32service.pyd'`` is used - - this is installed with the Win32 extensions and contains some basic - placeholder message definitions. Note that use of these placeholders will make - your event logs big, as the entire message source is held in the log. If you - want slimmer logs, you have to pass in the name of your own .dll or .exe which - contains the message definitions you want to use in the event log). The - *logtype* is one of ``'Application'``, ``'System'`` or ``'Security'``, and - defaults to ``'Application'``. - - - .. method:: close() - - At this point, you can remove the application name from the registry as a - source of event log entries. However, if you do this, you will not be able - to see the events as you intended in the Event Log Viewer - it needs to be - able to access the registry to get the .dll name. The current version does - not do this. - - - .. method:: emit(record) - - Determines the message ID, event category and event type, and then logs - the message in the NT event log. - - - .. method:: getEventCategory(record) - - Returns the event category for the record. Override this if you want to - specify your own categories. This version returns 0. - - - .. method:: getEventType(record) - - Returns the event type for the record. Override this if you want to - specify your own types. This version does a mapping using the handler's - typemap attribute, which is set up in :meth:`__init__` to a dictionary - which contains mappings for :const:`DEBUG`, :const:`INFO`, - :const:`WARNING`, :const:`ERROR` and :const:`CRITICAL`. If you are using - your own levels, you will either need to override this method or place a - suitable dictionary in the handler's *typemap* attribute. - - - .. method:: getMessageID(record) - - Returns the message ID for the record. If you are using your own messages, - you could do this by having the *msg* passed to the logger being an ID - rather than a format string. Then, in here, you could use a dictionary - lookup to get the message ID. This version returns 1, which is the base - message ID in :file:`win32service.pyd`. - -.. _smtp-handler: - -SMTPHandler -^^^^^^^^^^^ - -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to an email address via SMTP. - - -.. class:: SMTPHandler(mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None, timeout=1.0) - - Returns a new instance of the :class:`SMTPHandler` class. The instance is - initialized with the from and to addresses and subject line of the email. The - *toaddrs* should be a list of strings. To specify a non-standard SMTP port, use - the (host, port) tuple format for the *mailhost* argument. If you use a string, - the standard SMTP port is used. If your SMTP server requires authentication, you - can specify a (username, password) tuple for the *credentials* argument. - - To specify the use of a secure protocol (TLS), pass in a tuple to the - *secure* argument. This will only be used when authentication credentials are - supplied. The tuple should be either an empty tuple, or a single-value tuple - with the name of a keyfile, or a 2-value tuple with the names of the keyfile - and certificate file. (This tuple is passed to the - :meth:`smtplib.SMTP.starttls` method.) - - A timeout can be specified for communication with the SMTP server using the - *timeout* argument. - - .. versionadded:: 3.3 - The *timeout* argument was added. - - .. method:: emit(record) - - Formats the record and sends it to the specified addressees. - - - .. method:: getSubject(record) - - If you want to specify a subject line which is record-dependent, override - this method. - -.. _memory-handler: - -MemoryHandler -^^^^^^^^^^^^^ - -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, -supports buffering of logging records in memory, periodically flushing them to a -:dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an -event of a certain severity or greater is seen. - -:class:`MemoryHandler` is a subclass of the more general -:class:`BufferingHandler`, which is an abstract class. This buffers logging -records in memory. Whenever each record is added to the buffer, a check is made -by calling :meth:`shouldFlush` to see if the buffer should be flushed. If it -should, then :meth:`flush` is expected to do the flushing. - - -.. class:: BufferingHandler(capacity) - - Initializes the handler with a buffer of the specified capacity. - - - .. method:: emit(record) - - Appends the record to the buffer. If :meth:`shouldFlush` returns true, - calls :meth:`flush` to process the buffer. - - - .. method:: flush() - - You can override this to implement custom flushing behavior. This version - just zaps the buffer to empty. - - - .. method:: shouldFlush(record) - - Returns true if the buffer is up to capacity. This method can be - overridden to implement custom flushing strategies. - - -.. class:: MemoryHandler(capacity, flushLevel=ERROR, target=None, flushOnClose=True) - - Returns a new instance of the :class:`MemoryHandler` class. The instance is - initialized with a buffer size of *capacity*. If *flushLevel* is not specified, - :const:`ERROR` is used. If no *target* is specified, the target will need to be - set using :meth:`setTarget` before this handler does anything useful. If - *flushOnClose* is specified as ``False``, then the buffer is *not* flushed when - the handler is closed. If not specified or specified as ``True``, the previous - behaviour of flushing the buffer will occur when the handler is closed. - - .. versionchanged:: 3.6 - The *flushOnClose* parameter was added. - - - .. method:: close() - - Calls :meth:`flush`, sets the target to ``None`` and clears the - buffer. - - - .. method:: flush() - - For a :class:`MemoryHandler`, flushing means just sending the buffered - records to the target, if there is one. The buffer is also cleared when - this happens. Override if you want different behavior. - - - .. method:: setTarget(target) - - Sets the target handler for this handler. - - - .. method:: shouldFlush(record) - - Checks for buffer full or a record at the *flushLevel* or higher. - - -.. _http-handler: - -HTTPHandler -^^^^^^^^^^^ - -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a Web server, using either ``GET`` or -``POST`` semantics. - - -.. class:: HTTPHandler(host, url, method='GET', secure=False, credentials=None, context=None) - - Returns a new instance of the :class:`HTTPHandler` class. The *host* can be - of the form ``host:port``, should you need to use a specific port number. If - no *method* is specified, ``GET`` is used. If *secure* is true, a HTTPS - connection will be used. The *context* parameter may be set to a - :class:`ssl.SSLContext` instance to configure the SSL settings used for the - HTTPS connection. If *credentials* is specified, it should be a 2-tuple - consisting of userid and password, which will be placed in a HTTP - 'Authorization' header using Basic authentication. If you specify - credentials, you should also specify secure=True so that your userid and - password are not passed in cleartext across the wire. - - .. versionchanged:: 3.5 - The *context* parameter was added. - - .. method:: mapLogRecord(record) - - Provides a dictionary, based on ``record``, which is to be URL-encoded - and sent to the web server. The default implementation just returns - ``record.__dict__``. This method can be overridden if e.g. only a - subset of :class:`~logging.LogRecord` is to be sent to the web server, or - if more specific customization of what's sent to the server is required. - - .. method:: emit(record) - - Sends the record to the Web server as a URL-encoded dictionary. The - :meth:`mapLogRecord` method is used to convert the record to the - dictionary to be sent. - - .. note:: Since preparing a record for sending it to a Web server is not - the same as a generic formatting operation, using - :meth:`~logging.Handler.setFormatter` to specify a - :class:`~logging.Formatter` for a :class:`HTTPHandler` has no effect. - Instead of calling :meth:`~logging.Handler.format`, this handler calls - :meth:`mapLogRecord` and then :func:`urllib.parse.urlencode` to encode the - dictionary in a form suitable for sending to a Web server. - - -.. _queue-handler: - - -QueueHandler -^^^^^^^^^^^^ - -.. versionadded:: 3.2 - -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a queue, such as those implemented in the -:mod:`queue` or :mod:`multiprocessing` modules. - -Along with the :class:`QueueListener` class, :class:`QueueHandler` can be used -to let handlers do their work on a separate thread from the one which does the -logging. This is important in Web applications and also other service -applications where threads servicing clients need to respond as quickly as -possible, while any potentially slow operations (such as sending an email via -:class:`SMTPHandler`) are done on a separate thread. - -.. class:: QueueHandler(queue) - - Returns a new instance of the :class:`QueueHandler` class. The instance is - initialized with the queue to send messages to. The queue can be any - queue-like object; it's used as-is by the :meth:`enqueue` method, which needs - to know how to send messages to it. - - - .. method:: emit(record) - - Enqueues the result of preparing the LogRecord. - - .. method:: prepare(record) - - Prepares a record for queuing. The object returned by this - method is enqueued. - - The base implementation formats the record to merge the message - and arguments, and removes unpickleable items from the record - in-place. - - You might want to override this method if you want to convert - the record to a dict or JSON string, or send a modified copy - of the record while leaving the original intact. - - .. method:: enqueue(record) - - Enqueues the record on the queue using ``put_nowait()``; you may - want to override this if you want to use blocking behaviour, or a - timeout, or a customized queue implementation. - - - -.. _queue-listener: - -QueueListener -^^^^^^^^^^^^^ - -.. versionadded:: 3.2 - -The :class:`QueueListener` class, located in the :mod:`logging.handlers` -module, supports receiving logging messages from a queue, such as those -implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The -messages are received from a queue in an internal thread and passed, on -the same thread, to one or more handlers for processing. While -:class:`QueueListener` is not itself a handler, it is documented here -because it works hand-in-hand with :class:`QueueHandler`. - -Along with the :class:`QueueHandler` class, :class:`QueueListener` can be used -to let handlers do their work on a separate thread from the one which does the -logging. This is important in Web applications and also other service -applications where threads servicing clients need to respond as quickly as -possible, while any potentially slow operations (such as sending an email via -:class:`SMTPHandler`) are done on a separate thread. - -.. class:: QueueListener(queue, *handlers, respect_handler_level=False) - - Returns a new instance of the :class:`QueueListener` class. The instance is - initialized with the queue to send messages to and a list of handlers which - will handle entries placed on the queue. The queue can be any queue-like - object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. If ``respect_handler_level`` is ``True``, - a handler's level is respected (compared with the level for the message) when - deciding whether to pass messages to that handler; otherwise, the behaviour - is as in previous Python versions - to always pass each message to each - handler. - - .. versionchanged:: 3.5 - The ``respect_handler_levels`` argument was added. - - .. method:: dequeue(block) - - Dequeues a record and return it, optionally blocking. - - The base implementation uses ``get()``. You may want to override this - method if you want to use timeouts or work with custom queue - implementations. - - .. method:: prepare(record) - - Prepare a record for handling. - - This implementation just returns the passed-in record. You may want to - override this method if you need to do any custom marshalling or - manipulation of the record before passing it to the handlers. - - .. method:: handle(record) - - Handle a record. - - This just loops through the handlers offering them the record - to handle. The actual object passed to the handlers is that which - is returned from :meth:`prepare`. - - .. method:: start() - - Starts the listener. - - This starts up a background thread to monitor the queue for - LogRecords to process. - - .. method:: stop() - - Stops the listener. - - This asks the thread to terminate, and then waits for it to do so. - Note that if you don't call this before your application exits, there - may be some records still left on the queue, which won't be processed. - - .. method:: enqueue_sentinel() - - Writes a sentinel to the queue to tell the listener to quit. This - implementation uses ``put_nowait()``. You may want to override this - method if you want to use timeouts or work with custom queue - implementations. - - .. versionadded:: 3.3 - - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - diff --git a/third_party/python/Doc/library/logging.rst b/third_party/python/Doc/library/logging.rst deleted file mode 100644 index df74688f1..000000000 --- a/third_party/python/Doc/library/logging.rst +++ /dev/null @@ -1,1278 +0,0 @@ -:mod:`logging` --- Logging facility for Python -============================================== - -.. module:: logging - :synopsis: Flexible event logging system for applications. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/__init__.py` - -.. index:: pair: Errors; logging - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This module defines functions and classes which implement a flexible event -logging system for applications and libraries. - -The key benefit of having the logging API provided by a standard library module -is that all Python modules can participate in logging, so your application log -can include your own messages integrated with messages from third-party -modules. - -The module provides a lot of functionality and flexibility. If you are -unfamiliar with logging, the best way to get to grips with it is to see the -tutorials (see the links on the right). - -The basic classes defined by the module, together with their functions, are -listed below. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - - -.. _logger: - -Logger Objects --------------- - -Loggers have the following attributes and methods. Note that Loggers are never -instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - - -.. class:: Logger - - .. attribute:: Logger.propagate - - If this attribute evaluates to true, events logged to this logger will be - passed to the handlers of higher level (ancestor) loggers, in addition to - any handlers attached to this logger. Messages are passed directly to the - ancestor loggers' handlers - neither the level nor filters of the ancestor - loggers in question are considered. - - If this evaluates to false, logging messages are not passed to the handlers - of ancestor loggers. - - The constructor sets this attribute to ``True``. - - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. - - .. method:: Logger.setLevel(level) - - Sets the threshold for this logger to *level*. Logging messages which are less - severe than *level* will be ignored; logging messages which have severity *level* - or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *level*. - - When a logger is created, the level is set to :const:`NOTSET` (which causes - all messages to be processed when the logger is the root logger, or delegation - to the parent when the logger is a non-root logger). Note that the root logger - is created with level :const:`WARNING`. - - The term 'delegation to the parent' means that if a logger has a level of - NOTSET, its chain of ancestor loggers is traversed until either an ancestor with - a level other than NOTSET is found, or the root is reached. - - If an ancestor is found with a level other than NOTSET, then that ancestor's - level is treated as the effective level of the logger where the ancestor search - began, and is used to determine how a logging event is handled. - - If the root is reached, and it has a level of NOTSET, then all messages will be - processed. Otherwise, the root's level will be used as the effective level. - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. - - - .. method:: Logger.isEnabledFor(lvl) - - Indicates if a message of severity *lvl* would be processed by this logger. - This method checks first the module-level level set by - ``logging.disable(lvl)`` and then the logger's effective level as determined - by :meth:`getEffectiveLevel`. - - - .. method:: Logger.getEffectiveLevel() - - Indicates the effective level for this logger. If a value other than - :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, - the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. - - - .. method:: Logger.getChild(suffix) - - Returns a logger which is a descendant to this logger, as determined by the suffix. - Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same - logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a - convenience method, useful when the parent logger is named using e.g. ``__name__`` - rather than a literal string. - - .. versionadded:: 3.2 - - - .. method:: Logger.debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on this logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logger = logging.getLogger('tcpserver') - logger.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionadded:: 3.2 - The *stack_info* parameter was added. - - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - - - .. method:: Logger.info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on this logger. The arguments are - interpreted as for :meth:`debug`. - - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - .. method:: Logger.error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.log(lvl, msg, *args, **kwargs) - - Logs a message with integer level *lvl* on this logger. The other arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. Exception info is added to the logging - message. This method should only be called from an exception handler. - - - .. method:: Logger.addFilter(filter) - - Adds the specified filter *filter* to this logger. - - - .. method:: Logger.removeFilter(filter) - - Removes the specified filter *filter* from this logger. - - - .. method:: Logger.filter(record) - - Applies this logger's filters to the record and returns a true value if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. - - - .. method:: Logger.addHandler(hdlr) - - Adds the specified handler *hdlr* to this logger. - - - .. method:: Logger.removeHandler(hdlr) - - Removes the specified handler *hdlr* from this logger. - - - .. method:: Logger.findCaller(stack_info=False) - - Finds the caller's source filename and line number. Returns the filename, line - number, function name and stack information as a 4-element tuple. The stack - information is returned as ``None`` unless *stack_info* is ``True``. - - - .. method:: Logger.handle(record) - - Handles a record by passing it to all handlers associated with this logger and - its ancestors (until a false value of *propagate* is found). This method is used - for unpickled records received from a socket, as well as those created locally. - Logger-level filtering is applied using :meth:`~Logger.filter`. - - - .. method:: Logger.makeRecord(name, lvl, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) - - This is a factory method which can be overridden in subclasses to create - specialized :class:`LogRecord` instances. - - .. method:: Logger.hasHandlers() - - Checks to see if this logger has any handlers configured. This is done by - looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching - up the hierarchy whenever a logger with the 'propagate' attribute set to - false is found - that will be the last logger which is checked for the - existence of handlers. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Loggers can now be picked and unpickled. - -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - - -.. _handler: - -Handler Objects ---------------- - -Handlers have the following attributes and methods. Note that :class:`Handler` -is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call -:meth:`Handler.__init__`. - -.. class:: Handler - - .. method:: Handler.__init__(level=NOTSET) - - Initializes the :class:`Handler` instance by setting its level, setting the list - of filters to the empty list and creating a lock (using :meth:`createLock`) for - serializing access to an I/O mechanism. - - - .. method:: Handler.createLock() - - Initializes a thread lock which can be used to serialize access to underlying - I/O functionality which may not be threadsafe. - - - .. method:: Handler.acquire() - - Acquires the thread lock created with :meth:`createLock`. - - - .. method:: Handler.release() - - Releases the thread lock acquired with :meth:`acquire`. - - - .. method:: Handler.setLevel(level) - - Sets the threshold for this handler to *level*. Logging messages which are - less severe than *level* will be ignored. When a handler is created, the - level is set to :const:`NOTSET` (which causes all messages to be - processed). - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. - - - .. method:: Handler.setFormatter(fmt) - - Sets the :class:`Formatter` for this handler to *fmt*. - - - .. method:: Handler.addFilter(filter) - - Adds the specified filter *filter* to this handler. - - - .. method:: Handler.removeFilter(filter) - - Removes the specified filter *filter* from this handler. - - - .. method:: Handler.filter(record) - - Applies this handler's filters to the record and returns a true value if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. - - - .. method:: Handler.flush() - - Ensure all logging output has been flushed. This version does nothing and is - intended to be implemented by subclasses. - - - .. method:: Handler.close() - - Tidy up any resources used by the handler. This version does no output but - removes the handler from an internal list of handlers which is closed when - :func:`shutdown` is called. Subclasses should ensure that this gets called - from overridden :meth:`close` methods. - - - .. method:: Handler.handle(record) - - Conditionally emits the specified logging record, depending on filters which may - have been added to the handler. Wraps the actual emission of the record with - acquisition/release of the I/O thread lock. - - - .. method:: Handler.handleError(record) - - This method should be called from handlers when an exception is encountered - during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is - what is mostly wanted for a logging system - most users will not care about - errors in the logging system, they are more interested in application - errors. You could, however, replace this with a custom handler if you wish. - The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is - more useful during development). - - - .. method:: Handler.format(record) - - Do formatting for a record - if a formatter is set, use it. Otherwise, use the - default formatter for the module. - - - .. method:: Handler.emit(record) - - Do whatever it takes to actually log the specified logging record. This version - is intended to be implemented by subclasses and so raises a - :exc:`NotImplementedError`. - -For a list of handlers included as standard, see :mod:`logging.handlers`. - -.. _formatter-objects: - -Formatter Objects ------------------ - -.. currentmodule:: logging - -:class:`Formatter` objects have the following attributes and methods. They are -responsible for converting a :class:`LogRecord` to (usually) a string which can -be interpreted by either a human or an external system. The base -:class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. - -A Formatter can be initialized with a format string which makes use of knowledge -of the :class:`LogRecord` attributes - such as the default value mentioned above -making use of the fact that the user's message and arguments are pre-formatted -into a :class:`LogRecord`'s *message* attribute. This format string contains -standard Python %-style mapping keys. See section :ref:`old-string-formatting` -for more information on string formatting. - -The useful mapping keys in a :class:`LogRecord` are given in the section on -:ref:`logrecord-attributes`. - - -.. class:: Formatter(fmt=None, datefmt=None, style='%') - - Returns a new instance of the :class:`Formatter` class. The instance is - initialized with a format string for the message as a whole, as well as a - format string for the date/time portion of a message. If no *fmt* is - specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format - is used which is described in the :meth:`formatTime` documentation. - - The *style* parameter can be one of '%', '{' or '$' and determines how - the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. See :ref:`formatting-styles` - for more information on using {- and $-formatting for log messages. - - .. versionchanged:: 3.2 - The *style* parameter was added. - - - .. method:: format(record) - - The record's attribute dictionary is used as the operand to a string - formatting operation. Returns the resulting string. Before formatting the - dictionary, a couple of preparatory steps are carried out. The *message* - attribute of the record is computed using *msg* % *args*. If the - formatting string contains ``'(asctime)'``, :meth:`formatTime` is called - to format the event time. If there is exception information, it is - formatted using :meth:`formatException` and appended to the message. Note - that the formatted exception information is cached in attribute - *exc_text*. This is useful because the exception information can be - pickled and sent across the wire, but you should be careful if you have - more than one :class:`Formatter` subclass which customizes the formatting - of exception information. In this case, you will have to clear the cached - value after a formatter has done its formatting, so that the next - formatter to handle the event doesn't use the cached value but - recalculates it afresh. - - If stack information is available, it's appended after the exception - information, using :meth:`formatStack` to transform it if necessary. - - - .. method:: formatTime(record, datefmt=None) - - This method should be called from :meth:`format` by a formatter which - wants to make use of a formatted time. This method can be overridden in - formatters to provide for any specific requirement, but the basic behavior - is as follows: if *datefmt* (a string) is specified, it is used with - :func:`time.strftime` to format the creation time of the - record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the - uuu part is a millisecond value and the other letters are as per the - :func:`time.strftime` documentation. An example time in this format is - ``2003-01-23 00:29:50,411``. The resulting string is returned. - - This function uses a user-configurable function to convert the creation - time to a tuple. By default, :func:`time.localtime` is used; to change - this for a particular formatter instance, set the ``converter`` attribute - to a function with the same signature as :func:`time.localtime` or - :func:`time.gmtime`. To change it for all formatters, for example if you - want all logging times to be shown in GMT, set the ``converter`` - attribute in the ``Formatter`` class. - - .. versionchanged:: 3.3 - Previously, the default format was hard-coded as in this example: - ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not - have a format placeholder for milliseconds, the millisecond value is - appended using another format string, ``'%s,%03d'`` --- and both of these - format strings have been hardcoded into this method. With the change, - these strings are defined as class-level attributes which can be - overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) - and ``default_msec_format`` (for appending the millisecond value). - - .. method:: formatException(exc_info) - - Formats the specified exception information (a standard exception tuple as - returned by :func:`sys.exc_info`) as a string. This default implementation - just uses :func:`traceback.print_exception`. The resulting string is - returned. - - .. method:: formatStack(stack_info) - - Formats the specified stack information (a string as returned by - :func:`traceback.print_stack`, but with the last newline removed) as a - string. This default implementation just returns the input value. - -.. _filter: - -Filter Objects --------------- - -``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated -filtering than is provided by levels. The base filter class only allows events -which are below a certain point in the logger hierarchy. For example, a filter -initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', -'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the -empty string, all events are passed. - - -.. class:: Filter(name='') - - Returns an instance of the :class:`Filter` class. If *name* is specified, it - names a logger which, together with its children, will have its events allowed - through the filter. If *name* is the empty string, allows every event. - - - .. method:: filter(record) - - Is the specified record to be logged? Returns zero for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place by this - method. - -Note that filters attached to handlers are consulted before an event is -emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. - -You don't actually need to subclass ``Filter``: you can pass any instance -which has a ``filter`` method with the same semantics. - -.. versionchanged:: 3.2 - You don't need to create specialized ``Filter`` classes, or use other - classes with a ``filter`` method: you can use a function (or other - callable) as a filter. The filtering logic will check to see if the filter - object has a ``filter`` attribute: if it does, it's assumed to be a - ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's - assumed to be a callable and called with the record as the single - parameter. The returned value should conform to that returned by - :meth:`~Filter.filter`. - -Although filters are used primarily to filter records based on more -sophisticated criteria than levels, they get to see every record which is -processed by the handler or logger they're attached to: this can be useful if -you want to do things like counting how many records were processed by a -particular logger or handler, or adding, changing or removing attributes in -the LogRecord being processed. Obviously changing the LogRecord needs to be -done with some care, but it does allow the injection of contextual information -into logs (see :ref:`filters-contextual`). - -.. _log-record: - -LogRecord Objects ------------------ - -:class:`LogRecord` instances are created automatically by the :class:`Logger` -every time something is logged, and can be created manually via -:func:`makeLogRecord` (for example, from a pickled event received over the -wire). - - -.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) - - Contains all the information pertinent to the event being logged. - - The primary information is passed in :attr:`msg` and :attr:`args`, which - are combined using ``msg % args`` to create the :attr:`message` field of the - record. - - :param name: The name of the logger used to log the event represented by - this LogRecord. Note that this name will always have this - value, even though it may be emitted by a handler attached to - a different (ancestor) logger. - :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) - Note that this is converted to *two* attributes of the LogRecord: - ``levelno`` for the numeric value and ``levelname`` for the - corresponding level name. - :param pathname: The full pathname of the source file where the logging call - was made. - :param lineno: The line number in the source file where the logging call was - made. - :param msg: The event description message, possibly a format string with - placeholders for variable data. - :param args: Variable data to merge into the *msg* argument to obtain the - event description. - :param exc_info: An exception tuple with the current exception information, - or ``None`` if no exception information is available. - :param func: The name of the function or method from which the logging call - was invoked. - :param sinfo: A text string representing stack information from the base of - the stack in the current thread, up to the logging call. - - .. method:: getMessage() - - Returns the message for this :class:`LogRecord` instance after merging any - user-supplied arguments with the message. If the user-supplied message - argument to the logging call is not a string, :func:`str` is called on it to - convert it to a string. This allows use of user-defined classes as - messages, whose ``__str__`` method can return the actual format string to - be used. - - .. versionchanged:: 3.2 - The creation of a ``LogRecord`` has been made more configurable by - providing a factory which is used to create the record. The factory can be - set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` - (see this for the factory's signature). - - This functionality can be used to inject your own values into a - LogRecord at creation time. You can use the following pattern:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - - With this pattern, multiple factories could be chained, and as long - as they don't overwrite each other's attributes or unintentionally - overwrite the standard attributes listed above, there should be no - surprises. - - -.. _logrecord-attributes: - -LogRecord attributes --------------------- - -The LogRecord has a number of attributes, most of which are derived from the -parameters to the constructor. (Note that the names do not always correspond -exactly between the LogRecord constructor parameters and the LogRecord -attributes.) These attributes can be used to merge data from the record into -the format string. The following table lists (in alphabetical order) the -attribute names, their meanings and the corresponding placeholder in a %-style -format string. - -If you are using {}-formatting (:func:`str.format`), you can use -``{attrname}`` as the placeholder in the format string. If you are using -$-formatting (:class:`string.Template`), use the form ``${attrname}``. In -both cases, of course, replace ``attrname`` with the actual attribute name -you want to use. - -In the case of {}-formatting, you can specify formatting flags by placing them -after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as -``004``. Refer to the :meth:`str.format` documentation for full details on -the options available to you. - -+----------------+-------------------------+-----------------------------------------------+ -| Attribute name | Format | Description | -+================+=========================+===============================================+ -| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | -+----------------+-------------------------+-----------------------------------------------+ -| asctime | ``%(asctime)s`` | Human-readable time when the | -| | | :class:`LogRecord` was created. By default | -| | | this is of the form '2003-07-08 16:49:45,896' | -| | | (the numbers after the comma are millisecond | -| | | portion of the time). | -+----------------+-------------------------+-----------------------------------------------+ -| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | -+----------------+-------------------------+-----------------------------------------------+ -| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | -| | format this yourself. | if no exception has occurred, ``None``. | -+----------------+-------------------------+-----------------------------------------------+ -| filename | ``%(filename)s`` | Filename portion of ``pathname``. | -+----------------+-------------------------+-----------------------------------------------+ -| funcName | ``%(funcName)s`` | Name of function containing the logging call. | -+----------------+-------------------------+-----------------------------------------------+ -| levelname | ``%(levelname)s`` | Text logging level for the message | -| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | -| | | ``'ERROR'``, ``'CRITICAL'``). | -+----------------+-------------------------+-----------------------------------------------+ -| levelno | ``%(levelno)s`` | Numeric logging level for the message | -| | | (:const:`DEBUG`, :const:`INFO`, | -| | | :const:`WARNING`, :const:`ERROR`, | -| | | :const:`CRITICAL`). | -+----------------+-------------------------+-----------------------------------------------+ -| lineno | ``%(lineno)d`` | Source line number where the logging call was | -| | | issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| message | ``%(message)s`` | The logged message, computed as ``msg % | -| | | args``. This is set when | -| | | :meth:`Formatter.format` is invoked. | -+----------------+-------------------------+-----------------------------------------------+ -| module | ``%(module)s`` | Module (name portion of ``filename``). | -+----------------+-------------------------+-----------------------------------------------+ -| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | -| | | :class:`LogRecord` was created. | -+----------------+-------------------------+-----------------------------------------------+ -| msg | You shouldn't need to | The format string passed in the original | -| | format this yourself. | logging call. Merged with ``args`` to | -| | | produce ``message``, or an arbitrary object | -| | | (see :ref:`arbitrary-object-messages`). | -+----------------+-------------------------+-----------------------------------------------+ -| name | ``%(name)s`` | Name of the logger used to log the call. | -+----------------+-------------------------+-----------------------------------------------+ -| pathname | ``%(pathname)s`` | Full pathname of the source file where the | -| | | logging call was issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| process | ``%(process)d`` | Process ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| processName | ``%(processName)s`` | Process name (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | -| | | created, relative to the time the logging | -| | | module was loaded. | -+----------------+-------------------------+-----------------------------------------------+ -| stack_info | You shouldn't need to | Stack frame information (where available) | -| | format this yourself. | from the bottom of the stack in the current | -| | | thread, up to and including the stack frame | -| | | of the logging call which resulted in the | -| | | creation of this record. | -+----------------+-------------------------+-----------------------------------------------+ -| thread | ``%(thread)d`` | Thread ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| threadName | ``%(threadName)s`` | Thread name (if available). | -+----------------+-------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.1 - *processName* was added. - - -.. _logger-adapter: - -LoggerAdapter Objects ---------------------- - -:class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on -:ref:`adding contextual information to your logging output `. - -.. class:: LoggerAdapter(logger, extra) - - Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. - - .. method:: process(msg, kwargs) - - Modifies the message and/or keyword arguments passed to a logging call in - order to insert contextual information. This implementation takes the object - passed as *extra* to the constructor and adds it to *kwargs* using key - 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the - (possibly modified) versions of the arguments passed in. - -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. - -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. - - -Thread Safety -------------- - -The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading -locks; there is one lock to serialize access to the module's shared data, and -each handler also creates a lock to serialize access to its underlying I/O. - -If you are implementing asynchronous signal handlers using the :mod:`signal` -module, you may not be able to use logging from within such handlers. This is -because lock implementations in the :mod:`threading` module are not always -re-entrant, and so cannot be invoked from such signal handlers. - - -Module-Level Functions ----------------------- - -In addition to the classes described above, there are a number of module-level -functions. - - -.. function:: getLogger(name=None) - - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. - - All calls to this function with a given name return the same logger instance. - This means that logger instances never need to be passed between different parts - of an application. - - -.. function:: getLoggerClass() - - Return either the standard :class:`Logger` class, or the last class passed to - :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: - - class MyLogger(logging.getLoggerClass()): - # ... override behaviour here - - -.. function:: getLogRecordFactory() - - Return a callable which is used to create a :class:`LogRecord`. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`setLogRecordFactory`, - to allow developers more control over how the :class:`LogRecord` - representing a logging event is constructed. - - See :func:`setLogRecordFactory` for more information about the how the - factory is called. - -.. function:: debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionadded:: 3.2 - The *stack_info* parameter was added. - -.. function:: info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. - - .. note:: There is an obsolete function ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - -.. function:: error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. - - -.. function:: exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging - message. This function should only be called from an exception handler. - -.. function:: log(level, msg, *args, **kwargs) - - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. - - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. - -.. function:: disable(lvl=CRITICAL) - - Provides an overriding level *lvl* for all loggers which takes precedence over - the logger's own level. When the need arises to temporarily throttle logging - output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *lvl* and below, so that - if you call it with a value of INFO, then all INFO and DEBUG events would be - discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. - - Note that if you have defined any custom logging level higher than - ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *lvl* parameter, but will have to explicitly supply a - suitable value. - - .. versionchanged:: 3.7 - The *lvl* parameter was defaulted to level ``CRITICAL``. See Issue - #28524 for more information about this change. - -.. function:: addLevelName(lvl, levelName) - - Associates level *lvl* with text *levelName* in an internal dictionary, which is - used to map numeric levels to a textual representation, for example when a - :class:`Formatter` formats a message. This function can also be used to define - your own levels. The only constraints are that all levels used must be - registered using this function, levels should be positive integers and they - should increase in increasing order of severity. - - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. - -.. function:: getLevelName(lvl) - - Returns the textual representation of logging level *lvl*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *lvl* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % lvl is returned. - - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. - -.. function:: makeLogRecord(attrdict) - - Creates and returns a new :class:`LogRecord` instance whose attributes are - defined by *attrdict*. This function is useful for taking a pickled - :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting - it as a :class:`LogRecord` instance at the receiving end. - - -.. function:: basicConfig(**kwargs) - - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the - root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, - :func:`error` and :func:`critical` will call :func:`basicConfig` automatically - if no handlers are defined for the root logger. - - This function does nothing if the root logger already has handlers - configured for it. - - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. - - The following keyword arguments are supported. - - .. tabularcolumns:: |l|L| - - +--------------+---------------------------------------------+ - | Format | Description | - +==============+=============================================+ - | *filename* | Specifies that a FileHandler be created, | - | | using the specified filename, rather than a | - | | StreamHandler. | - +--------------+---------------------------------------------+ - | *filemode* | If *filename* is specified, open the file | - | | in this :ref:`mode `. Defaults | - | | to ``'a'``. | - +--------------+---------------------------------------------+ - | *format* | Use the specified format string for the | - | | handler. | - +--------------+---------------------------------------------+ - | *datefmt* | Use the specified date/time format, as | - | | accepted by :func:`time.strftime`. | - +--------------+---------------------------------------------+ - | *style* | If *format* is specified, use this style | - | | for the format string. One of ``'%'``, | - | | ``'{'`` or ``'$'`` for :ref:`printf-style | - | | `, | - | | :meth:`str.format` or | - | | :class:`string.Template` respectively. | - | | Defaults to ``'%'``. | - +--------------+---------------------------------------------+ - | *level* | Set the root logger level to the specified | - | | :ref:`level `. | - +--------------+---------------------------------------------+ - | *stream* | Use the specified stream to initialize the | - | | StreamHandler. Note that this argument is | - | | incompatible with *filename* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - | *handlers* | If specified, this should be an iterable of | - | | already created handlers to add to the root | - | | logger. Any handlers which don't already | - | | have a formatter set will be assigned the | - | | default formatter created in this function. | - | | Note that this argument is incompatible | - | | with *filename* or *stream* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - - .. versionchanged:: 3.2 - The *style* argument was added. - - .. versionchanged:: 3.3 - The *handlers* argument was added. Additional checks were added to - catch situations where incompatible arguments are specified (e.g. - *handlers* together with *stream* or *filename*, or *stream* - together with *filename*). - -.. function:: shutdown() - - Informs the logging system to perform an orderly shutdown by flushing and - closing all handlers. This should be called at application exit and no - further use of the logging system should be made after this call. - - -.. function:: setLoggerClass(klass) - - Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This - function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. - - -.. function:: setLogRecordFactory(factory) - - Set a callable which is used to create a :class:`LogRecord`. - - :param factory: The factory callable to be used to instantiate a log record. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`getLogRecordFactory`, to - allow developers more control over how the :class:`LogRecord` representing - a logging event is constructed. - - The factory has the following signature: - - ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` - - :name: The logger name. - :level: The logging level (numeric). - :fn: The full pathname of the file where the logging call was made. - :lno: The line number in the file where the logging call was made. - :msg: The logging message. - :args: The arguments for the logging message. - :exc_info: An exception tuple, or ``None``. - :func: The name of the function or method which invoked the logging - call. - :sinfo: A stack traceback such as is provided by - :func:`traceback.print_stack`, showing the call hierarchy. - :kwargs: Additional keyword arguments. - - -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - -Integration with the warnings module ------------------------------------- - -The :func:`captureWarnings` function can be used to integrate :mod:`logging` -with the :mod:`warnings` module. - -.. function:: captureWarnings(capture) - - This function is used to turn the capture of warnings by logging on and - off. - - If *capture* is ``True``, warnings issued by the :mod:`warnings` module will - be redirected to the logging system. Specifically, a warning will be - formatted using :func:`warnings.formatwarning` and the resulting string - logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. - - If *capture* is ``False``, the redirection of warnings to the logging system - will stop, and warnings will be redirected to their original destinations - (i.e. those in effect before ``captureWarnings(True)`` was called). - - -.. seealso:: - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :pep:`282` - A Logging System - The proposal which described this feature for inclusion in the Python standard - library. - - `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the - package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard - library. diff --git a/third_party/python/Doc/library/lzma.rst b/third_party/python/Doc/library/lzma.rst deleted file mode 100644 index cce6c23e6..000000000 --- a/third_party/python/Doc/library/lzma.rst +++ /dev/null @@ -1,434 +0,0 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== - -.. module:: lzma - :synopsis: A Python wrapper for the liblzma compression library. - -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Nadeem Vawda - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/lzma.py` - --------------- - -This module provides classes and convenience functions for compressing and -decompressing data using the LZMA compression algorithm. Also included is a file -interface supporting the ``.xz`` and legacy ``.lzma`` file formats used by the -:program:`xz` utility, as well as raw compressed streams. - -The interface provided by this module is very similar to that of the :mod:`bz2` -module. However, note that :class:`LZMAFile` is *not* thread-safe, unlike -:class:`bz2.BZ2File`, so if you need to use a single :class:`LZMAFile` instance -from multiple threads, it is necessary to protect it with a lock. - - -.. exception:: LZMAError - - This exception is raised when an error occurs during compression or - decompression, or while initializing the compressor/decompressor state. - - -Reading and writing compressed files ------------------------------------- - -.. function:: open(filename, mode="rb", \*, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) - - Open an LZMA-compressed file in binary or text mode, returning a :term:`file - object`. - - The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like ` object), in - which case the named file is opened, or it can be an existing file object - to read from or write to. - - The *mode* argument can be any of ``"r"``, ``"rb"``, ``"w"``, ``"wb"``, - ``"x"``, ``"xb"``, ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, - ``"wt"``, ``"xt"``, or ``"at"`` for text mode. The default is ``"rb"``. - - When opening a file for reading, the *format* and *filters* arguments have - the same meanings as for :class:`LZMADecompressor`. In this case, the *check* - and *preset* arguments should not be used. - - When opening a file for writing, the *format*, *check*, *preset* and - *filters* arguments have the same meanings as for :class:`LZMACompressor`. - - For binary mode, this function is equivalent to the :class:`LZMAFile` - constructor: ``LZMAFile(filename, mode, ...)``. In this case, the *encoding*, - *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`LZMAFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.4 - Added support for the ``"x"``, ``"xb"`` and ``"xt"`` modes. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None) - - Open an LZMA-compressed file in binary mode. - - An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate - directly on a named file. The *filename* argument specifies either the file - object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like ` object). When wrapping an - existing file object, the wrapped file will not be closed when the - :class:`LZMAFile` is closed. - - The *mode* argument can be either ``"r"`` for reading (default), ``"w"`` for - overwriting, ``"x"`` for exclusive creation, or ``"a"`` for appending. These - can equivalently be given as ``"rb"``, ``"wb"``, ``"xb"`` and ``"ab"`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of - ``"w"`` does not truncate the file, and is instead equivalent to ``"a"``. - - When opening a file for reading, the input file may be the concatenation of - multiple separate compressed streams. These are transparently decoded as a - single logical stream. - - When opening a file for reading, the *format* and *filters* arguments have - the same meanings as for :class:`LZMADecompressor`. In this case, the *check* - and *preset* arguments should not be used. - - When opening a file for writing, the *format*, *check*, *preset* and - *filters* arguments have the same meanings as for :class:`LZMACompressor`. - - :class:`LZMAFile` supports all the members specified by - :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`. - Iteration and the :keyword:`with` statement are supported. - - The following method is also provided: - - .. method:: peek(size=-1) - - Return buffered data without advancing the file position. At least one - byte of data will be returned, unless EOF has been reached. The exact - number of bytes returned is unspecified (the *size* argument is ignored). - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`LZMAFile`, it may change the position of the underlying - file object (e.g. if the :class:`LZMAFile` was constructed by passing a - file object for *filename*). - - .. versionchanged:: 3.4 - Added support for the ``"x"`` and ``"xb"`` modes. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -Compressing and decompressing data in memory --------------------------------------------- - -.. class:: LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None) - - Create a compressor object, which can be used to compress data incrementally. - - For a more convenient way of compressing a single chunk of data, see - :func:`compress`. - - The *format* argument specifies what container format should be used. - Possible values are: - - * :const:`FORMAT_XZ`: The ``.xz`` container format. - This is the default format. - - * :const:`FORMAT_ALONE`: The legacy ``.lzma`` container format. - This format is more limited than ``.xz`` -- it does not support integrity - checks or multiple filters. - - * :const:`FORMAT_RAW`: A raw data stream, not using any container format. - This format specifier does not support integrity checks, and requires that - you always specify a custom filter chain (for both compression and - decompression). Additionally, data compressed in this manner cannot be - decompressed using :const:`FORMAT_AUTO` (see :class:`LZMADecompressor`). - - The *check* argument specifies the type of integrity check to include in the - compressed data. This check is used when decompressing, to ensure that the - data has not been corrupted. Possible values are: - - * :const:`CHECK_NONE`: No integrity check. - This is the default (and the only acceptable value) for - :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. - - * :const:`CHECK_CRC32`: 32-bit Cyclic Redundancy Check. - - * :const:`CHECK_CRC64`: 64-bit Cyclic Redundancy Check. - This is the default for :const:`FORMAT_XZ`. - - * :const:`CHECK_SHA256`: 256-bit Secure Hash Algorithm. - - If the specified check is not supported, an :class:`LZMAError` is raised. - - The compression settings can be specified either as a preset compression - level (with the *preset* argument), or in detail as a custom filter chain - (with the *filters* argument). - - The *preset* argument (if provided) should be an integer between ``0`` and - ``9`` (inclusive), optionally OR-ed with the constant - :const:`PRESET_EXTREME`. If neither *preset* nor *filters* are given, the - default behavior is to use :const:`PRESET_DEFAULT` (preset level ``6``). - Higher presets produce smaller output, but make the compression process - slower. - - .. note:: - - In addition to being more CPU-intensive, compression with higher presets - also requires much more memory (and produces output that needs more memory - to decompress). With preset ``9`` for example, the overhead for an - :class:`LZMACompressor` object can be as high as 800 MiB. For this reason, - it is generally best to stick with the default preset. - - The *filters* argument (if provided) should be a filter chain specifier. - See :ref:`filter-chain-specs` for details. - - .. method:: compress(data) - - Compress *data* (a :class:`bytes` object), returning a :class:`bytes` - object containing compressed data for at least part of the input. Some of - *data* may be buffered internally, for use in later calls to - :meth:`compress` and :meth:`flush`. The returned data should be - concatenated with the output of any previous calls to :meth:`compress`. - - .. method:: flush() - - Finish the compression process, returning a :class:`bytes` object - containing any data stored in the compressor's internal buffers. - - The compressor cannot be used after this method has been called. - - -.. class:: LZMADecompressor(format=FORMAT_AUTO, memlimit=None, filters=None) - - Create a decompressor object, which can be used to decompress data - incrementally. - - For a more convenient way of decompressing an entire compressed stream at - once, see :func:`decompress`. - - The *format* argument specifies the container format that should be used. The - default is :const:`FORMAT_AUTO`, which can decompress both ``.xz`` and - ``.lzma`` files. Other possible values are :const:`FORMAT_XZ`, - :const:`FORMAT_ALONE`, and :const:`FORMAT_RAW`. - - The *memlimit* argument specifies a limit (in bytes) on the amount of memory - that the decompressor can use. When this argument is used, decompression will - fail with an :class:`LZMAError` if it is not possible to decompress the input - within the given memory limit. - - The *filters* argument specifies the filter chain that was used to create - the stream being decompressed. This argument is required if *format* is - :const:`FORMAT_RAW`, but should not be used for other formats. - See :ref:`filter-chain-specs` for more information about filter chains. - - .. note:: - This class does not transparently handle inputs containing multiple - compressed streams, unlike :func:`decompress` and :class:`LZMAFile`. To - decompress a multi-stream input with :class:`LZMADecompressor`, you must - create a new decompressor for each stream. - - .. method:: decompress(data, max_length=-1) - - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. - - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. - - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an `EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. - - .. attribute:: check - - The ID of the integrity check used by the input stream. This may be - :const:`CHECK_UNKNOWN` until enough of the input has been decoded to - determine what integrity check it uses. - - .. attribute:: eof - - ``True`` if the end-of-stream marker has been reached. - - .. attribute:: unused_data - - Data found after the end of the compressed stream. - - Before the end of the stream is reached, this will be ``b""``. - - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 - -.. function:: compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None) - - Compress *data* (a :class:`bytes` object), returning the compressed data as a - :class:`bytes` object. - - See :class:`LZMACompressor` above for a description of the *format*, *check*, - *preset* and *filters* arguments. - - -.. function:: decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None) - - Decompress *data* (a :class:`bytes` object), returning the uncompressed data - as a :class:`bytes` object. - - If *data* is the concatenation of multiple distinct compressed streams, - decompress all of these streams, and return the concatenation of the results. - - See :class:`LZMADecompressor` above for a description of the *format*, - *memlimit* and *filters* arguments. - - -Miscellaneous -------------- - -.. function:: is_check_supported(check) - - Returns true if the given integrity check is supported on this system. - - :const:`CHECK_NONE` and :const:`CHECK_CRC32` are always supported. - :const:`CHECK_CRC64` and :const:`CHECK_SHA256` may be unavailable if you are - using a version of :program:`liblzma` that was compiled with a limited - feature set. - - -.. _filter-chain-specs: - -Specifying custom filter chains -------------------------------- - -A filter chain specifier is a sequence of dictionaries, where each dictionary -contains the ID and options for a single filter. Each dictionary must contain -the key ``"id"``, and may contain additional keys to specify filter-dependent -options. Valid filter IDs are as follows: - -* Compression filters: - * :const:`FILTER_LZMA1` (for use with :const:`FORMAT_ALONE`) - * :const:`FILTER_LZMA2` (for use with :const:`FORMAT_XZ` and :const:`FORMAT_RAW`) - -* Delta filter: - * :const:`FILTER_DELTA` - -* Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` - -A filter chain can consist of up to 4 filters, and cannot be empty. The last -filter in the chain must be a compression filter, and any other filters must be -delta or BCJ filters. - -Compression filters support the following options (specified as additional -entries in the dictionary representing the filter): - - * ``preset``: A compression preset to use as a source of default values for - options that are not specified explicitly. - * ``dict_size``: Dictionary size in bytes. This should be between 4 KiB and - 1.5 GiB (inclusive). - * ``lc``: Number of literal context bits. - * ``lp``: Number of literal position bits. The sum ``lc + lp`` must be at - most 4. - * ``pb``: Number of position bits; must be at most 4. - * ``mode``: :const:`MODE_FAST` or :const:`MODE_NORMAL`. - * ``nice_len``: What should be considered a "nice length" for a match. - This should be 273 or less. - * ``mf``: What match finder to use -- :const:`MF_HC3`, :const:`MF_HC4`, - :const:`MF_BT2`, :const:`MF_BT3`, or :const:`MF_BT4`. - * ``depth``: Maximum search depth used by match finder. 0 (default) means to - select automatically based on other filter options. - -The delta filter stores the differences between bytes, producing more repetitive -input for the compressor in certain circumstances. It supports one option, -``dist``. This indicates the distance between bytes to be subtracted. The -default is 1, i.e. take the differences between adjacent bytes. - -The BCJ filters are intended to be applied to machine code. They convert -relative branches, calls and jumps in the code to use absolute addressing, with -the aim of increasing the redundancy that can be exploited by the compressor. -These filters support one option, ``start_offset``. This specifies the address -that should be mapped to the beginning of the input data. The default is 0. - - -Examples --------- - -Reading in a compressed file:: - - import lzma - with lzma.open("file.xz") as f: - file_content = f.read() - -Creating a compressed file:: - - import lzma - data = b"Insert Data Here" - with lzma.open("file.xz", "w") as f: - f.write(data) - -Compressing data in memory:: - - import lzma - data_in = b"Insert Data Here" - data_out = lzma.compress(data_in) - -Incremental compression:: - - import lzma - lzc = lzma.LZMACompressor() - out1 = lzc.compress(b"Some data\n") - out2 = lzc.compress(b"Another piece of data\n") - out3 = lzc.compress(b"Even more data\n") - out4 = lzc.flush() - # Concatenate all the partial results: - result = b"".join([out1, out2, out3, out4]) - -Writing compressed data to an already-open file:: - - import lzma - with open("file.xz", "wb") as f: - f.write(b"This data will not be compressed\n") - with lzma.open(f, "w") as lzf: - lzf.write(b"This *will* be compressed\n") - f.write(b"Not compressed\n") - -Creating a compressed file using a custom filter chain:: - - import lzma - my_filters = [ - {"id": lzma.FILTER_DELTA, "dist": 5}, - {"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_EXTREME}, - ] - with lzma.open("file.xz", "w", filters=my_filters) as f: - f.write(b"blah blah blah") diff --git a/third_party/python/Doc/library/macpath.rst b/third_party/python/Doc/library/macpath.rst deleted file mode 100644 index b08bbe080..000000000 --- a/third_party/python/Doc/library/macpath.rst +++ /dev/null @@ -1,19 +0,0 @@ -:mod:`macpath` --- Mac OS 9 path manipulation functions -======================================================= - -.. module:: macpath - :synopsis: Mac OS 9 path manipulation functions. - -**Source code:** :source:`Lib/macpath.py` - --------------- - -This module is the Mac OS 9 (and earlier) implementation of the :mod:`os.path` -module. It can be used to manipulate old-style Macintosh pathnames on Mac OS X -(or any other platform). - -The following functions are available in this module: :func:`normcase`, -:func:`normpath`, :func:`isabs`, :func:`join`, :func:`split`, :func:`isdir`, -:func:`isfile`, :func:`walk`, :func:`exists`. For other functions available in -:mod:`os.path` dummy counterparts are available. - diff --git a/third_party/python/Doc/library/mailbox.rst b/third_party/python/Doc/library/mailbox.rst deleted file mode 100644 index 48d76a832..000000000 --- a/third_party/python/Doc/library/mailbox.rst +++ /dev/null @@ -1,1593 +0,0 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== - -.. module:: mailbox - :synopsis: Manipulate mailboxes in various formats - -.. moduleauthor:: Gregory K. Johnson -.. sectionauthor:: Gregory K. Johnson - -**Source code:** :source:`Lib/mailbox.py` - --------------- - -This module defines two classes, :class:`Mailbox` and :class:`Message`, for -accessing and manipulating on-disk mailboxes and the messages they contain. -:class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's -:class:`~email.message.Message` class with format-specific state and behavior. -Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. - - -.. seealso:: - - Module :mod:`email` - Represent and manipulate messages. - - -.. _mailbox-objects: - -:class:`Mailbox` objects ------------------------- - -.. class:: Mailbox - - A mailbox, which may be inspected and modified. - - The :class:`Mailbox` class defines an interface and is not intended to be - instantiated. Instead, format-specific subclasses should inherit from - :class:`Mailbox` and your code should instantiate a particular subclass. - - The :class:`Mailbox` interface is dictionary-like, with small keys - corresponding to messages. Keys are issued by the :class:`Mailbox` instance - with which they will be used and are only meaningful to that :class:`Mailbox` - instance. A key continues to identify a message even if the corresponding - message is modified, such as by replacing it with another message. - - Messages may be added to a :class:`Mailbox` instance using the set-like - method :meth:`add` and removed using a ``del`` statement or the set-like - methods :meth:`remove` and :meth:`discard`. - - :class:`Mailbox` interface semantics differ from dictionary semantics in some - noteworthy ways. Each time a message is requested, a new representation - (typically a :class:`Message` instance) is generated based upon the current - state of the mailbox. Similarly, when a message is added to a - :class:`Mailbox` instance, the provided message representation's contents are - copied. In neither case is a reference to the message representation kept by - the :class:`Mailbox` instance. - - The default :class:`Mailbox` iterator iterates over message representations, - not keys as the default dictionary iterator does. Moreover, modification of a - mailbox during iteration is safe and well-defined. Messages added to the - mailbox after an iterator is created will not be seen by the - iterator. Messages removed from the mailbox before the iterator yields them - will be silently skipped, though using a key from an iterator may result in a - :exc:`KeyError` exception if the corresponding message is subsequently - removed. - - .. warning:: - - Be very cautious when modifying mailboxes that might be simultaneously - changed by some other process. The safest mailbox format to use for such - tasks is Maildir; try to avoid using single-file formats such as mbox for - concurrent writing. If you're modifying a mailbox, you *must* lock it by - calling the :meth:`lock` and :meth:`unlock` methods *before* reading any - messages in the file or making any changes by adding or deleting a - message. Failing to lock the mailbox runs the risk of losing messages or - corrupting the entire mailbox. - - :class:`Mailbox` instances have the following methods: - - - .. method:: add(message) - - Add *message* to the mailbox and return the key that has been assigned to - it. - - Parameter *message* may be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, a byte string, or a - file-like object (which should be open in binary mode). If *message* is - an instance of the - appropriate format-specific :class:`Message` subclass (e.g., if it's an - :class:`mboxMessage` instance and this is an :class:`mbox` instance), its - format-specific information is used. Otherwise, reasonable defaults for - format-specific information are used. - - .. versionchanged:: 3.2 - Support for binary input was added. - - - .. method:: remove(key) - __delitem__(key) - discard(key) - - Delete the message corresponding to *key* from the mailbox. - - If no such message exists, a :exc:`KeyError` exception is raised if the - method was called as :meth:`remove` or :meth:`__delitem__` but no - exception is raised if the method was called as :meth:`discard`. The - behavior of :meth:`discard` may be preferred if the underlying mailbox - format supports concurrent modification by other processes. - - - .. method:: __setitem__(key, message) - - Replace the message corresponding to *key* with *message*. Raise a - :exc:`KeyError` exception if no message already corresponds to *key*. - - As with :meth:`add`, parameter *message* may be a :class:`Message` - instance, an :class:`email.message.Message` instance, a string, a byte - string, or a file-like object (which should be open in binary mode). If - *message* is an - instance of the appropriate format-specific :class:`Message` subclass - (e.g., if it's an :class:`mboxMessage` instance and this is an - :class:`mbox` instance), its format-specific information is - used. Otherwise, the format-specific information of the message that - currently corresponds to *key* is left unchanged. - - - .. method:: iterkeys() - keys() - - Return an iterator over all keys if called as :meth:`iterkeys` or return a - list of keys if called as :meth:`keys`. - - - .. method:: itervalues() - __iter__() - values() - - Return an iterator over representations of all messages if called as - :meth:`itervalues` or :meth:`__iter__` or return a list of such - representations if called as :meth:`values`. The messages are represented - as instances of the appropriate format-specific :class:`Message` subclass - unless a custom message factory was specified when the :class:`Mailbox` - instance was initialized. - - .. note:: - - The behavior of :meth:`__iter__` is unlike that of dictionaries, which - iterate over keys. - - - .. method:: iteritems() - items() - - Return an iterator over (*key*, *message*) pairs, where *key* is a key and - *message* is a message representation, if called as :meth:`iteritems` or - return a list of such pairs if called as :meth:`items`. The messages are - represented as instances of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: get(key, default=None) - __getitem__(key) - - Return a representation of the message corresponding to *key*. If no such - message exists, *default* is returned if the method was called as - :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`__getitem__`. The message is represented as an instance - of the appropriate format-specific :class:`Message` subclass unless a - custom message factory was specified when the :class:`Mailbox` instance - was initialized. - - - .. method:: get_message(key) - - Return a representation of the message corresponding to *key* as an - instance of the appropriate format-specific :class:`Message` subclass, or - raise a :exc:`KeyError` exception if no such message exists. - - - .. method:: get_bytes(key) - - Return a byte representation of the message corresponding to *key*, or - raise a :exc:`KeyError` exception if no such message exists. - - .. versionadded:: 3.2 - - - .. method:: get_string(key) - - Return a string representation of the message corresponding to *key*, or - raise a :exc:`KeyError` exception if no such message exists. The - message is processed through :class:`email.message.Message` to - convert it to a 7bit clean representation. - - - .. method:: get_file(key) - - Return a file-like representation of the message corresponding to *key*, - or raise a :exc:`KeyError` exception if no such message exists. The - file-like object behaves as if open in binary mode. This file should be - closed once it is no longer needed. - - .. versionchanged:: 3.2 - The file object really is a binary file; previously it was incorrectly - returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to - automatically close it. - - .. note:: - - Unlike other representations of messages, file-like representations are - not necessarily independent of the :class:`Mailbox` instance that - created them or of the underlying mailbox. More specific documentation - is provided by each subclass. - - - .. method:: __contains__(key) - - Return ``True`` if *key* corresponds to a message, ``False`` otherwise. - - - .. method:: __len__() - - Return a count of messages in the mailbox. - - - .. method:: clear() - - Delete all messages from the mailbox. - - - .. method:: pop(key, default=None) - - Return a representation of the message corresponding to *key* and delete - the message. If no such message exists, return *default*. The message is - represented as an instance of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: popitem() - - Return an arbitrary (*key*, *message*) pair, where *key* is a key and - *message* is a message representation, and delete the corresponding - message. If the mailbox is empty, raise a :exc:`KeyError` exception. The - message is represented as an instance of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: update(arg) - - Parameter *arg* should be a *key*-to-*message* mapping or an iterable of - (*key*, *message*) pairs. Updates the mailbox so that, for each given - *key* and *message*, the message corresponding to *key* is set to - *message* as if by using :meth:`__setitem__`. As with :meth:`__setitem__`, - each *key* must already correspond to a message in the mailbox or else a - :exc:`KeyError` exception will be raised, so in general it is incorrect - for *arg* to be a :class:`Mailbox` instance. - - .. note:: - - Unlike with dictionaries, keyword arguments are not supported. - - - .. method:: flush() - - Write any pending changes to the filesystem. For some :class:`Mailbox` - subclasses, changes are always written immediately and :meth:`flush` does - nothing, but you should still make a habit of calling this method. - - - .. method:: lock() - - Acquire an exclusive advisory lock on the mailbox so that other processes - know not to modify it. An :exc:`ExternalClashError` is raised if the lock - is not available. The particular locking mechanisms used depend upon the - mailbox format. You should *always* lock the mailbox before making any - modifications to its contents. - - - .. method:: unlock() - - Release the lock on the mailbox, if any. - - - .. method:: close() - - Flush the mailbox, unlock it if necessary, and close any open files. For - some :class:`Mailbox` subclasses, this method does nothing. - - -.. _mailbox-maildir: - -:class:`Maildir` -^^^^^^^^^^^^^^^^ - - -.. class:: Maildir(dirname, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in Maildir format. Parameter - *factory* is a callable object that accepts a file-like message representation - (which behaves as if opened in binary mode) and returns a custom representation. - If *factory* is ``None``, :class:`MaildirMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - It is for historical reasons that *dirname* is named as such rather than *path*. - - Maildir is a directory-based mailbox format invented for the qmail mail - transfer agent and now widely supported by other programs. Messages in a - Maildir mailbox are stored in separate files within a common directory - structure. This design allows Maildir mailboxes to be accessed and modified - by multiple unrelated programs without data corruption, so file locking is - unnecessary. - - Maildir mailboxes contain three subdirectories, namely: :file:`tmp`, - :file:`new`, and :file:`cur`. Messages are created momentarily in the - :file:`tmp` subdirectory and then moved to the :file:`new` subdirectory to - finalize delivery. A mail user agent may subsequently move the message to the - :file:`cur` subdirectory and store information about the state of the message - in a special "info" section appended to its file name. - - Folders of the style introduced by the Courier mail transfer agent are also - supported. Any subdirectory of the main mailbox is considered a folder if - ``'.'`` is the first character in its name. Folder names are represented by - :class:`Maildir` without the leading ``'.'``. Each folder is itself a Maildir - mailbox but should not contain other folders. Instead, a logical nesting is - indicated using ``'.'`` to delimit levels, e.g., "Archived.2005.07". - - .. note:: - - The Maildir specification requires the use of a colon (``':'``) in certain - message file names. However, some operating systems do not permit this - character in file names, If you wish to use a Maildir-like format on such - an operating system, you should specify another character to use - instead. The exclamation point (``'!'``) is a popular choice. For - example:: - - import mailbox - mailbox.Maildir.colon = '!' - - The :attr:`colon` attribute may also be set on a per-instance basis. - - :class:`Maildir` instances have all of the methods of :class:`Mailbox` in - addition to the following: - - - .. method:: list_folders() - - Return a list of the names of all folders. - - - .. method:: get_folder(folder) - - Return a :class:`Maildir` instance representing the folder whose name is - *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder - does not exist. - - - .. method:: add_folder(folder) - - Create a folder whose name is *folder* and return a :class:`Maildir` - instance representing it. - - - .. method:: remove_folder(folder) - - Delete the folder whose name is *folder*. If the folder contains any - messages, a :exc:`NotEmptyError` exception will be raised and the folder - will not be deleted. - - - .. method:: clean() - - Delete temporary files from the mailbox that have not been accessed in the - last 36 hours. The Maildir specification says that mail-reading programs - should do this occasionally. - - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special - remarks: - - - .. method:: add(message) - __setitem__(key, message) - update(arg) - - .. warning:: - - These methods generate unique file names based upon the current process - ID. When using multiple threads, undetected name clashes may occur and - cause corruption of the mailbox unless threads are coordinated to avoid - using these methods to manipulate the same mailbox simultaneously. - - - .. method:: flush() - - All changes to Maildir mailboxes are immediately applied, so this method - does nothing. - - - .. method:: lock() - unlock() - - Maildir mailboxes do not support (or require) locking, so these methods do - nothing. - - - .. method:: close() - - :class:`Maildir` instances do not keep any open files and the underlying - mailboxes do not support locking, so this method does nothing. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to modify or - remove the underlying message while the returned file remains open. - - -.. seealso:: - - `maildir man page from qmail `_ - The original specification of the format. - - `Using maildir format `_ - Notes on Maildir by its inventor. Includes an updated name-creation scheme and - details on "info" semantics. - - `maildir man page from Courier `_ - Another specification of the format. Describes a common extension for supporting - folders. - - -.. _mailbox-mbox: - -:class:`mbox` -^^^^^^^^^^^^^ - - -.. class:: mbox(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in mbox format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`mboxMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - The mbox format is the classic format for storing mail on Unix systems. All - messages in an mbox mailbox are stored in a single file with the beginning of - each message indicated by a line whose first five characters are "From ". - - Several variations of the mbox format exist to address perceived shortcomings in - the original. In the interest of compatibility, :class:`mbox` implements the - original format, which is sometimes referred to as :dfn:`mboxo`. This means that - the :mailheader:`Content-Length` header, if present, is ignored and that any - occurrences of "From " at the beginning of a line in a message body are - transformed to ">From " when storing the message, although occurrences of ">From - " are not transformed to "From " when reading the message. - - Some :class:`Mailbox` methods implemented by :class:`mbox` deserve special - remarks: - - - .. method:: get_file(key) - - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`mbox` instance may yield unpredictable results or raise an - exception. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. - - -.. seealso:: - - `mbox man page from qmail `_ - A specification of the format and its variations. - - `mbox man page from tin `_ - Another specification of the format, with details on locking. - - `Configuring Netscape Mail on Unix: Why The Content-Length Format is Bad `_ - An argument for using the original mbox format rather than a variation. - - `"mbox" is a family of several mutually incompatible mailbox formats `_ - A history of mbox variations. - - -.. _mailbox-mh: - -:class:`MH` -^^^^^^^^^^^ - - -.. class:: MH(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in MH format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`MHMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - MH is a directory-based mailbox format invented for the MH Message Handling - System, a mail user agent. Each message in an MH mailbox resides in its own - file. An MH mailbox may contain other MH mailboxes (called :dfn:`folders`) in - addition to messages. Folders may be nested indefinitely. MH mailboxes also - support :dfn:`sequences`, which are named lists used to logically group - messages without moving them to sub-folders. Sequences are defined in a file - called :file:`.mh_sequences` in each folder. - - The :class:`MH` class manipulates MH mailboxes, but it does not attempt to - emulate all of :program:`mh`'s behaviors. In particular, it does not modify - and is not affected by the :file:`context` or :file:`.mh_profile` files that - are used by :program:`mh` to store its state and configuration. - - :class:`MH` instances have all of the methods of :class:`Mailbox` in addition - to the following: - - - .. method:: list_folders() - - Return a list of the names of all folders. - - - .. method:: get_folder(folder) - - Return an :class:`MH` instance representing the folder whose name is - *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder - does not exist. - - - .. method:: add_folder(folder) - - Create a folder whose name is *folder* and return an :class:`MH` instance - representing it. - - - .. method:: remove_folder(folder) - - Delete the folder whose name is *folder*. If the folder contains any - messages, a :exc:`NotEmptyError` exception will be raised and the folder - will not be deleted. - - - .. method:: get_sequences() - - Return a dictionary of sequence names mapped to key lists. If there are no - sequences, the empty dictionary is returned. - - - .. method:: set_sequences(sequences) - - Re-define the sequences that exist in the mailbox based upon *sequences*, - a dictionary of names mapped to key lists, like returned by - :meth:`get_sequences`. - - - .. method:: pack() - - Rename messages in the mailbox as necessary to eliminate gaps in - numbering. Entries in the sequences list are updated correspondingly. - - .. note:: - - Already-issued keys are invalidated by this operation and should not be - subsequently used. - - Some :class:`Mailbox` methods implemented by :class:`MH` deserve special - remarks: - - - .. method:: remove(key) - __delitem__(key) - discard(key) - - These methods immediately delete the message. The MH convention of marking - a message for deletion by prepending a comma to its name is not used. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. For MH mailboxes, locking - the mailbox means locking the :file:`.mh_sequences` file and, only for the - duration of any operations that affect them, locking individual message - files. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to remove the - underlying message while the returned file remains open. - - - .. method:: flush() - - All changes to MH mailboxes are immediately applied, so this method does - nothing. - - - .. method:: close() - - :class:`MH` instances do not keep any open files, so this method is - equivalent to :meth:`unlock`. - - -.. seealso:: - - `nmh - Message Handling System `_ - Home page of :program:`nmh`, an updated version of the original :program:`mh`. - - `MH & nmh: Email for Users & Programmers `_ - A GPL-licensed book on :program:`mh` and :program:`nmh`, with some information - on the mailbox format. - - -.. _mailbox-babyl: - -:class:`Babyl` -^^^^^^^^^^^^^^ - - -.. class:: Babyl(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in Babyl format. Parameter - *factory* is a callable object that accepts a file-like message representation - (which behaves as if opened in binary mode) and returns a custom representation. - If *factory* is ``None``, :class:`BabylMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - Babyl is a single-file mailbox format used by the Rmail mail user agent - included with Emacs. The beginning of a message is indicated by a line - containing the two characters Control-Underscore (``'\037'``) and Control-L - (``'\014'``). The end of a message is indicated by the start of the next - message or, in the case of the last message, a line containing a - Control-Underscore (``'\037'``) character. - - Messages in a Babyl mailbox have two sets of headers, original headers and - so-called visible headers. Visible headers are typically a subset of the - original headers that have been reformatted or abridged to be more - attractive. Each message in a Babyl mailbox also has an accompanying list of - :dfn:`labels`, or short strings that record extra information about the - message, and a list of all user-defined labels found in the mailbox is kept - in the Babyl options section. - - :class:`Babyl` instances have all of the methods of :class:`Mailbox` in - addition to the following: - - - .. method:: get_labels() - - Return a list of the names of all user-defined labels used in the mailbox. - - .. note:: - - The actual messages are inspected to determine which labels exist in - the mailbox rather than consulting the list of labels in the Babyl - options section, but the Babyl section is updated whenever the mailbox - is modified. - - Some :class:`Mailbox` methods implemented by :class:`Babyl` deserve special - remarks: - - - .. method:: get_file(key) - - In Babyl mailboxes, the headers of a message are not stored contiguously - with the body of the message. To generate a file-like representation, the - headers and body are copied together into an :class:`io.BytesIO` instance, - which has an API identical to that of a - file. As a result, the file-like object is truly independent of the - underlying mailbox but does not save memory compared to a string - representation. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. - - -.. seealso:: - - `Format of Version 5 Babyl Files `_ - A specification of the Babyl format. - - `Reading Mail with Rmail `_ - The Rmail manual, with some information on Babyl semantics. - - -.. _mailbox-mmdf: - -:class:`MMDF` -^^^^^^^^^^^^^ - - -.. class:: MMDF(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in MMDF format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`MMDFMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - MMDF is a single-file mailbox format invented for the Multichannel Memorandum - Distribution Facility, a mail transfer agent. Each message is in the same - form as an mbox message but is bracketed before and after by lines containing - four Control-A (``'\001'``) characters. As with the mbox format, the - beginning of each message is indicated by a line whose first five characters - are "From ", but additional occurrences of "From " are not transformed to - ">From " when storing messages because the extra message separator lines - prevent mistaking such occurrences for the starts of subsequent messages. - - Some :class:`Mailbox` methods implemented by :class:`MMDF` deserve special - remarks: - - - .. method:: get_file(key) - - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`MMDF` instance may yield unpredictable results or raise an - exception. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. - - -.. seealso:: - - `mmdf man page from tin `_ - A specification of MMDF format from the documentation of tin, a newsreader. - - `MMDF `_ - A Wikipedia article describing the Multichannel Memorandum Distribution - Facility. - - -.. _mailbox-message-objects: - -:class:`Message` objects ------------------------- - - -.. class:: Message(message=None) - - A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add - mailbox-format-specific state and behavior. - - If *message* is omitted, the new instance is created in a default, empty state. - If *message* is an :class:`email.message.Message` instance, its contents are - copied; furthermore, any format-specific information is converted insofar as - possible if *message* is a :class:`Message` instance. If *message* is a string, - a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read - and parsed. Files should be open in binary mode, but text mode files - are accepted for backward compatibility. - - The format-specific state and behaviors offered by subclasses vary, but in - general it is only the properties that are not specific to a particular - mailbox that are supported (although presumably the properties are specific - to a particular mailbox format). For example, file offsets for single-file - mailbox formats and file names for directory-based mailbox formats are not - retained, because they are only applicable to the original mailbox. But state - such as whether a message has been read by the user or marked as important is - retained, because it applies to the message itself. - - There is no requirement that :class:`Message` instances be used to represent - messages retrieved using :class:`Mailbox` instances. In some situations, the - time and memory required to generate :class:`Message` representations might - not be acceptable. For such situations, :class:`Mailbox` instances also - offer string and file-like representations, and a custom message factory may - be specified when a :class:`Mailbox` instance is initialized. - - -.. _mailbox-maildirmessage: - -:class:`MaildirMessage` -^^^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: MaildirMessage(message=None) - - A message with Maildir-specific behaviors. Parameter *message* has the same - meaning as with the :class:`Message` constructor. - - Typically, a mail user agent application moves all of the messages in the - :file:`new` subdirectory to the :file:`cur` subdirectory after the first time - the user opens and closes the mailbox, recording that the messages are old - whether or not they've actually been read. Each message in :file:`cur` has an - "info" section added to its file name to store information about its state. - (Some mail readers may also add an "info" section to messages in - :file:`new`.) The "info" section may take one of two forms: it may contain - "2," followed by a list of standardized flags (e.g., "2,FR") or it may - contain "1," followed by so-called experimental information. Standard flags - for Maildir messages are as follows: - - +------+---------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+=========+================================+ - | D | Draft | Under composition | - +------+---------+--------------------------------+ - | F | Flagged | Marked as important | - +------+---------+--------------------------------+ - | P | Passed | Forwarded, resent, or bounced | - +------+---------+--------------------------------+ - | R | Replied | Replied to | - +------+---------+--------------------------------+ - | S | Seen | Read | - +------+---------+--------------------------------+ - | T | Trashed | Marked for subsequent deletion | - +------+---------+--------------------------------+ - - :class:`MaildirMessage` instances offer the following methods: - - - .. method:: get_subdir() - - Return either "new" (if the message should be stored in the :file:`new` - subdirectory) or "cur" (if the message should be stored in the :file:`cur` - subdirectory). - - .. note:: - - A message is typically moved from :file:`new` to :file:`cur` after its - mailbox has been accessed, whether or not the message is has been - read. A message ``msg`` has been read if ``"S" in msg.get_flags()`` is - ``True``. - - - .. method:: set_subdir(subdir) - - Set the subdirectory the message should be stored in. Parameter *subdir* - must be either "new" or "cur". - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the standard Maildir format, the result is the - concatenation in alphabetical order of zero or one occurrence of each of - ``'D'``, ``'F'``, ``'P'``, ``'R'``, ``'S'``, and ``'T'``. The empty string - is returned if no flags are set or if "info" contains experimental - semantics. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. The current "info" is overwritten whether or not it contains - experimental information rather than flags. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. If "info" contains experimental information rather than - flags, the current "info" is not modified. - - - .. method:: get_date() - - Return the delivery date of the message as a floating-point number - representing seconds since the epoch. - - - .. method:: set_date(date) - - Set the delivery date of the message to *date*, a floating-point number - representing seconds since the epoch. - - - .. method:: get_info() - - Return a string containing the "info" for a message. This is useful for - accessing and modifying "info" that is experimental (i.e., not a list of - flags). - - - .. method:: set_info(info) - - Set "info" to *info*, which should be a string. - -When a :class:`MaildirMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+--------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+====================+==============================================+ -| "cur" subdirectory | O flag | -+--------------------+----------------------------------------------+ -| F flag | F flag | -+--------------------+----------------------------------------------+ -| R flag | A flag | -+--------------------+----------------------------------------------+ -| S flag | R flag | -+--------------------+----------------------------------------------+ -| T flag | D flag | -+--------------------+----------------------------------------------+ - -When a :class:`MaildirMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===============================+==========================+ -| "cur" subdirectory | "unseen" sequence | -+-------------------------------+--------------------------+ -| "cur" subdirectory and S flag | no "unseen" sequence | -+-------------------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------------------+--------------------------+ -| R flag | "replied" sequence | -+-------------------------------+--------------------------+ - -When a :class:`MaildirMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------------------+-------------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===============================+===============================+ -| "cur" subdirectory | "unseen" label | -+-------------------------------+-------------------------------+ -| "cur" subdirectory and S flag | no "unseen" label | -+-------------------------------+-------------------------------+ -| P flag | "forwarded" or "resent" label | -+-------------------------------+-------------------------------+ -| R flag | "answered" label | -+-------------------------------+-------------------------------+ -| T flag | "deleted" label | -+-------------------------------+-------------------------------+ - - -.. _mailbox-mboxmessage: - -:class:`mboxMessage` -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: mboxMessage(message=None) - - A message with mbox-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - Messages in an mbox mailbox are stored together in a single file. The - sender's envelope address and the time of delivery are typically stored in a - line beginning with "From " that is used to indicate the start of a message, - though there is considerable variation in the exact format of this data among - mbox implementations. Flags that indicate the state of the message, such as - whether it has been read or marked as important, are typically stored in - :mailheader:`Status` and :mailheader:`X-Status` headers. - - Conventional flags for mbox messages are as follows: - - +------+----------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+==========+================================+ - | R | Read | Read | - +------+----------+--------------------------------+ - | O | Old | Previously detected by MUA | - +------+----------+--------------------------------+ - | D | Deleted | Marked for subsequent deletion | - +------+----------+--------------------------------+ - | F | Flagged | Marked as important | - +------+----------+--------------------------------+ - | A | Answered | Replied to | - +------+----------+--------------------------------+ - - The "R" and "O" flags are stored in the :mailheader:`Status` header, and the - "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The - flags and headers typically appear in the order mentioned. - - :class:`mboxMessage` instances offer the following methods: - - - .. method:: get_from() - - Return a string representing the "From " line that marks the start of the - message in an mbox mailbox. The leading "From " and the trailing newline - are excluded. - - - .. method:: set_from(from_, time_=None) - - Set the "From " line to *from_*, which should be specified without a - leading "From " or trailing newline. For convenience, *time_* may be - specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the conventional format, the result is the - concatenation in the following order of zero or one occurrence of each of - ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. Parameter *flags* - should be the concatenation in any order of zero or more occurrences of - each of ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. - -When an :class:`mboxMessage` instance is created based upon a -:class:`MaildirMessage` instance, a "From " line is generated based upon the -:class:`MaildirMessage` instance's delivery date, and the following conversions -take place: - -+-----------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+=================+===============================+ -| R flag | S flag | -+-----------------+-------------------------------+ -| O flag | "cur" subdirectory | -+-----------------+-------------------------------+ -| D flag | T flag | -+-----------------+-------------------------------+ -| F flag | F flag | -+-----------------+-------------------------------+ -| A flag | R flag | -+-----------------+-------------------------------+ - -When an :class:`mboxMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===================+==========================+ -| R flag and O flag | no "unseen" sequence | -+-------------------+--------------------------+ -| O flag | "unseen" sequence | -+-------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------+--------------------------+ -| A flag | "replied" sequence | -+-------------------+--------------------------+ - -When an :class:`mboxMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===================+=============================+ -| R flag and O flag | no "unseen" label | -+-------------------+-----------------------------+ -| O flag | "unseen" label | -+-------------------+-----------------------------+ -| D flag | "deleted" label | -+-------------------+-----------------------------+ -| A flag | "answered" label | -+-------------------+-----------------------------+ - -When a :class:`Message` instance is created based upon an :class:`MMDFMessage` -instance, the "From " line is copied and all flags directly correspond: - -+-----------------+----------------------------+ -| Resulting state | :class:`MMDFMessage` state | -+=================+============================+ -| R flag | R flag | -+-----------------+----------------------------+ -| O flag | O flag | -+-----------------+----------------------------+ -| D flag | D flag | -+-----------------+----------------------------+ -| F flag | F flag | -+-----------------+----------------------------+ -| A flag | A flag | -+-----------------+----------------------------+ - - -.. _mailbox-mhmessage: - -:class:`MHMessage` -^^^^^^^^^^^^^^^^^^ - - -.. class:: MHMessage(message=None) - - A message with MH-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - MH messages do not support marks or flags in the traditional sense, but they - do support sequences, which are logical groupings of arbitrary messages. Some - mail reading programs (although not the standard :program:`mh` and - :program:`nmh`) use sequences in much the same way flags are used with other - formats, as follows: - - +----------+------------------------------------------+ - | Sequence | Explanation | - +==========+==========================================+ - | unseen | Not read, but previously detected by MUA | - +----------+------------------------------------------+ - | replied | Replied to | - +----------+------------------------------------------+ - | flagged | Marked as important | - +----------+------------------------------------------+ - - :class:`MHMessage` instances offer the following methods: - - - .. method:: get_sequences() - - Return a list of the names of sequences that include this message. - - - .. method:: set_sequences(sequences) - - Set the list of sequences that include this message. - - - .. method:: add_sequence(sequence) - - Add *sequence* to the list of sequences that include this message. - - - .. method:: remove_sequence(sequence) - - Remove *sequence* from the list of sequences that include this message. - -When an :class:`MHMessage` instance is created based upon a -:class:`MaildirMessage` instance, the following conversions take place: - -+--------------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+====================+===============================+ -| "unseen" sequence | no S flag | -+--------------------+-------------------------------+ -| "replied" sequence | R flag | -+--------------------+-------------------------------+ -| "flagged" sequence | F flag | -+--------------------+-------------------------------+ - -When an :class:`MHMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+--------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+====================+==============================================+ -| "unseen" sequence | no R flag | -+--------------------+----------------------------------------------+ -| "replied" sequence | A flag | -+--------------------+----------------------------------------------+ -| "flagged" sequence | F flag | -+--------------------+----------------------------------------------+ - -When an :class:`MHMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+--------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+====================+=============================+ -| "unseen" sequence | "unseen" label | -+--------------------+-----------------------------+ -| "replied" sequence | "answered" label | -+--------------------+-----------------------------+ - - -.. _mailbox-babylmessage: - -:class:`BabylMessage` -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: BabylMessage(message=None) - - A message with Babyl-specific behaviors. Parameter *message* has the same - meaning as with the :class:`Message` constructor. - - Certain message labels, called :dfn:`attributes`, are defined by convention - to have special meanings. The attributes are as follows: - - +-----------+------------------------------------------+ - | Label | Explanation | - +===========+==========================================+ - | unseen | Not read, but previously detected by MUA | - +-----------+------------------------------------------+ - | deleted | Marked for subsequent deletion | - +-----------+------------------------------------------+ - | filed | Copied to another file or mailbox | - +-----------+------------------------------------------+ - | answered | Replied to | - +-----------+------------------------------------------+ - | forwarded | Forwarded | - +-----------+------------------------------------------+ - | edited | Modified by the user | - +-----------+------------------------------------------+ - | resent | Resent | - +-----------+------------------------------------------+ - - By default, Rmail displays only visible headers. The :class:`BabylMessage` - class, though, uses the original headers because they are more - complete. Visible headers may be accessed explicitly if desired. - - :class:`BabylMessage` instances offer the following methods: - - - .. method:: get_labels() - - Return a list of labels on the message. - - - .. method:: set_labels(labels) - - Set the list of labels on the message to *labels*. - - - .. method:: add_label(label) - - Add *label* to the list of labels on the message. - - - .. method:: remove_label(label) - - Remove *label* from the list of labels on the message. - - - .. method:: get_visible() - - Return an :class:`Message` instance whose headers are the message's - visible headers and whose body is empty. - - - .. method:: set_visible(visible) - - Set the message's visible headers to be the same as the headers in - *message*. Parameter *visible* should be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, or a file-like object - (which should be open in text mode). - - - .. method:: update_visible() - - When a :class:`BabylMessage` instance's original headers are modified, the - visible headers are not automatically modified to correspond. This method - updates the visible headers as follows: each visible header with a - corresponding original header is set to the value of the original header, - each visible header without a corresponding original header is removed, - and any of :mailheader:`Date`, :mailheader:`From`, :mailheader:`Reply-To`, - :mailheader:`To`, :mailheader:`CC`, and :mailheader:`Subject` that are - present in the original headers but not the visible headers are added to - the visible headers. - -When a :class:`BabylMessage` instance is created based upon a -:class:`MaildirMessage` instance, the following conversions take place: - -+-------------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+===================+===============================+ -| "unseen" label | no S flag | -+-------------------+-------------------------------+ -| "deleted" label | T flag | -+-------------------+-------------------------------+ -| "answered" label | R flag | -+-------------------+-------------------------------+ -| "forwarded" label | P flag | -+-------------------+-------------------------------+ - -When a :class:`BabylMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+==================+==============================================+ -| "unseen" label | no R flag | -+------------------+----------------------------------------------+ -| "deleted" label | D flag | -+------------------+----------------------------------------------+ -| "answered" label | A flag | -+------------------+----------------------------------------------+ - -When a :class:`BabylMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+==================+==========================+ -| "unseen" label | "unseen" sequence | -+------------------+--------------------------+ -| "answered" label | "replied" sequence | -+------------------+--------------------------+ - - -.. _mailbox-mmdfmessage: - -:class:`MMDFMessage` -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: MMDFMessage(message=None) - - A message with MMDF-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - As with message in an mbox mailbox, MMDF messages are stored with the - sender's address and the delivery date in an initial line beginning with - "From ". Likewise, flags that indicate the state of the message are - typically stored in :mailheader:`Status` and :mailheader:`X-Status` headers. - - Conventional flags for MMDF messages are identical to those of mbox message - and are as follows: - - +------+----------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+==========+================================+ - | R | Read | Read | - +------+----------+--------------------------------+ - | O | Old | Previously detected by MUA | - +------+----------+--------------------------------+ - | D | Deleted | Marked for subsequent deletion | - +------+----------+--------------------------------+ - | F | Flagged | Marked as important | - +------+----------+--------------------------------+ - | A | Answered | Replied to | - +------+----------+--------------------------------+ - - The "R" and "O" flags are stored in the :mailheader:`Status` header, and the - "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The - flags and headers typically appear in the order mentioned. - - :class:`MMDFMessage` instances offer the following methods, which are - identical to those offered by :class:`mboxMessage`: - - - .. method:: get_from() - - Return a string representing the "From " line that marks the start of the - message in an mbox mailbox. The leading "From " and the trailing newline - are excluded. - - - .. method:: set_from(from_, time_=None) - - Set the "From " line to *from_*, which should be specified without a - leading "From " or trailing newline. For convenience, *time_* may be - specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the conventional format, the result is the - concatenation in the following order of zero or one occurrence of each of - ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. Parameter *flags* - should be the concatenation in any order of zero or more occurrences of - each of ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. - -When an :class:`MMDFMessage` instance is created based upon a -:class:`MaildirMessage` instance, a "From " line is generated based upon the -:class:`MaildirMessage` instance's delivery date, and the following conversions -take place: - -+-----------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+=================+===============================+ -| R flag | S flag | -+-----------------+-------------------------------+ -| O flag | "cur" subdirectory | -+-----------------+-------------------------------+ -| D flag | T flag | -+-----------------+-------------------------------+ -| F flag | F flag | -+-----------------+-------------------------------+ -| A flag | R flag | -+-----------------+-------------------------------+ - -When an :class:`MMDFMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===================+==========================+ -| R flag and O flag | no "unseen" sequence | -+-------------------+--------------------------+ -| O flag | "unseen" sequence | -+-------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------+--------------------------+ -| A flag | "replied" sequence | -+-------------------+--------------------------+ - -When an :class:`MMDFMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===================+=============================+ -| R flag and O flag | no "unseen" label | -+-------------------+-----------------------------+ -| O flag | "unseen" label | -+-------------------+-----------------------------+ -| D flag | "deleted" label | -+-------------------+-----------------------------+ -| A flag | "answered" label | -+-------------------+-----------------------------+ - -When an :class:`MMDFMessage` instance is created based upon an -:class:`mboxMessage` instance, the "From " line is copied and all flags directly -correspond: - -+-----------------+----------------------------+ -| Resulting state | :class:`mboxMessage` state | -+=================+============================+ -| R flag | R flag | -+-----------------+----------------------------+ -| O flag | O flag | -+-----------------+----------------------------+ -| D flag | D flag | -+-----------------+----------------------------+ -| F flag | F flag | -+-----------------+----------------------------+ -| A flag | A flag | -+-----------------+----------------------------+ - - -Exceptions ----------- - -The following exception classes are defined in the :mod:`mailbox` module: - - -.. exception:: Error() - - The based class for all other module-specific exceptions. - - -.. exception:: NoSuchMailboxError() - - Raised when a mailbox is expected but is not found, such as when instantiating a - :class:`Mailbox` subclass with a path that does not exist (and with the *create* - parameter set to ``False``), or when opening a folder that does not exist. - - -.. exception:: NotEmptyError() - - Raised when a mailbox is not empty but is expected to be, such as when deleting - a folder that contains messages. - - -.. exception:: ExternalClashError() - - Raised when some mailbox-related condition beyond the control of the program - causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely-generated file name - already exists. - - -.. exception:: FormatError() - - Raised when the data in a file cannot be parsed, such as when an :class:`MH` - instance attempts to read a corrupted :file:`.mh_sequences` file. - - -.. _mailbox-examples: - -Examples --------- - -A simple example of printing the subjects of all messages in a mailbox that seem -interesting:: - - import mailbox - for message in mailbox.mbox('~/mbox'): - subject = message['subject'] # Could possibly be None. - if subject and 'python' in subject.lower(): - print(subject) - -To copy all mail from a Babyl mailbox to an MH mailbox, converting all of the -format-specific information that can be converted:: - - import mailbox - destination = mailbox.MH('~/Mail') - destination.lock() - for message in mailbox.Babyl('~/RMAIL'): - destination.add(mailbox.MHMessage(message)) - destination.flush() - destination.unlock() - -This example sorts mail from several mailing lists into different mailboxes, -being careful to avoid mail corruption due to concurrent modification by other -programs, mail loss due to interruption of the program, or premature termination -due to malformed messages in the mailbox:: - - import mailbox - import email.errors - - list_names = ('python-list', 'python-dev', 'python-bugs') - - boxes = {name: mailbox.mbox('~/email/%s' % name) for name in list_names} - inbox = mailbox.Maildir('~/Maildir', factory=None) - - for key in inbox.iterkeys(): - try: - message = inbox[key] - except email.errors.MessageParseError: - continue # The message is malformed. Just leave it. - - for name in list_names: - list_id = message['list-id'] - if list_id and name in list_id: - # Get mailbox to use - box = boxes[name] - - # Write copy to disk before removing original. - # If there's a crash, you might duplicate a message, but - # that's better than losing a message completely. - box.lock() - box.add(message) - box.flush() - box.unlock() - - # Remove original message - inbox.lock() - inbox.discard(key) - inbox.flush() - inbox.unlock() - break # Found destination, so stop looking. - - for box in boxes.itervalues(): - box.close() - diff --git a/third_party/python/Doc/library/mailcap.rst b/third_party/python/Doc/library/mailcap.rst deleted file mode 100644 index 896afd1d7..000000000 --- a/third_party/python/Doc/library/mailcap.rst +++ /dev/null @@ -1,76 +0,0 @@ -:mod:`mailcap` --- Mailcap file handling -======================================== - -.. module:: mailcap - :synopsis: Mailcap file handling. - -**Source code:** :source:`Lib/mailcap.py` - --------------- - -Mailcap files are used to configure how MIME-aware applications such as mail -readers and Web browsers react to files with different MIME types. (The name -"mailcap" is derived from the phrase "mail capability".) For example, a mailcap -file might contain a line like ``video/mpeg; xmpeg %s``. Then, if the user -encounters an email message or Web document with the MIME type -:mimetype:`video/mpeg`, ``%s`` will be replaced by a filename (usually one -belonging to a temporary file) and the :program:`xmpeg` program can be -automatically started to view the file. - -The mailcap format is documented in :rfc:`1524`, "A User Agent Configuration -Mechanism For Multimedia Mail Format Information," but is not an Internet -standard. However, mailcap files are supported on most Unix systems. - - -.. function:: findmatch(caps, MIMEtype, key='view', filename='/dev/null', plist=[]) - - Return a 2-tuple; the first element is a string containing the command line to - be executed (which can be passed to :func:`os.system`), and the second element - is the mailcap entry for a given MIME type. If no matching MIME type can be - found, ``(None, None)`` is returned. - - *key* is the name of the field desired, which represents the type of activity to - be performed; the default value is 'view', since in the most common case you - simply want to view the body of the MIME-typed data. Other possible values - might be 'compose' and 'edit', if you wanted to create a new body of the given - MIME type or alter the existing body data. See :rfc:`1524` for a complete list - of these fields. - - *filename* is the filename to be substituted for ``%s`` in the command line; the - default value is ``'/dev/null'`` which is almost certainly not what you want, so - usually you'll override it by specifying a filename. - - *plist* can be a list containing named parameters; the default value is simply - an empty list. Each entry in the list must be a string containing the parameter - name, an equals sign (``'='``), and the parameter's value. Mailcap entries can - contain named parameters like ``%{foo}``, which will be replaced by the value - of the parameter named 'foo'. For example, if the command line ``showpartial - %{id} %{number} %{total}`` was in a mailcap file, and *plist* was set to - ``['id=1', 'number=2', 'total=3']``, the resulting command line would be - ``'showpartial 1 2 3'``. - - In a mailcap file, the "test" field can optionally be specified to test some - external condition (such as the machine architecture, or the window system in - use) to determine whether or not the mailcap line applies. :func:`findmatch` - will automatically check such conditions and skip the entry if the check fails. - - -.. function:: getcaps() - - Returns a dictionary mapping MIME types to a list of mailcap file entries. This - dictionary must be passed to the :func:`findmatch` function. An entry is stored - as a list of dictionaries, but it shouldn't be necessary to know the details of - this representation. - - The information is derived from all of the mailcap files found on the system. - Settings in the user's mailcap file :file:`$HOME/.mailcap` will override - settings in the system mailcap files :file:`/etc/mailcap`, - :file:`/usr/etc/mailcap`, and :file:`/usr/local/etc/mailcap`. - -An example usage:: - - >>> import mailcap - >>> d = mailcap.getcaps() - >>> mailcap.findmatch(d, 'video/mpeg', filename='tmp1223') - ('xmpeg tmp1223', {'view': 'xmpeg %s'}) - diff --git a/third_party/python/Doc/library/markup.rst b/third_party/python/Doc/library/markup.rst deleted file mode 100644 index 1588aa8a6..000000000 --- a/third_party/python/Doc/library/markup.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _markup: - -********************************** -Structured Markup Processing Tools -********************************** - -Python supports a variety of modules to work with various forms of structured -data markup. This includes modules to work with the Standard Generalized Markup -Language (SGML) and the Hypertext Markup Language (HTML), and several interfaces -for working with the Extensible Markup Language (XML). - - -.. toctree:: - - html.rst - html.parser.rst - html.entities.rst - xml.rst - xml.etree.elementtree.rst - xml.dom.rst - xml.dom.minidom.rst - xml.dom.pulldom.rst - xml.sax.rst - xml.sax.handler.rst - xml.sax.utils.rst - xml.sax.reader.rst - pyexpat.rst diff --git a/third_party/python/Doc/library/marshal.rst b/third_party/python/Doc/library/marshal.rst deleted file mode 100644 index d65afc200..000000000 --- a/third_party/python/Doc/library/marshal.rst +++ /dev/null @@ -1,118 +0,0 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= - -.. module:: marshal - :synopsis: Convert Python objects to streams of bytes and back (with different - constraints). - --------------- - -This module contains functions that can read and write Python values in a binary -format. The format is specific to Python, but independent of machine -architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are -undocumented on purpose; it may change between Python versions (although it -rarely does). [#]_ - -.. index:: - module: pickle - module: shelve - -This is not a general "persistence" module. For general persistence and -transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and -writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. -Therefore, the Python maintainers reserve the right to modify the marshal format -in backward incompatible ways should the need arise. If you're serializing and -de-serializing Python objects, use the :mod:`pickle` module instead -- the -performance is comparable, version independence is guaranteed, and pickle -supports a substantially wider range of objects than marshal. - -.. warning:: - - The :mod:`marshal` module is not intended to be secure against erroneous or - maliciously constructed data. Never unmarshal data received from an - untrusted or unauthenticated source. - -.. index:: object; code, code object - -Not all Python object types are supported; in general, only objects whose value -is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating -point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects, where it should be understood that -tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported. The -singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be -marshalled and unmarshalled. -For format *version* lower than 3, recursive lists, sets and dictionaries cannot -be written (see below). - -There are functions that read/write files as well as functions operating on -bytes-like objects. - -The module defines these functions: - - -.. function:: dump(value, file[, version]) - - Write the value on the open file. The value must be a supported type. The - file must be a writeable :term:`binary file`. - - If the value has (or contains an object that has) an unsupported type, a - :exc:`ValueError` exception is raised --- but garbage data will also be written - to the file. The object will not be properly read back by :func:`load`. - - The *version* argument indicates the data format that ``dump`` should use - (see below). - - -.. function:: load(file) - - Read one value from the open file and return it. If no valid value is read - (e.g. because the data has a different Python version's incompatible marshal - format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be a readable :term:`binary file`. - - .. note:: - - If an object containing an unsupported type was marshalled with :func:`dump`, - :func:`load` will substitute ``None`` for the unmarshallable type. - - -.. function:: dumps(value[, version]) - - Return the bytes object that would be written to a file by ``dump(value, file)``. The - value must be a supported type. Raise a :exc:`ValueError` exception if value - has (or contains an object that has) an unsupported type. - - The *version* argument indicates the data format that ``dumps`` should use - (see below). - - -.. function:: loads(bytes) - - Convert the :term:`bytes-like object` to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the - input are ignored. - - -In addition, the following constants are defined: - -.. data:: version - - Indicates the format that the module uses. Version 0 is the historical - format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. - Version 3 adds support for object instancing and recursion. - The current version is 4. - - -.. rubric:: Footnotes - -.. [#] The name of this module stems from a bit of terminology used by the designers of - Modula-3 (amongst others), who use the term "marshalling" for shipping of data - around in a self-contained form. Strictly speaking, "to marshal" means to - convert some data from internal to external form (in an RPC buffer for instance) - and "unmarshalling" for the reverse process. - diff --git a/third_party/python/Doc/library/math.rst b/third_party/python/Doc/library/math.rst deleted file mode 100644 index 0f36a3fa9..000000000 --- a/third_party/python/Doc/library/math.rst +++ /dev/null @@ -1,477 +0,0 @@ -:mod:`math` --- Mathematical functions -====================================== - -.. module:: math - :synopsis: Mathematical functions (sin() etc.). - -.. testsetup:: - - from math import fsum - --------------- - -This module is always available. It provides access to the mathematical -functions defined by the C standard. - -These functions cannot be used with complex numbers; use the functions of the -same name from the :mod:`cmath` module if you require support for complex -numbers. The distinction between functions which support complex numbers and -those which don't is made since most users do not want to learn quite as much -mathematics as required to understand complex numbers. Receiving an exception -instead of a complex result allows earlier detection of the unexpected complex -number used as a parameter, so that the programmer can determine how and why it -was generated in the first place. - -The following functions are provided by this module. Except when explicitly -noted otherwise, all return values are floats. - - -Number-theoretic and representation functions ---------------------------------------------- - -.. function:: ceil(x) - - Return the ceiling of *x*, the smallest integer greater than or equal to *x*. - If *x* is not a float, delegates to ``x.__ceil__()``, which should return an - :class:`~numbers.Integral` value. - - -.. function:: copysign(x, y) - - Return a float with the magnitude (absolute value) of *x* but the sign of - *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` - returns *-1.0*. - -.. function:: fabs(x) - - Return the absolute value of *x*. - -.. function:: factorial(x) - - Return *x* factorial. Raises :exc:`ValueError` if *x* is not integral or - is negative. - -.. function:: floor(x) - - Return the floor of *x*, the largest integer less than or equal to *x*. - If *x* is not a float, delegates to ``x.__floor__()``, which should return an - :class:`~numbers.Integral` value. - - -.. function:: fmod(x, y) - - Return ``fmod(x, y)``, as defined by the platform C library. Note that the - Python expression ``x % y`` may not return the same result. The intent of the C - standard is that ``fmod(x, y)`` be exactly (mathematically; to infinite - precision) equal to ``x - n*y`` for some integer *n* such that the result has - the same sign as *x* and magnitude less than ``abs(y)``. Python's ``x % y`` - returns a result with the sign of *y* instead, and may not be exactly computable - for float arguments. For example, ``fmod(-1e-100, 1e100)`` is ``-1e-100``, but - the result of Python's ``-1e-100 % 1e100`` is ``1e100-1e-100``, which cannot be - represented exactly as a float, and rounds to the surprising ``1e100``. For - this reason, function :func:`fmod` is generally preferred when working with - floats, while Python's ``x % y`` is preferred when working with integers. - - -.. function:: frexp(x) - - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. - - -.. function:: fsum(iterable) - - Return an accurate floating point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums:: - - >>> sum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 0.9999999999999999 - >>> fsum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 1.0 - - The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the - typical case where the rounding mode is half-even. On some non-Windows - builds, the underlying C library uses extended precision addition and may - occasionally double-round an intermediate sum causing it to be off in its - least significant bit. - - For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating point summation - `_\. - - -.. function:: gcd(a, b) - - Return the greatest common divisor of the integers *a* and *b*. If either - *a* or *b* is nonzero, then the value of ``gcd(a, b)`` is the largest - positive integer that divides both *a* and *b*. ``gcd(0, 0)`` returns - ``0``. - - .. versionadded:: 3.5 - - -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - -.. function:: isfinite(x) - - Return ``True`` if *x* is neither an infinity nor a NaN, and - ``False`` otherwise. (Note that ``0.0`` *is* considered finite.) - - .. versionadded:: 3.2 - - -.. function:: isinf(x) - - Return ``True`` if *x* is a positive or negative infinity, and - ``False`` otherwise. - - -.. function:: isnan(x) - - Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. - - -.. function:: ldexp(x, i) - - Return ``x * (2**i)``. This is essentially the inverse of function - :func:`frexp`. - - -.. function:: modf(x) - - Return the fractional and integer parts of *x*. Both results carry the sign - of *x* and are floats. - - -.. function:: trunc(x) - - Return the :class:`~numbers.Real` value *x* truncated to an - :class:`~numbers.Integral` (usually an integer). Delegates to - :meth:`x.__trunc__() `. - - -Note that :func:`frexp` and :func:`modf` have a different call/return pattern -than their C equivalents: they take a single argument and return a pair of -values, rather than returning their second return value through an 'output -parameter' (there is no such thing in Python). - -For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* -floating-point numbers of sufficiently large magnitude are exact integers. -Python floats typically carry no more than 53 bits of precision (the same as the -platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` -necessarily has no fractional bits. - - -Power and logarithmic functions -------------------------------- - -.. function:: exp(x) - - Return ``e**x``. - - -.. function:: expm1(x) - - Return ``e**x - 1``. For small floats *x*, the subtraction in ``exp(x) - 1`` - can result in a `significant loss of precision - `_\; the :func:`expm1` - function provides a way to compute this quantity to full precision:: - - >>> from math import exp, expm1 - >>> exp(1e-5) - 1 # gives result accurate to 11 places - 1.0000050000069649e-05 - >>> expm1(1e-5) # result accurate to full precision - 1.0000050000166668e-05 - - .. versionadded:: 3.2 - - -.. function:: log(x[, base]) - - With one argument, return the natural logarithm of *x* (to base *e*). - - With two arguments, return the logarithm of *x* to the given *base*, - calculated as ``log(x)/log(base)``. - - -.. function:: log1p(x) - - Return the natural logarithm of *1+x* (base *e*). The - result is calculated in a way which is accurate for *x* near zero. - - -.. function:: log2(x) - - Return the base-2 logarithm of *x*. This is usually more accurate than - ``log(x, 2)``. - - .. versionadded:: 3.3 - - .. seealso:: - - :meth:`int.bit_length` returns the number of bits necessary to represent - an integer in binary, excluding the sign and leading zeros. - - -.. function:: log10(x) - - Return the base-10 logarithm of *x*. This is usually more accurate - than ``log(x, 10)``. - - -.. function:: pow(x, y) - - Return ``x`` raised to the power ``y``. Exceptional cases follow - Annex 'F' of the C99 standard as far as possible. In particular, - ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` - is undefined, and raises :exc:`ValueError`. - - Unlike the built-in ``**`` operator, :func:`math.pow` converts both - its arguments to type :class:`float`. Use ``**`` or the built-in - :func:`pow` function for computing exact integer powers. - - -.. function:: sqrt(x) - - Return the square root of *x*. - -Trigonometric functions ------------------------ - - -.. function:: acos(x) - - Return the arc cosine of *x*, in radians. - - -.. function:: asin(x) - - Return the arc sine of *x*, in radians. - - -.. function:: atan(x) - - Return the arc tangent of *x*, in radians. - - -.. function:: atan2(y, x) - - Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. - The vector in the plane from the origin to point ``(x, y)`` makes this angle - with the positive X axis. The point of :func:`atan2` is that the signs of both - inputs are known to it, so it can compute the correct quadrant for the angle. - For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, - -1)`` is ``-3*pi/4``. - - -.. function:: cos(x) - - Return the cosine of *x* radians. - - -.. function:: hypot(x, y) - - Return the Euclidean norm, ``sqrt(x*x + y*y)``. This is the length of the vector - from the origin to point ``(x, y)``. - - -.. function:: sin(x) - - Return the sine of *x* radians. - - -.. function:: tan(x) - - Return the tangent of *x* radians. - -Angular conversion ------------------- - - -.. function:: degrees(x) - - Convert angle *x* from radians to degrees. - - -.. function:: radians(x) - - Convert angle *x* from degrees to radians. - -Hyperbolic functions --------------------- - -`Hyperbolic functions `_ -are analogs of trigonometric functions that are based on hyperbolas -instead of circles. - -.. function:: acosh(x) - - Return the inverse hyperbolic cosine of *x*. - - -.. function:: asinh(x) - - Return the inverse hyperbolic sine of *x*. - - -.. function:: atanh(x) - - Return the inverse hyperbolic tangent of *x*. - - -.. function:: cosh(x) - - Return the hyperbolic cosine of *x*. - - -.. function:: sinh(x) - - Return the hyperbolic sine of *x*. - - -.. function:: tanh(x) - - Return the hyperbolic tangent of *x*. - - -Special functions ------------------ - -.. function:: erf(x) - - Return the `error function `_ at - *x*. - - The :func:`erf` function can be used to compute traditional statistical - functions such as the `cumulative standard normal distribution - `_:: - - def phi(x): - 'Cumulative distribution function for the standard normal distribution' - return (1.0 + erf(x / sqrt(2.0))) / 2.0 - - .. versionadded:: 3.2 - - -.. function:: erfc(x) - - Return the complementary error function at *x*. The `complementary error - function `_ is defined as - ``1.0 - erf(x)``. It is used for large values of *x* where a subtraction - from one would cause a `loss of significance - `_\. - - .. versionadded:: 3.2 - - -.. function:: gamma(x) - - Return the `Gamma function `_ at - *x*. - - .. versionadded:: 3.2 - - -.. function:: lgamma(x) - - Return the natural logarithm of the absolute value of the Gamma - function at *x*. - - .. versionadded:: 3.2 - - -Constants ---------- - -.. data:: pi - - The mathematical constant π = 3.141592..., to available precision. - - -.. data:: e - - The mathematical constant e = 2.718281..., to available precision. - -.. data:: tau - - The mathematical constant τ = 6.283185..., to available precision. - Tau is a circle constant equal to 2π, the ratio of a circle's circumference to - its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) - Wrong `_, and start celebrating - `Tau day `_ by eating twice as much pie! - - .. versionadded:: 3.6 - -.. data:: inf - - A floating-point positive infinity. (For negative infinity, use - ``-math.inf``.) Equivalent to the output of ``float('inf')``. - - .. versionadded:: 3.5 - - -.. data:: nan - - A floating-point "not a number" (NaN) value. Equivalent to the output of - ``float('nan')``. - - .. versionadded:: 3.5 - - -.. impl-detail:: - - The :mod:`math` module consists mostly of thin wrappers around the platform C - math library functions. Behavior in exceptional cases follows Annex F of - the C99 standard where appropriate. The current implementation will raise - :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` - (where C99 Annex F recommends signaling invalid operation or divide-by-zero), - and :exc:`OverflowError` for results that overflow (for example, - ``exp(1000.0)``). A NaN will not be returned from any of the functions - above unless one or more of the input arguments was a NaN; in that case, - most functions will return a NaN, but (again following C99 Annex F) there - are some exceptions to this rule, for example ``pow(float('nan'), 0.0)`` or - ``hypot(float('nan'), float('inf'))``. - - Note that Python makes no effort to distinguish signaling NaNs from - quiet NaNs, and behavior for signaling NaNs remains unspecified. - Typical behavior is to treat all NaNs as though they were quiet. - - -.. seealso:: - - Module :mod:`cmath` - Complex number versions of many of these functions. diff --git a/third_party/python/Doc/library/mimetypes.rst b/third_party/python/Doc/library/mimetypes.rst deleted file mode 100644 index 67b7a7178..000000000 --- a/third_party/python/Doc/library/mimetypes.rst +++ /dev/null @@ -1,263 +0,0 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ - -.. module:: mimetypes - :synopsis: Mapping of filename extensions to MIME types. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/mimetypes.py` - -.. index:: pair: MIME; content type - --------------- - -The :mod:`mimetypes` module converts between a filename or URL and the MIME type -associated with the filename extension. Conversions are provided from filename -to MIME type and from MIME type to filename extension; encodings are not -supported for the latter conversion. - -The module provides one class and a number of convenience functions. The -functions are the normal interface to this module, but some applications may be -interested in the class as well. - -The functions described below provide the primary interface for this module. If -the module has not been initialized, they will call :func:`init` if they rely on -the information :func:`init` sets up. - - -.. function:: guess_type(url, strict=True) - - .. index:: pair: MIME; headers - - Guess the type of a file based on its filename or URL, given by *url*. The - return value is a tuple ``(type, encoding)`` where *type* is ``None`` if the - type can't be guessed (missing or unknown suffix) or a string of the form - ``'type/subtype'``, usable for a MIME :mailheader:`content-type` header. - - *encoding* is ``None`` for no encoding or the name of the program used to encode - (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use - as a :mailheader:`Content-Encoding` header, **not** as a - :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. - - The optional *strict* argument is a flag specifying whether the list of known MIME types - is limited to only the official types `registered with IANA - `_. - When *strict* is ``True`` (the default), only the IANA types are supported; when - *strict* is ``False``, some additional non-standard but commonly used MIME types - are also recognized. - - -.. function:: guess_all_extensions(type, strict=True) - - Guess the extensions for a file based on its MIME type, given by *type*. The - return value is a list of strings giving all possible filename extensions, - including the leading dot (``'.'``). The extensions are not guaranteed to have - been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. - - The optional *strict* argument has the same meaning as with the :func:`guess_type` function. - - -.. function:: guess_extension(type, strict=True) - - Guess the extension for a file based on its MIME type, given by *type*. The - return value is a string giving a filename extension, including the leading dot - (``'.'``). The extension is not guaranteed to have been associated with any - particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. - - The optional *strict* argument has the same meaning as with the :func:`guess_type` function. - -Some additional functions and data items are available for controlling the -behavior of the module. - - -.. function:: init(files=None) - - Initialize the internal data structures. If given, *files* must be a sequence - of file names which should be used to augment the default type map. If omitted, - the file names to use are taken from :const:`knownfiles`; on Windows, the - current registry settings are loaded. Each file named in *files* or - :const:`knownfiles` takes precedence over those named before it. Calling - :func:`init` repeatedly is allowed. - - Specifying an empty list for *files* will prevent the system defaults from - being applied: only the well-known values will be present from a built-in list. - - .. versionchanged:: 3.2 - Previously, Windows registry settings were ignored. - - -.. function:: read_mime_types(filename) - - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. - - -.. function:: add_type(type, ext, strict=True) - - Add a mapping from the MIME type *type* to the extension *ext*. When the - extension is already known, the new type will replace the old one. When the type - is already known the extension will be added to the list of known extensions. - - When *strict* is ``True`` (the default), the mapping will be added to the - official MIME types, otherwise to the non-standard ones. - - -.. data:: inited - - Flag indicating whether or not the global data structures have been initialized. - This is set to ``True`` by :func:`init`. - - -.. data:: knownfiles - - .. index:: single: file; mime.types - - List of type map file names commonly installed. These files are typically named - :file:`mime.types` and are installed in different locations by different - packages. - - -.. data:: suffix_map - - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. - - -.. data:: encodings_map - - Dictionary mapping filename extensions to encoding types. - - -.. data:: types_map - - Dictionary mapping filename extensions to MIME types. - - -.. data:: common_types - - Dictionary mapping filename extensions to non-standard, but commonly found MIME - types. - - -An example usage of the module:: - - >>> import mimetypes - >>> mimetypes.init() - >>> mimetypes.knownfiles - ['/etc/mime.types', '/etc/httpd/mime.types', ... ] - >>> mimetypes.suffix_map['.tgz'] - '.tar.gz' - >>> mimetypes.encodings_map['.gz'] - 'gzip' - >>> mimetypes.types_map['.tgz'] - 'application/x-tar-gz' - - -.. _mimetypes-objects: - -MimeTypes Objects ------------------ - -The :class:`MimeTypes` class may be useful for applications which may want more -than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. - - -.. class:: MimeTypes(filenames=(), strict=True) - - This class represents a MIME-types database. By default, it provides access to - the same database as the rest of this module. The initial database is a copy of - that provided by the module, and may be extended by loading additional - :file:`mime.types`\ -style files into the database using the :meth:`read` or - :meth:`readfp` methods. The mapping dictionaries may also be cleared before - loading additional data if the default data is not desired. - - The optional *filenames* parameter can be used to cause additional files to be - loaded "on top" of the default database. - - - .. attribute:: MimeTypes.suffix_map - - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. - - - .. attribute:: MimeTypes.encodings_map - - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. - - - .. attribute:: MimeTypes.types_map - - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. - - - .. attribute:: MimeTypes.types_map_inv - - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. - - - .. method:: MimeTypes.guess_extension(type, strict=True) - - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. - - - .. method:: MimeTypes.guess_type(url, strict=True) - - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. - - - .. method:: MimeTypes.guess_all_extensions(type, strict=True) - - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. - - - .. method:: MimeTypes.read(filename, strict=True) - - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. - - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. - - - .. method:: MimeTypes.readfp(fp, strict=True) - - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. - - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. - - - .. method:: MimeTypes.read_windows_registry(strict=True) - - Load MIME type information from the Windows registry. Availability: Windows. - - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. - - .. versionadded:: 3.2 diff --git a/third_party/python/Doc/library/misc.rst b/third_party/python/Doc/library/misc.rst deleted file mode 100644 index 094323524..000000000 --- a/third_party/python/Doc/library/misc.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _misc: - -********************** -Miscellaneous Services -********************** - -The modules described in this chapter provide miscellaneous services that are -available in all Python versions. Here's an overview: - - -.. toctree:: - - formatter.rst diff --git a/third_party/python/Doc/library/mm.rst b/third_party/python/Doc/library/mm.rst deleted file mode 100644 index c8f79c4de..000000000 --- a/third_party/python/Doc/library/mm.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _mmedia: - -******************* -Multimedia Services -******************* - -The modules described in this chapter implement various algorithms or interfaces -that are mainly useful for multimedia applications. They are available at the -discretion of the installation. Here's an overview: - - -.. toctree:: - - audioop.rst - aifc.rst - sunau.rst - wave.rst - chunk.rst - colorsys.rst - imghdr.rst - sndhdr.rst - ossaudiodev.rst diff --git a/third_party/python/Doc/library/mmap.rst b/third_party/python/Doc/library/mmap.rst deleted file mode 100644 index 6c0f9a0d1..000000000 --- a/third_party/python/Doc/library/mmap.rst +++ /dev/null @@ -1,286 +0,0 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== - -.. module:: mmap - :synopsis: Interface to memory-mapped files for Unix and Windows. - --------------- - -Memory-mapped file objects behave like both :class:`bytearray` and like -:term:`file objects `. You can use mmap objects in most places -where :class:`bytearray` are expected; for example, you can use the :mod:`re` -module to search through a memory-mapped file. You can also change a single -byte by doing ``obj[index] = 97``, or change a subsequence by assigning to a -slice: ``obj[i1:i2] = b'...'``. You can also read and write data starting at -the current file position, and :meth:`seek` through the file to different positions. - -A memory-mapped file is created by the :class:`~mmap.mmap` constructor, which is -different on Unix and on Windows. In either case you must provide a file -descriptor for a file opened for update. If you wish to map an existing Python -file object, use its :meth:`fileno` method to obtain the correct value for the -*fileno* parameter. Otherwise, you can open the file using the -:func:`os.open` function, which returns a file descriptor directly (the file -still needs to be closed when done). - -.. note:: - If you want to create a memory-mapping for a writable, buffered file, you - should :func:`~io.IOBase.flush` the file first. This is necessary to ensure - that local modifications to the buffers are actually available to the - mapping. - -For both the Unix and Windows versions of the constructor, *access* may be -specified as an optional keyword parameter. *access* accepts one of three -values: :const:`ACCESS_READ`, :const:`ACCESS_WRITE`, or :const:`ACCESS_COPY` -to specify read-only, write-through or copy-on-write memory respectively. -*access* can be used on both Unix and Windows. If *access* is not specified, -Windows mmap returns a write-through mapping. The initial memory values for -all three access types are taken from the specified file. Assignment to an -:const:`ACCESS_READ` memory map raises a :exc:`TypeError` exception. -Assignment to an :const:`ACCESS_WRITE` memory map affects both memory and the -underlying file. Assignment to an :const:`ACCESS_COPY` memory map affects -memory but does not update the underlying file. - -To map anonymous memory, -1 should be passed as the fileno along with the length. - -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset]) - - **(Windows version)** Maps *length* bytes from the file specified by the - file handle *fileno*, and creates a mmap object. If *length* is larger - than the current size of the file, the file is extended to contain *length* - bytes. If *length* is ``0``, the maximum length of the map is the current - size of the file, except that if the file is empty Windows raises an - exception (you cannot create an empty mapping on Windows). - - *tagname*, if specified and not ``None``, is a string giving a tag name for - the mapping. Windows allows you to have many different mappings against - the same file. If you specify the name of an existing tag, that tag is - opened, otherwise a new tag of this name is created. If this parameter is - omitted or ``None``, the mapping is created without a name. Avoiding the - use of the tag parameter will assist in keeping your code portable between - Unix and Windows. - - *offset* may be specified as a non-negative integer offset. mmap references - will be relative to the offset from the beginning of the file. *offset* - defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. - - -.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) - :noindex: - - **(Unix version)** Maps *length* bytes from the file specified by the file - descriptor *fileno*, and returns a mmap object. If *length* is ``0``, the - maximum length of the map will be the current size of the file when - :class:`~mmap.mmap` is called. - - *flags* specifies the nature of the mapping. :const:`MAP_PRIVATE` creates a - private copy-on-write mapping, so changes to the contents of the mmap - object will be private to this process, and :const:`MAP_SHARED` creates a - mapping that's shared with all other processes mapping the same areas of - the file. The default value is :const:`MAP_SHARED`. - - *prot*, if specified, gives the desired memory protection; the two most - useful values are :const:`PROT_READ` and :const:`PROT_WRITE`, to specify - that the pages may be read or written. *prot* defaults to - :const:`PROT_READ \| PROT_WRITE`. - - *access* may be specified in lieu of *flags* and *prot* as an optional - keyword parameter. It is an error to specify both *flags*, *prot* and - *access*. See the description of *access* above for information on how to - use this parameter. - - *offset* may be specified as a non-negative integer offset. mmap references - will be relative to the offset from the beginning of the file. *offset* - defaults to 0. *offset* must be a multiple of :const:`ALLOCATIONGRANULARITY` - which is equal to :const:`PAGESIZE` on Unix systems. - - To ensure validity of the created memory mapping the file specified - by the descriptor *fileno* is internally automatically synchronized - with physical backing store on Mac OS X and OpenVMS. - - This example shows a simple way of using :class:`~mmap.mmap`:: - - import mmap - - # write a simple example file - with open("hello.txt", "wb") as f: - f.write(b"Hello Python!\n") - - with open("hello.txt", "r+b") as f: - # memory-map the file, size 0 means whole file - mm = mmap.mmap(f.fileno(), 0) - # read content via standard file methods - print(mm.readline()) # prints b"Hello Python!\n" - # read content via slice notation - print(mm[:5]) # prints b"Hello" - # update content using slice notation; - # note that new content must have same size - mm[6:] = b" world!\n" - # ... and read again using standard file methods - mm.seek(0) - print(mm.readline()) # prints b"Hello world!\n" - # close the map - mm.close() - - - :class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with` - statement:: - - import mmap - - with mmap.mmap(-1, 13) as mm: - mm.write(b"Hello world!") - - .. versionadded:: 3.2 - Context manager support. - - - The next example demonstrates how to create an anonymous map and exchange - data between the parent and child processes:: - - import mmap - import os - - mm = mmap.mmap(-1, 13) - mm.write(b"Hello world!") - - pid = os.fork() - - if pid == 0: # In a child process - mm.seek(0) - print(mm.readline()) - - mm.close() - - - Memory-mapped file objects support the following methods: - - .. method:: close() - - Closes the mmap. Subsequent calls to other methods of the object will - result in a ValueError exception being raised. This will not close - the open file. - - - .. attribute:: closed - - ``True`` if the file is closed. - - .. versionadded:: 3.2 - - - .. method:: find(sub[, start[, end]]) - - Returns the lowest index in the object where the subsequence *sub* is - found, such that *sub* is contained in the range [*start*, *end*]. - Optional arguments *start* and *end* are interpreted as in slice notation. - Returns ``-1`` on failure. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - - .. method:: flush([offset[, size]]) - - Flushes changes made to the in-memory copy of a file back to disk. Without - use of this call there is no guarantee that changes are written back before - the object is destroyed. If *offset* and *size* are specified, only - changes to the given range of bytes will be flushed to disk; otherwise, the - whole extent of the mapping is flushed. *offset* must be a multiple of the - :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. - - **(Windows version)** A nonzero value returned indicates success; zero - indicates failure. - - **(Unix version)** A zero value is returned to indicate success. An - exception is raised when the call failed. - - - .. method:: move(dest, src, count) - - Copy the *count* bytes starting at offset *src* to the destination index - *dest*. If the mmap was created with :const:`ACCESS_READ`, then calls to - move will raise a :exc:`TypeError` exception. - - - .. method:: read([n]) - - Return a :class:`bytes` containing up to *n* bytes starting from the - current file position. If the argument is omitted, ``None`` or negative, - return all bytes from the current file position to the end of the - mapping. The file position is updated to point after the bytes that were - returned. - - .. versionchanged:: 3.3 - Argument can be omitted or ``None``. - - .. method:: read_byte() - - Returns a byte at the current file position as an integer, and advances - the file position by 1. - - - .. method:: readline() - - Returns a single line, starting at the current file position and up to the - next newline. - - - .. method:: resize(newsize) - - Resizes the map and the underlying file, if any. If the mmap was created - with :const:`ACCESS_READ` or :const:`ACCESS_COPY`, resizing the map will - raise a :exc:`TypeError` exception. - - - .. method:: rfind(sub[, start[, end]]) - - Returns the highest index in the object where the subsequence *sub* is - found, such that *sub* is contained in the range [*start*, *end*]. - Optional arguments *start* and *end* are interpreted as in slice notation. - Returns ``-1`` on failure. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - - .. method:: seek(pos[, whence]) - - Set the file's current position. *whence* argument is optional and - defaults to ``os.SEEK_SET`` or ``0`` (absolute file positioning); other - values are ``os.SEEK_CUR`` or ``1`` (seek relative to the current - position) and ``os.SEEK_END`` or ``2`` (seek relative to the file's end). - - - .. method:: size() - - Return the length of the file, which can be larger than the size of the - memory-mapped area. - - - .. method:: tell() - - Returns the current position of the file pointer. - - - .. method:: write(bytes) - - Write the bytes in *bytes* into memory at the current position of the - file pointer and return the number of bytes written (never less than - ``len(bytes)``, since if the write fails, a :exc:`ValueError` will be - raised). The file position is updated to point after the bytes that - were written. If the mmap was created with :const:`ACCESS_READ`, then - writing to it will raise a :exc:`TypeError` exception. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - .. versionchanged:: 3.6 - The number of bytes written is now returned. - - - .. method:: write_byte(byte) - - Write the integer *byte* into memory at the current - position of the file pointer; the file position is advanced by ``1``. If - the mmap was created with :const:`ACCESS_READ`, then writing to it will - raise a :exc:`TypeError` exception. diff --git a/third_party/python/Doc/library/modulefinder.rst b/third_party/python/Doc/library/modulefinder.rst deleted file mode 100644 index 7b39ce7d1..000000000 --- a/third_party/python/Doc/library/modulefinder.rst +++ /dev/null @@ -1,114 +0,0 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== - -.. module:: modulefinder - :synopsis: Find modules used by a script. - -.. sectionauthor:: A.M. Kuchling - -**Source code:** :source:`Lib/modulefinder.py` - --------------- - -This module provides a :class:`ModuleFinder` class that can be used to determine -the set of modules imported by a script. ``modulefinder.py`` can also be run as -a script, giving the filename of a Python script as its argument, after which a -report of the imported modules will be printed. - - -.. function:: AddPackagePath(pkg_name, path) - - Record that the package named *pkg_name* can be found in the specified *path*. - - -.. function:: ReplacePackage(oldname, newname) - - Allows specifying that the module named *oldname* is in fact the package named - *newname*. - - -.. class:: ModuleFinder(path=None, debug=0, excludes=[], replace_paths=[]) - - This class provides :meth:`run_script` and :meth:`report` methods to determine - the set of modules imported by a script. *path* can be a list of directories to - search for modules; if not specified, ``sys.path`` is used. *debug* sets the - debugging level; higher values make the class print debugging messages about - what it's doing. *excludes* is a list of module names to exclude from the - analysis. *replace_paths* is a list of ``(oldpath, newpath)`` tuples that will - be replaced in module paths. - - - .. method:: report() - - Print a report to standard output that lists the modules imported by the - script and their paths, as well as modules that are missing or seem to be - missing. - - .. method:: run_script(pathname) - - Analyze the contents of the *pathname* file, which must contain Python - code. - - .. attribute:: modules - - A dictionary mapping module names to modules. See - :ref:`modulefinder-example`. - - -.. _modulefinder-example: - -Example usage of :class:`ModuleFinder` --------------------------------------- - -The script that is going to get analyzed later on (bacon.py):: - - import re, itertools - - try: - import baconhameggs - except ImportError: - pass - - try: - import guido.python.ham - except ImportError: - pass - - -The script that will output the report of bacon.py:: - - from modulefinder import ModuleFinder - - finder = ModuleFinder() - finder.run_script('bacon.py') - - print('Loaded modules:') - for name, mod in finder.modules.items(): - print('%s: ' % name, end='') - print(','.join(list(mod.globalnames.keys())[:3])) - - print('-'*50) - print('Modules not imported:') - print('\n'.join(finder.badmodules.keys())) - -Sample output (may vary depending on the architecture):: - - Loaded modules: - _types: - copyreg: _inverted_registry,_slotnames,__all__ - sre_compile: isstring,_sre,_optimize_unicode - _sre: - sre_constants: REPEAT_ONE,makedict,AT_END_LINE - sys: - re: __module__,finditer,_expand - itertools: - __main__: re,itertools,baconhameggs - sre_parse: _PATTERNENDERS,SRE_FLAG_UNICODE - array: - types: __module__,IntType,TypeType - --------------------------------------------------- - Modules not imported: - guido.python.ham - baconhameggs - - diff --git a/third_party/python/Doc/library/modules.rst b/third_party/python/Doc/library/modules.rst deleted file mode 100644 index 6b2a40a1b..000000000 --- a/third_party/python/Doc/library/modules.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _modules: - -***************** -Importing Modules -***************** - -The modules described in this chapter provide new ways to import other Python -modules and hooks for customizing the import process. - -The full list of modules described in this chapter is: - - -.. toctree:: - - zipimport.rst - pkgutil.rst - modulefinder.rst - runpy.rst - importlib.rst diff --git a/third_party/python/Doc/library/msilib.rst b/third_party/python/Doc/library/msilib.rst deleted file mode 100644 index 854431742..000000000 --- a/third_party/python/Doc/library/msilib.rst +++ /dev/null @@ -1,557 +0,0 @@ -:mod:`msilib` --- Read and write Microsoft Installer files -========================================================== - -.. module:: msilib - :platform: Windows - :synopsis: Creation of Microsoft Installer files, and CAB files. - -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/msilib/__init__.py` - -.. index:: single: msi - --------------- - -The :mod:`msilib` supports the creation of Microsoft Installer (``.msi``) files. -Because these files often contain an embedded "cabinet" file (``.cab``), it also -exposes an API to create CAB files. Support for reading ``.cab`` files is -currently not implemented; read support for the ``.msi`` database is possible. - -This package aims to provide complete access to all tables in an ``.msi`` file, -therefore, it is a fairly low-level API. Two primary applications of this -package are the :mod:`distutils` command ``bdist_msi``, and the creation of -Python installer package itself (although that currently uses a different -version of ``msilib``). - -The package contents can be roughly split into four parts: low-level CAB -routines, low-level MSI routines, higher-level MSI routines, and standard table -structures. - - -.. function:: FCICreate(cabname, files) - - Create a new CAB file named *cabname*. *files* must be a list of tuples, each - containing the name of the file on disk, and the name of the file inside the CAB - file. - - The files are added to the CAB file in the order they appear in the list. All - files are added into a single CAB file, using the MSZIP compression algorithm. - - Callbacks to Python for the various steps of MSI creation are currently not - exposed. - - -.. function:: UuidCreate() - - Return the string representation of a new unique identifier. This wraps the - Windows API functions :c:func:`UuidCreate` and :c:func:`UuidToString`. - - -.. function:: OpenDatabase(path, persist) - - Return a new database object by calling MsiOpenDatabase. *path* is the file - name of the MSI file; *persist* can be one of the constants - ``MSIDBOPEN_CREATEDIRECT``, ``MSIDBOPEN_CREATE``, ``MSIDBOPEN_DIRECT``, - ``MSIDBOPEN_READONLY``, or ``MSIDBOPEN_TRANSACT``, and may include the flag - ``MSIDBOPEN_PATCHFILE``. See the Microsoft documentation for the meaning of - these flags; depending on the flags, an existing database is opened, or a new - one created. - - -.. function:: CreateRecord(count) - - Return a new record object by calling :c:func:`MSICreateRecord`. *count* is the - number of fields of the record. - - -.. function:: init_database(name, schema, ProductName, ProductCode, ProductVersion, Manufacturer) - - Create and return a new database *name*, initialize it with *schema*, and set - the properties *ProductName*, *ProductCode*, *ProductVersion*, and - *Manufacturer*. - - *schema* must be a module object containing ``tables`` and - ``_Validation_records`` attributes; typically, :mod:`msilib.schema` should be - used. - - The database will contain just the schema and the validation records when this - function returns. - - -.. function:: add_data(database, table, records) - - Add all *records* to the table named *table* in *database*. - - The *table* argument must be one of the predefined tables in the MSI schema, - e.g. ``'Feature'``, ``'File'``, ``'Component'``, ``'Dialog'``, ``'Control'``, - etc. - - *records* should be a list of tuples, each one containing all fields of a - record according to the schema of the table. For optional fields, - ``None`` can be passed. - - Field values can be ints, strings, or instances of the Binary class. - - -.. class:: Binary(filename) - - Represents entries in the Binary table; inserting such an object using - :func:`add_data` reads the file named *filename* into the table. - - -.. function:: add_tables(database, module) - - Add all table content from *module* to *database*. *module* must contain an - attribute *tables* listing all tables for which content should be added, and one - attribute per table that has the actual content. - - This is typically used to install the sequence tables. - - -.. function:: add_stream(database, name, path) - - Add the file *path* into the ``_Stream`` table of *database*, with the stream - name *name*. - - -.. function:: gen_uuid() - - Return a new UUID, in the format that MSI typically requires (i.e. in curly - braces, and with all hexdigits in upper-case). - - -.. seealso:: - - `FCICreate `_ - `UuidCreate `_ - `UuidToString `_ - -.. _database-objects: - -Database Objects ----------------- - - -.. method:: Database.OpenView(sql) - - Return a view object, by calling :c:func:`MSIDatabaseOpenView`. *sql* is the SQL - statement to execute. - - -.. method:: Database.Commit() - - Commit the changes pending in the current transaction, by calling - :c:func:`MSIDatabaseCommit`. - - -.. method:: Database.GetSummaryInformation(count) - - Return a new summary information object, by calling - :c:func:`MsiGetSummaryInformation`. *count* is the maximum number of updated - values. - - -.. seealso:: - - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ - -.. _view-objects: - -View Objects ------------- - - -.. method:: View.Execute(params) - - Execute the SQL query of the view, through :c:func:`MSIViewExecute`. If - *params* is not ``None``, it is a record describing actual values of the - parameter tokens in the query. - - -.. method:: View.GetColumnInfo(kind) - - Return a record describing the columns of the view, through calling - :c:func:`MsiViewGetColumnInfo`. *kind* can be either ``MSICOLINFO_NAMES`` or - ``MSICOLINFO_TYPES``. - - -.. method:: View.Fetch() - - Return a result record of the query, through calling :c:func:`MsiViewFetch`. - - -.. method:: View.Modify(kind, data) - - Modify the view, by calling :c:func:`MsiViewModify`. *kind* can be one of - ``MSIMODIFY_SEEK``, ``MSIMODIFY_REFRESH``, ``MSIMODIFY_INSERT``, - ``MSIMODIFY_UPDATE``, ``MSIMODIFY_ASSIGN``, ``MSIMODIFY_REPLACE``, - ``MSIMODIFY_MERGE``, ``MSIMODIFY_DELETE``, ``MSIMODIFY_INSERT_TEMPORARY``, - ``MSIMODIFY_VALIDATE``, ``MSIMODIFY_VALIDATE_NEW``, - ``MSIMODIFY_VALIDATE_FIELD``, or ``MSIMODIFY_VALIDATE_DELETE``. - - *data* must be a record describing the new data. - - -.. method:: View.Close() - - Close the view, through :c:func:`MsiViewClose`. - - -.. seealso:: - - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ - -.. _summary-objects: - -Summary Information Objects ---------------------------- - - -.. method:: SummaryInformation.GetProperty(field) - - Return a property of the summary, through :c:func:`MsiSummaryInfoGetProperty`. - *field* is the name of the property, and can be one of the constants - ``PID_CODEPAGE``, ``PID_TITLE``, ``PID_SUBJECT``, ``PID_AUTHOR``, - ``PID_KEYWORDS``, ``PID_COMMENTS``, ``PID_TEMPLATE``, ``PID_LASTAUTHOR``, - ``PID_REVNUMBER``, ``PID_LASTPRINTED``, ``PID_CREATE_DTM``, - ``PID_LASTSAVE_DTM``, ``PID_PAGECOUNT``, ``PID_WORDCOUNT``, ``PID_CHARCOUNT``, - ``PID_APPNAME``, or ``PID_SECURITY``. - - -.. method:: SummaryInformation.GetPropertyCount() - - Return the number of summary properties, through - :c:func:`MsiSummaryInfoGetPropertyCount`. - - -.. method:: SummaryInformation.SetProperty(field, value) - - Set a property through :c:func:`MsiSummaryInfoSetProperty`. *field* can have the - same values as in :meth:`GetProperty`, *value* is the new value of the property. - Possible value types are integer and string. - - -.. method:: SummaryInformation.Persist() - - Write the modified properties to the summary information stream, using - :c:func:`MsiSummaryInfoPersist`. - - -.. seealso:: - - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ - -.. _record-objects: - -Record Objects --------------- - - -.. method:: Record.GetFieldCount() - - Return the number of fields of the record, through - :c:func:`MsiRecordGetFieldCount`. - - -.. method:: Record.GetInteger(field) - - Return the value of *field* as an integer where possible. *field* must - be an integer. - - -.. method:: Record.GetString(field) - - Return the value of *field* as a string where possible. *field* must - be an integer. - - -.. method:: Record.SetString(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetString`. *field* must be an - integer; *value* a string. - - -.. method:: Record.SetStream(field, value) - - Set *field* to the contents of the file named *value*, through - :c:func:`MsiRecordSetStream`. *field* must be an integer; *value* a string. - - -.. method:: Record.SetInteger(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetInteger`. Both *field* and - *value* must be an integer. - - -.. method:: Record.ClearData() - - Set all fields of the record to 0, through :c:func:`MsiRecordClearData`. - - -.. seealso:: - - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClearData `_ - -.. _msi-errors: - -Errors ------- - -All wrappers around MSI functions raise :exc:`MSIError`; the string inside the -exception will contain more detail. - - -.. _cab: - -CAB Objects ------------ - - -.. class:: CAB(name) - - The class :class:`CAB` represents a CAB file. During MSI construction, files - will be added simultaneously to the ``Files`` table, and to a CAB file. Then, - when all files have been added, the CAB file can be written, then added to the - MSI file. - - *name* is the name of the CAB file in the MSI file. - - - .. method:: append(full, file, logical) - - Add the file with the pathname *full* to the CAB file, under the name - *logical*. If there is already a file named *logical*, a new file name is - created. - - Return the index of the file in the CAB file, and the new name of the file - inside the CAB file. - - - .. method:: commit(database) - - Generate a CAB file, add it as a stream to the MSI file, put it into the - ``Media`` table, and remove the generated file from the disk. - - -.. _msi-directory: - -Directory Objects ------------------ - - -.. class:: Directory(database, cab, basedir, physical, logical, default, [componentflags]) - - Create a new directory in the Directory table. There is a current component at - each point in time for the directory, which is either explicitly created through - :meth:`start_component`, or implicitly when files are added for the first time. - Files are added into the current component, and into the cab file. To create a - directory, a base directory object needs to be specified (can be ``None``), the - path to the physical directory, and a logical directory name. *default* - specifies the DefaultDir slot in the directory table. *componentflags* specifies - the default flags that new components get. - - - .. method:: start_component(component=None, feature=None, flags=None, keyfile=None, uuid=None) - - Add an entry to the Component table, and make this component the current - component for this directory. If no component name is given, the directory - name is used. If no *feature* is given, the current feature is used. If no - *flags* are given, the directory's default flags are used. If no *keyfile* - is given, the KeyPath is left null in the Component table. - - - .. method:: add_file(file, src=None, version=None, language=None) - - Add a file to the current component of the directory, starting a new one - if there is no current component. By default, the file name in the source - and the file table will be identical. If the *src* file is specified, it - is interpreted relative to the current directory. Optionally, a *version* - and a *language* can be specified for the entry in the File table. - - - .. method:: glob(pattern, exclude=None) - - Add a list of files to the current component as specified in the glob - pattern. Individual files can be excluded in the *exclude* list. - - - .. method:: remove_pyc() - - Remove ``.pyc`` files on uninstall. - - -.. seealso:: - - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ - -.. _features: - -Features --------- - - -.. class:: Feature(db, id, title, desc, display, level=1, parent=None, directory=None, attributes=0) - - Add a new record to the ``Feature`` table, using the values *id*, *parent.id*, - *title*, *desc*, *display*, *level*, *directory*, and *attributes*. The - resulting feature object can be passed to the :meth:`start_component` method of - :class:`Directory`. - - - .. method:: set_current() - - Make this feature the current feature of :mod:`msilib`. New components are - automatically added to the default feature, unless a feature is explicitly - specified. - - -.. seealso:: - - `Feature Table `_ - -.. _msi-gui: - -GUI classes ------------ - -:mod:`msilib` provides several classes that wrap the GUI tables in an MSI -database. However, no standard user interface is provided; use -:mod:`~distutils.command.bdist_msi` to create MSI files with a user-interface -for installing Python packages. - - -.. class:: Control(dlg, name) - - Base class of the dialog controls. *dlg* is the dialog object the control - belongs to, and *name* is the control's name. - - - .. method:: event(event, argument, condition=1, ordering=None) - - Make an entry into the ``ControlEvent`` table for this control. - - - .. method:: mapping(event, attribute) - - Make an entry into the ``EventMapping`` table for this control. - - - .. method:: condition(action, condition) - - Make an entry into the ``ControlCondition`` table for this control. - - -.. class:: RadioButtonGroup(dlg, name, property) - - Create a radio button control named *name*. *property* is the installer property - that gets set when a radio button is selected. - - - .. method:: add(name, x, y, width, height, text, value=None) - - Add a radio button named *name* to the group, at the coordinates *x*, *y*, - *width*, *height*, and with the label *text*. If *value* is ``None``, it - defaults to *name*. - - -.. class:: Dialog(db, name, x, y, w, h, attr, title, first, default, cancel) - - Return a new :class:`Dialog` object. An entry in the ``Dialog`` table is made, - with the specified coordinates, dialog attributes, title, name of the first, - default, and cancel controls. - - - .. method:: control(name, type, x, y, width, height, attributes, property, text, control_next, help) - - Return a new :class:`Control` object. An entry in the ``Control`` table is - made with the specified parameters. - - This is a generic method; for specific types, specialized methods are - provided. - - - .. method:: text(name, x, y, width, height, attributes, text) - - Add and return a ``Text`` control. - - - .. method:: bitmap(name, x, y, width, height, text) - - Add and return a ``Bitmap`` control. - - - .. method:: line(name, x, y, width, height) - - Add and return a ``Line`` control. - - - .. method:: pushbutton(name, x, y, width, height, attributes, text, next_control) - - Add and return a ``PushButton`` control. - - - .. method:: radiogroup(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``RadioButtonGroup`` control. - - - .. method:: checkbox(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``CheckBox`` control. - - -.. seealso:: - - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ - -.. _msi-tables: - -Precomputed tables ------------------- - -:mod:`msilib` provides a few subpackages that contain only schema and table -definitions. Currently, these definitions are based on MSI version 2.0. - - -.. data:: schema - - This is the standard MSI schema for MSI 2.0, with the *tables* variable - providing a list of table definitions, and *_Validation_records* providing the - data for MSI validation. - - -.. data:: sequence - - This module contains table contents for the standard sequence tables: - *AdminExecuteSequence*, *AdminUISequence*, *AdvtExecuteSequence*, - *InstallExecuteSequence*, and *InstallUISequence*. - - -.. data:: text - - This module contains definitions for the UIText and ActionText tables, for the - standard installer actions. diff --git a/third_party/python/Doc/library/msvcrt.rst b/third_party/python/Doc/library/msvcrt.rst deleted file mode 100644 index bd34ffb1b..000000000 --- a/third_party/python/Doc/library/msvcrt.rst +++ /dev/null @@ -1,154 +0,0 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== - -.. module:: msvcrt - :platform: Windows - :synopsis: Miscellaneous useful routines from the MS VC++ runtime. - -.. sectionauthor:: Fred L. Drake, Jr. - --------------- - -These functions provide access to some useful capabilities on Windows platforms. -Some higher-level modules use these functions to build the Windows -implementations of their services. For example, the :mod:`getpass` module uses -this in the implementation of the :func:`getpass` function. - -Further documentation on these functions can be found in the Platform API -documentation. - -The module implements both the normal and wide char variants of the console I/O -api. The normal API deals only with ASCII characters and is of limited use -for internationalized applications. The wide char API should be used where -ever possible. - -.. versionchanged:: 3.3 - Operations in this module now raise :exc:`OSError` where :exc:`IOError` - was raised. - - -.. _msvcrt-files: - -File Operations ---------------- - - -.. function:: locking(fd, mode, nbytes) - - Lock part of a file based on file descriptor *fd* from the C runtime. Raises - :exc:`OSError` on failure. The locked region of the file extends from the - current file position for *nbytes* bytes, and may continue beyond the end of the - file. *mode* must be one of the :const:`LK_\*` constants listed below. Multiple - regions in a file may be locked at the same time, but may not overlap. Adjacent - regions are not merged; they must be unlocked individually. - - -.. data:: LK_LOCK - LK_RLCK - - Locks the specified bytes. If the bytes cannot be locked, the program - immediately tries again after 1 second. If, after 10 attempts, the bytes cannot - be locked, :exc:`OSError` is raised. - - -.. data:: LK_NBLCK - LK_NBRLCK - - Locks the specified bytes. If the bytes cannot be locked, :exc:`OSError` is - raised. - - -.. data:: LK_UNLCK - - Unlocks the specified bytes, which must have been previously locked. - - -.. function:: setmode(fd, flags) - - Set the line-end translation mode for the file descriptor *fd*. To set it to - text mode, *flags* should be :const:`os.O_TEXT`; for binary, it should be - :const:`os.O_BINARY`. - - -.. function:: open_osfhandle(handle, flags) - - Create a C runtime file descriptor from the file handle *handle*. The *flags* - parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter - to :func:`os.fdopen` to create a file object. - - -.. function:: get_osfhandle(fd) - - Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if - *fd* is not recognized. - - -.. _msvcrt-console: - -Console I/O ------------ - - -.. function:: kbhit() - - Return true if a keypress is waiting to be read. - - -.. function:: getch() - - Read a keypress and return the resulting character as a byte string. - Nothing is echoed to the console. This call will block if a keypress - is not already available, but will not wait for :kbd:`Enter` to be - pressed. If the pressed key was a special function key, this will - return ``'\000'`` or ``'\xe0'``; the next call will return the keycode. - The :kbd:`Control-C` keypress cannot be read with this function. - - -.. function:: getwch() - - Wide char variant of :func:`getch`, returning a Unicode value. - - -.. function:: getche() - - Similar to :func:`getch`, but the keypress will be echoed if it represents a - printable character. - - -.. function:: getwche() - - Wide char variant of :func:`getche`, returning a Unicode value. - - -.. function:: putch(char) - - Print the byte string *char* to the console without buffering. - - -.. function:: putwch(unicode_char) - - Wide char variant of :func:`putch`, accepting a Unicode value. - - -.. function:: ungetch(char) - - Cause the byte string *char* to be "pushed back" into the console buffer; - it will be the next character read by :func:`getch` or :func:`getche`. - - -.. function:: ungetwch(unicode_char) - - Wide char variant of :func:`ungetch`, accepting a Unicode value. - - -.. _msvcrt-other: - -Other Functions ---------------- - - -.. function:: heapmin() - - Force the :c:func:`malloc` heap to clean itself up and return unused blocks to - the operating system. On failure, this raises :exc:`OSError`. diff --git a/third_party/python/Doc/library/multiprocessing.rst b/third_party/python/Doc/library/multiprocessing.rst deleted file mode 100644 index a3cdfd74e..000000000 --- a/third_party/python/Doc/library/multiprocessing.rst +++ /dev/null @@ -1,2835 +0,0 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== - -.. module:: multiprocessing - :synopsis: Process-based parallelism. - -**Source code:** :source:`Lib/multiprocessing/` - --------------- - -Introduction ------------- - -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package -offers both local and remote concurrency, effectively side-stepping the -:term:`Global Interpreter Lock` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully -leverage multiple processors on a given machine. It runs on both Unix and -Windows. - -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the -:class:`~multiprocessing.pool.Pool` object which offers a convenient means of -parallelizing the execution of a function across multiple input values, -distributing the input data across processes (data parallelism). The following -example demonstrates the common practice of defining such functions in a module -so that child processes can successfully import that module. This basic example -of data parallelism using :class:`~multiprocessing.pool.Pool`, :: - - from multiprocessing import Pool - - def f(x): - return x*x - - if __name__ == '__main__': - with Pool(5) as p: - print(p.map(f, [1, 2, 3])) - -will print to standard output :: - - [1, 4, 9] - - -The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` -object and then calling its :meth:`~Process.start` method. :class:`Process` -follows the API of :class:`threading.Thread`. A trivial example of a -multiprocess program is :: - - from multiprocessing import Process - - def f(name): - print('hello', name) - - if __name__ == '__main__': - p = Process(target=f, args=('bob',)) - p.start() - p.join() - -To show the individual process IDs involved, here is an expanded example:: - - from multiprocessing import Process - import os - - def info(title): - print(title) - print('module name:', __name__) - print('parent process:', os.getppid()) - print('process id:', os.getpid()) - - def f(name): - info('function f') - print('hello', name) - - if __name__ == '__main__': - info('main line') - p = Process(target=f, args=('bob',)) - p.start() - p.join() - -For an explanation of why the ``if __name__ == '__main__'`` part is -necessary, see :ref:`multiprocessing-programming`. - - - -Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. _multiprocessing-start-methods: - -Depending on the platform, :mod:`multiprocessing` supports three ways -to start a process. These *start methods* are - - *spawn* - The parent process starts a fresh python interpreter process. The - child process will only inherit those resources necessary to run - the process objects :meth:`~Process.run` method. In particular, - unnecessary file descriptors and handles from the parent process - will not be inherited. Starting a process using this method is - rather slow compared to using *fork* or *forkserver*. - - Available on Unix and Windows. The default on Windows. - - *fork* - The parent process uses :func:`os.fork` to fork the Python - interpreter. The child process, when it begins, is effectively - identical to the parent process. All resources of the parent are - inherited by the child process. Note that safely forking a - multithreaded process is problematic. - - Available on Unix only. The default on Unix. - - *forkserver* - When the program starts and selects the *forkserver* start method, - a server process is started. From then on, whenever a new process - is needed, the parent process connects to the server and requests - that it fork a new process. The fork server process is single - threaded so it is safe for it to use :func:`os.fork`. No - unnecessary resources are inherited. - - Available on Unix platforms which support passing file descriptors - over Unix pipes. - -.. versionchanged:: 3.4 - *spawn* added on all unix platforms, and *forkserver* added for - some unix platforms. - Child processes no longer inherit all of the parents inheritable - handles on Windows. - -On Unix using the *spawn* or *forkserver* start methods will also -start a *semaphore tracker* process which tracks the unlinked named -semaphores created by processes of the program. When all processes -have exited the semaphore tracker unlinks any remaining semaphores. -Usually there should be none, but if a process was killed by a signal -there may be some "leaked" semaphores. (Unlinking the named semaphores -is a serious matter since the system allows only a limited number, and -they will not be automatically unlinked until the next reboot.) - -To select a start method you use the :func:`set_start_method` in -the ``if __name__ == '__main__'`` clause of the main module. For -example:: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - mp.set_start_method('spawn') - q = mp.Queue() - p = mp.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -:func:`set_start_method` should not be used more than once in the -program. - -Alternatively, you can use :func:`get_context` to obtain a context -object. Context objects have the same API as the multiprocessing -module, and allow one to use multiple start methods in the same -program. :: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - ctx = mp.get_context('spawn') - q = ctx.Queue() - p = ctx.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -Note that objects related to one context may not be compatible with -processes for a different context. In particular, locks created using -the *fork* context cannot be passed to processes started using the -*spawn* or *forkserver* start methods. - -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. - - -Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:mod:`multiprocessing` supports two types of communication channel between -processes: - -**Queues** - - The :class:`Queue` class is a near clone of :class:`queue.Queue`. For - example:: - - from multiprocessing import Process, Queue - - def f(q): - q.put([42, None, 'hello']) - - if __name__ == '__main__': - q = Queue() - p = Process(target=f, args=(q,)) - p.start() - print(q.get()) # prints "[42, None, 'hello']" - p.join() - - Queues are thread and process safe. - -**Pipes** - - The :func:`Pipe` function returns a pair of connection objects connected by a - pipe which by default is duplex (two-way). For example:: - - from multiprocessing import Process, Pipe - - def f(conn): - conn.send([42, None, 'hello']) - conn.close() - - if __name__ == '__main__': - parent_conn, child_conn = Pipe() - p = Process(target=f, args=(child_conn,)) - p.start() - print(parent_conn.recv()) # prints "[42, None, 'hello']" - p.join() - - The two connection objects returned by :func:`Pipe` represent the two ends of - the pipe. Each connection object has :meth:`~Connection.send` and - :meth:`~Connection.recv` methods (among others). Note that data in a pipe - may become corrupted if two processes (or threads) try to read from or write - to the *same* end of the pipe at the same time. Of course there is no risk - of corruption from processes using different ends of the pipe at the same - time. - - -Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:mod:`multiprocessing` contains equivalents of all the synchronization -primitives from :mod:`threading`. For instance one can use a lock to ensure -that only one process prints to standard output at a time:: - - from multiprocessing import Process, Lock - - def f(l, i): - l.acquire() - try: - print('hello world', i) - finally: - l.release() - - if __name__ == '__main__': - lock = Lock() - - for num in range(10): - Process(target=f, args=(lock, num)).start() - -Without using the lock output from the different processes is liable to get all -mixed up. - - -Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As mentioned above, when doing concurrent programming it is usually best to -avoid using shared state as far as possible. This is particularly true when -using multiple processes. - -However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. - -**Shared memory** - - Data can be stored in a shared memory map using :class:`Value` or - :class:`Array`. For example, the following code :: - - from multiprocessing import Process, Value, Array - - def f(n, a): - n.value = 3.1415927 - for i in range(len(a)): - a[i] = -a[i] - - if __name__ == '__main__': - num = Value('d', 0.0) - arr = Array('i', range(10)) - - p = Process(target=f, args=(num, arr)) - p.start() - p.join() - - print(num.value) - print(arr[:]) - - will print :: - - 3.1415927 - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] - - The ``'d'`` and ``'i'`` arguments used when creating ``num`` and ``arr`` are - typecodes of the kind used by the :mod:`array` module: ``'d'`` indicates a - double precision float and ``'i'`` indicates a signed integer. These shared - objects will be process and thread-safe. - - For more flexibility in using shared memory one can use the - :mod:`multiprocessing.sharedctypes` module which supports the creation of - arbitrary ctypes objects allocated from shared memory. - -**Server process** - - A manager object returned by :func:`Manager` controls a server process which - holds Python objects and allows other processes to manipulate them using - proxies. - - A manager returned by :func:`Manager` will support types - :class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`, - :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, - :class:`Condition`, :class:`Event`, :class:`Barrier`, - :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: - - from multiprocessing import Process, Manager - - def f(d, l): - d[1] = '1' - d['2'] = 2 - d[0.25] = None - l.reverse() - - if __name__ == '__main__': - with Manager() as manager: - d = manager.dict() - l = manager.list(range(10)) - - p = Process(target=f, args=(d, l)) - p.start() - p.join() - - print(d) - print(l) - - will print :: - - {0.25: None, 1: '1', '2': 2} - [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] - - Server process managers are more flexible than using shared memory objects - because they can be made to support arbitrary object types. Also, a single - manager can be shared by processes on different computers over a network. - They are, however, slower than using shared memory. - - -Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`~multiprocessing.pool.Pool` class represents a pool of worker -processes. It has methods which allows tasks to be offloaded to the worker -processes in a few different ways. - -For example:: - - from multiprocessing import Pool, TimeoutError - import time - import os - - def f(x): - return x*x - - if __name__ == '__main__': - # start 4 worker processes - with Pool(processes=4) as pool: - - # print "[0, 1, 4,..., 81]" - print(pool.map(f, range(10))) - - # print same numbers in arbitrary order - for i in pool.imap_unordered(f, range(10)): - print(i) - - # evaluate "f(20)" asynchronously - res = pool.apply_async(f, (20,)) # runs in *only* one process - print(res.get(timeout=1)) # prints "400" - - # evaluate "os.getpid()" asynchronously - res = pool.apply_async(os.getpid, ()) # runs in *only* one process - print(res.get(timeout=1)) # prints the PID of that process - - # launching multiple evaluations asynchronously *may* use more processes - multiple_results = [pool.apply_async(os.getpid, ()) for i in range(4)] - print([res.get(timeout=1) for res in multiple_results]) - - # make a single worker sleep for 10 secs - res = pool.apply_async(time.sleep, (10,)) - try: - print(res.get(timeout=1)) - except TimeoutError: - print("We lacked patience and got a multiprocessing.TimeoutError") - - print("For the moment, the pool remains available for more work") - - # exiting the 'with'-block has stopped the pool - print("Now the pool is closed and no longer available") - -Note that the methods of a pool should only ever be used by the -process which created it. - -.. note:: - - Functionality within this package requires that the ``__main__`` module be - importable by the children. This is covered in :ref:`multiprocessing-programming` - however it is worth pointing out here. This means that some examples, such - as the :class:`multiprocessing.pool.Pool` examples will not work in the - interactive interpreter. For example:: - - >>> from multiprocessing import Pool - >>> p = Pool(5) - >>> def f(x): - ... return x*x - ... - >>> p.map(f, [1,2,3]) - Process PoolWorker-1: - Process PoolWorker-2: - Process PoolWorker-3: - Traceback (most recent call last): - Traceback (most recent call last): - Traceback (most recent call last): - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' - - (If you try this it will actually output three full tracebacks - interleaved in a semi-random fashion, and then you may have to - stop the master process somehow.) - - -Reference ---------- - -The :mod:`multiprocessing` package mostly replicates the API of the -:mod:`threading` module. - - -:class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ - *, daemon=None) - - Process objects represent activity that is run in a separate process. The - :class:`Process` class has equivalents of all the methods of - :class:`threading.Thread`. - - The constructor should always be called with keyword arguments. *group* - should always be ``None``; it exists solely for compatibility with - :class:`threading.Thread`. *target* is the callable object to be invoked by - the :meth:`run()` method. It defaults to ``None``, meaning nothing is - called. *name* is the process name (see :attr:`name` for more details). - *args* is the argument tuple for the target invocation. *kwargs* is a - dictionary of keyword arguments for the target invocation. If provided, - the keyword-only *daemon* argument sets the process :attr:`daemon` flag - to ``True`` or ``False``. If ``None`` (the default), this flag will be - inherited from the creating process. - - By default, no arguments are passed to *target*. - - If a subclass overrides the constructor, it must make sure it invokes the - base class constructor (:meth:`Process.__init__`) before doing anything else - to the process. - - .. versionchanged:: 3.3 - Added the *daemon* argument. - - .. method:: run() - - Method representing the process's activity. - - You may override this method in a subclass. The standard :meth:`run` - method invokes the callable object passed to the object's constructor as - the target argument, if any, with sequential and keyword arguments taken - from the *args* and *kwargs* arguments, respectively. - - .. method:: start() - - Start the process's activity. - - This must be called at most once per process object. It arranges for the - object's :meth:`run` method to be invoked in a separate process. - - .. method:: join([timeout]) - - If the optional argument *timeout* is ``None`` (the default), the method - blocks until the process whose :meth:`join` method is called terminates. - If *timeout* is a positive number, it blocks at most *timeout* seconds. - Note that the method returns ``None`` if its process terminates or if the - method times out. Check the process's :attr:`exitcode` to determine if - it terminated. - - A process can be joined many times. - - A process cannot join itself because this would cause a deadlock. It is - an error to attempt to join a process before it has been started. - - .. attribute:: name - - The process's name. The name is a string used for identification purposes - only. It has no semantics. Multiple processes may be given the same - name. - - The initial name is set by the constructor. If no explicit name is - provided to the constructor, a name of the form - 'Process-N\ :sub:`1`:N\ :sub:`2`:...:N\ :sub:`k`' is constructed, where - each N\ :sub:`k` is the N-th child of its parent. - - .. method:: is_alive - - Return whether the process is alive. - - Roughly, a process object is alive from the moment the :meth:`start` - method returns until the child process terminates. - - .. attribute:: daemon - - The process's daemon flag, a Boolean value. This must be set before - :meth:`start` is called. - - The initial value is inherited from the creating process. - - When a process exits, it attempts to terminate all of its daemonic child - processes. - - Note that a daemonic process is not allowed to create child processes. - Otherwise a daemonic process would leave its children orphaned if it gets - terminated when its parent process exits. Additionally, these are **not** - Unix daemons or services, they are normal processes that will be - terminated (and not joined) if non-daemonic processes have exited. - - In addition to the :class:`threading.Thread` API, :class:`Process` objects - also support the following attributes and methods: - - .. attribute:: pid - - Return the process ID. Before the process is spawned, this will be - ``None``. - - .. attribute:: exitcode - - The child's exit code. This will be ``None`` if the process has not yet - terminated. A negative value *-N* indicates that the child was terminated - by signal *N*. - - .. attribute:: authkey - - The process's authentication key (a byte string). - - When :mod:`multiprocessing` is initialized the main process is assigned a - random string using :func:`os.urandom`. - - When a :class:`Process` object is created, it will inherit the - authentication key of its parent process, although this may be changed by - setting :attr:`authkey` to another byte string. - - See :ref:`multiprocessing-auth-keys`. - - .. attribute:: sentinel - - A numeric handle of a system object which will become "ready" when - the process ends. - - You can use this value if you want to wait on several events at - once using :func:`multiprocessing.connection.wait`. Otherwise - calling :meth:`join()` is simpler. - - On Windows, this is an OS handle usable with the ``WaitForSingleObject`` - and ``WaitForMultipleObjects`` family of API calls. On Unix, this is - a file descriptor usable with primitives from the :mod:`select` module. - - .. versionadded:: 3.3 - - .. method:: terminate() - - Terminate the process. On Unix this is done using the ``SIGTERM`` signal; - on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and - finally clauses, etc., will not be executed. - - Note that descendant processes of the process will *not* be terminated -- - they will simply become orphaned. - - .. warning:: - - If this method is used when the associated process is using a pipe or - queue then the pipe or queue is liable to become corrupted and may - become unusable by other process. Similarly, if the process has - acquired a lock or semaphore etc. then terminating it is liable to - cause other processes to deadlock. - - Note that the :meth:`start`, :meth:`join`, :meth:`is_alive`, - :meth:`terminate` and :attr:`exitcode` methods should only be called by - the process that created the process object. - - Example usage of some of the methods of :class:`Process`: - - .. doctest:: - - >>> import multiprocessing, time, signal - >>> p = multiprocessing.Process(target=time.sleep, args=(1000,)) - >>> print(p, p.is_alive()) - False - >>> p.start() - >>> print(p, p.is_alive()) - True - >>> p.terminate() - >>> time.sleep(0.1) - >>> print(p, p.is_alive()) - False - >>> p.exitcode == -signal.SIGTERM - True - -.. exception:: ProcessError - - The base class of all :mod:`multiprocessing` exceptions. - -.. exception:: BufferTooShort - - Exception raised by :meth:`Connection.recv_bytes_into()` when the supplied - buffer object is too small for the message read. - - If ``e`` is an instance of :exc:`BufferTooShort` then ``e.args[0]`` will give - the message as a byte string. - -.. exception:: AuthenticationError - - Raised when there is an authentication error. - -.. exception:: TimeoutError - - Raised by methods with a timeout when the timeout expires. - -Pipes and Queues -~~~~~~~~~~~~~~~~ - -When using multiple processes, one generally uses message passing for -communication between processes and avoids having to use any synchronization -primitives like locks. - -For passing messages one can use :func:`Pipe` (for a connection between two -processes) or a queue (which allows multiple producers and consumers). - -The :class:`Queue`, :class:`SimpleQueue` and :class:`JoinableQueue` types -are multi-producer, multi-consumer :abbr:`FIFO (first-in, first-out)` -queues modelled on the :class:`queue.Queue` class in the -standard library. They differ in that :class:`Queue` lacks the -:meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join` methods introduced -into Python 2.5's :class:`queue.Queue` class. - -If you use :class:`JoinableQueue` then you **must** call -:meth:`JoinableQueue.task_done` for each task removed from the queue or else the -semaphore used to count the number of unfinished tasks may eventually overflow, -raising an exception. - -Note that one can also create a shared queue by using a manager object -- see -:ref:`multiprocessing-managers`. - -.. note:: - - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and - :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from - :mod:`queue`. - -.. note:: - - When an object is put on a queue, the object is pickled and a - background thread later flushes the pickled data to an underlying - pipe. This has some consequences which are a little surprising, - but should not cause any practical difficulties -- if they really - bother you then you can instead use a queue created with a - :ref:`manager `. - - (1) After putting an object on an empty queue there may be an - infinitesimal delay before the queue's :meth:`~Queue.empty` - method returns :const:`False` and :meth:`~Queue.get_nowait` can - return without raising :exc:`queue.Empty`. - - (2) If multiple processes are enqueuing objects, it is possible for - the objects to be received at the other end out-of-order. - However, objects enqueued by the same process will always be in - the expected order with respect to each other. - -.. warning:: - - If a process is killed using :meth:`Process.terminate` or :func:`os.kill` - while it is trying to use a :class:`Queue`, then the data in the queue is - likely to become corrupted. This may cause any other process to get an - exception when it tries to use the queue later on. - -.. warning:: - - As mentioned above, if a child process has put items on a queue (and it has - not used :meth:`JoinableQueue.cancel_join_thread - `), then that process will - not terminate until all buffered items have been flushed to the pipe. - - This means that if you try joining that process you may get a deadlock unless - you are sure that all items which have been put on the queue have been - consumed. Similarly, if the child process is non-daemonic then the parent - process may hang on exit when it tries to join all its non-daemonic children. - - Note that a queue created using a manager does not have this issue. See - :ref:`multiprocessing-programming`. - -For an example of the usage of queues for interprocess communication see -:ref:`multiprocessing-examples`. - - -.. function:: Pipe([duplex]) - - Returns a pair ``(conn1, conn2)`` of - :class:`~multiprocessing.connection.Connection` objects representing the - ends of a pipe. - - If *duplex* is ``True`` (the default) then the pipe is bidirectional. If - *duplex* is ``False`` then the pipe is unidirectional: ``conn1`` can only be - used for receiving messages and ``conn2`` can only be used for sending - messages. - - -.. class:: Queue([maxsize]) - - Returns a process shared queue implemented using a pipe and a few - locks/semaphores. When a process first puts an item on the queue a feeder - thread is started which transfers objects from a buffer into the pipe. - - The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the - standard library's :mod:`queue` module are raised to signal timeouts. - - :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. - - .. method:: qsize() - - Return the approximate size of the queue. Because of - multithreading/multiprocessing semantics, this number is not reliable. - - Note that this may raise :exc:`NotImplementedError` on Unix platforms like - Mac OS X where ``sem_getvalue()`` is not implemented. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. Because of - multithreading/multiprocessing semantics, this is not reliable. - - .. method:: full() - - Return ``True`` if the queue is full, ``False`` otherwise. Because of - multithreading/multiprocessing semantics, this is not reliable. - - .. method:: put(obj[, block[, timeout]]) - - Put obj into the queue. If the optional argument *block* is ``True`` - (the default) and *timeout* is ``None`` (the default), block if necessary until - a free slot is available. If *timeout* is a positive number, it blocks at - most *timeout* seconds and raises the :exc:`queue.Full` exception if no - free slot was available within that time. Otherwise (*block* is - ``False``), put an item on the queue if a free slot is immediately - available, else raise the :exc:`queue.Full` exception (*timeout* is - ignored in that case). - - .. method:: put_nowait(obj) - - Equivalent to ``put(obj, False)``. - - .. method:: get([block[, timeout]]) - - Remove and return an item from the queue. If optional args *block* is - ``True`` (the default) and *timeout* is ``None`` (the default), block if - necessary until an item is available. If *timeout* is a positive number, - it blocks at most *timeout* seconds and raises the :exc:`queue.Empty` - exception if no item was available within that time. Otherwise (block is - ``False``), return an item if one is immediately available, else raise the - :exc:`queue.Empty` exception (*timeout* is ignored in that case). - - .. method:: get_nowait() - - Equivalent to ``get(False)``. - - :class:`multiprocessing.Queue` has a few additional methods not found in - :class:`queue.Queue`. These methods are usually unnecessary for most - code: - - .. method:: close() - - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered - data to the pipe. This is called automatically when the queue is garbage - collected. - - .. method:: join_thread() - - Join the background thread. This can only be used after :meth:`close` has - been called. It blocks until the background thread exits, ensuring that - all data in the buffer has been flushed to the pipe. - - By default if a process is not the creator of the queue then on exit it - will attempt to join the queue's background thread. The process can call - :meth:`cancel_join_thread` to make :meth:`join_thread` do nothing. - - .. method:: cancel_join_thread() - - Prevent :meth:`join_thread` from blocking. In particular, this prevents - the background thread from being joined automatically when the process - exits -- see :meth:`join_thread`. - - A better name for this method might be - ``allow_exit_without_flush()``. It is likely to cause enqueued - data to lost, and you almost certainly will not need to use it. - It is really only there if you need the current process to exit - immediately without waiting to flush enqueued data to the - underlying pipe, and you don't care about lost data. - - .. note:: - - This class's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - functionality in this class will be disabled, and attempts to - instantiate a :class:`Queue` will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. The same holds true for any - of the specialized queue types listed below. - -.. class:: SimpleQueue() - - It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: get() - - Remove and return an item from the queue. - - .. method:: put(item) - - Put *item* into the queue. - - -.. class:: JoinableQueue([maxsize]) - - :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which - additionally has :meth:`task_done` and :meth:`join` methods. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. Used by queue - consumers. For each :meth:`~Queue.get` used to fetch a task, a subsequent - call to :meth:`task_done` tells the queue that the processing on the task - is complete. - - If a :meth:`~queue.Queue.join` is currently blocking, it will resume when all - items have been processed (meaning that a :meth:`task_done` call was - received for every item that had been :meth:`~Queue.put` into the queue). - - Raises a :exc:`ValueError` if called more times than there were items - placed in the queue. - - - .. method:: join() - - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`~queue.Queue.join` unblocks. - - -Miscellaneous -~~~~~~~~~~~~~ - -.. function:: active_children() - - Return list of all live children of the current process. - - Calling this has the side effect of "joining" any processes which have - already finished. - -.. function:: cpu_count() - - Return the number of CPUs in the system. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` - - May raise :exc:`NotImplementedError`. - - .. seealso:: - :func:`os.cpu_count` - -.. function:: current_process() - - Return the :class:`Process` object corresponding to the current process. - - An analogue of :func:`threading.current_thread`. - -.. function:: freeze_support() - - Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, - **PyInstaller** and **cx_Freeze**.) - - One needs to call this function straight after the ``if __name__ == - '__main__'`` line of the main module. For example:: - - from multiprocessing import Process, freeze_support - - def f(): - print('hello world!') - - if __name__ == '__main__': - freeze_support() - Process(target=f).start() - - If the ``freeze_support()`` line is omitted then trying to run the frozen - executable will raise :exc:`RuntimeError`. - - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. - -.. function:: get_all_start_methods() - - Returns a list of the supported start methods, the first of which - is the default. The possible start methods are ``'fork'``, - ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is - available. On Unix ``'fork'`` and ``'spawn'`` are always - supported, with ``'fork'`` being the default. - - .. versionadded:: 3.4 - -.. function:: get_context(method=None) - - Return a context object which has the same attributes as the - :mod:`multiprocessing` module. - - If *method* is ``None`` then the default context is returned. - Otherwise *method* should be ``'fork'``, ``'spawn'``, - ``'forkserver'``. :exc:`ValueError` is raised if the specified - start method is not available. - - .. versionadded:: 3.4 - -.. function:: get_start_method(allow_none=False) - - Return the name of start method used for starting processes. - - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. - - The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` - or ``None``. ``'fork'`` is the default on Unix, while ``'spawn'`` is - the default on Windows. - - .. versionadded:: 3.4 - -.. function:: set_executable() - - Sets the path of the Python interpreter to use when starting a child process. - (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: - - set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) - - before they can create child processes. - - .. versionchanged:: 3.4 - Now supported on Unix when the ``'spawn'`` start method is used. - -.. function:: set_start_method(method) - - Set the method which should be used to start child processes. - *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. - - Note that this should be called at most once, and it should be - protected inside the ``if __name__ == '__main__'`` clause of the - main module. - - .. versionadded:: 3.4 - -.. note:: - - :mod:`multiprocessing` contains no analogues of - :func:`threading.active_count`, :func:`threading.enumerate`, - :func:`threading.settrace`, :func:`threading.setprofile`, - :class:`threading.Timer`, or :class:`threading.local`. - - -Connection Objects -~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: multiprocessing.connection - -Connection objects allow the sending and receiving of picklable objects or -strings. They can be thought of as message oriented connected sockets. - -Connection objects are usually created using -:func:`Pipe ` -- see also -:ref:`multiprocessing-listeners-clients`. - -.. class:: Connection - - .. method:: send(obj) - - Send an object to the other end of the connection which should be read - using :meth:`recv`. - - The object must be picklable. Very large pickles (approximately 32 MB+, - though it depends on the OS) may raise a :exc:`ValueError` exception. - - .. method:: recv() - - Return an object sent from the other end of the connection using - :meth:`send`. Blocks until there is something to receive. Raises - :exc:`EOFError` if there is nothing left to receive - and the other end was closed. - - .. method:: fileno() - - Return the file descriptor or handle used by the connection. - - .. method:: close() - - Close the connection. - - This is called automatically when the connection is garbage collected. - - .. method:: poll([timeout]) - - Return whether there is any data available to be read. - - If *timeout* is not specified then it will return immediately. If - *timeout* is a number then this specifies the maximum time in seconds to - block. If *timeout* is ``None`` then an infinite timeout is used. - - Note that multiple connection objects may be polled at once by - using :func:`multiprocessing.connection.wait`. - - .. method:: send_bytes(buffer[, offset[, size]]) - - Send byte data from a :term:`bytes-like object` as a complete message. - - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large - buffers (approximately 32 MB+, though it depends on the OS) may raise a - :exc:`ValueError` exception - - .. method:: recv_bytes([maxlength]) - - Return a complete message of byte data sent from the other end of the - connection as a string. Blocks until there is something to receive. - Raises :exc:`EOFError` if there is nothing left - to receive and the other end has closed. - - If *maxlength* is specified and the message is longer than *maxlength* - then :exc:`OSError` is raised and the connection will no longer be - readable. - - .. versionchanged:: 3.3 - This function used to raise :exc:`IOError`, which is now an - alias of :exc:`OSError`. - - - .. method:: recv_bytes_into(buffer[, offset]) - - Read into *buffer* a complete message of byte data sent from the other end - of the connection and return the number of bytes in the message. Blocks - until there is something to receive. Raises - :exc:`EOFError` if there is nothing left to receive and the other end was - closed. - - *buffer* must be a writable :term:`bytes-like object`. If - *offset* is given then the message will be written into the buffer from - that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). - - If the buffer is too short then a :exc:`BufferTooShort` exception is - raised and the complete message is available as ``e.args[0]`` where ``e`` - is the exception instance. - - .. versionchanged:: 3.3 - Connection objects themselves can now be transferred between processes - using :meth:`Connection.send` and :meth:`Connection.recv`. - - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. - -For example: - -.. doctest:: - - >>> from multiprocessing import Pipe - >>> a, b = Pipe() - >>> a.send([1, 'hello', None]) - >>> b.recv() - [1, 'hello', None] - >>> b.send_bytes(b'thank you') - >>> a.recv_bytes() - b'thank you' - >>> import array - >>> arr1 = array.array('i', range(5)) - >>> arr2 = array.array('i', [0] * 10) - >>> a.send_bytes(arr1) - >>> count = b.recv_bytes_into(arr2) - >>> assert count == len(arr1) * arr1.itemsize - >>> arr2 - array('i', [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]) - - -.. warning:: - - The :meth:`Connection.recv` method automatically unpickles the data it - receives, which can be a security risk unless you can trust the process - which sent the message. - - Therefore, unless the connection object was produced using :func:`Pipe` you - should only use the :meth:`~Connection.recv` and :meth:`~Connection.send` - methods after performing some sort of authentication. See - :ref:`multiprocessing-auth-keys`. - -.. warning:: - - If a process is killed while it is trying to read or write to a pipe then - the data in the pipe is likely to become corrupted, because it may become - impossible to be sure where the message boundaries lie. - - -Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: multiprocessing - -Generally synchronization primitives are not as necessary in a multiprocess -program as they are in a multithreaded program. See the documentation for -:mod:`threading` module. - -Note that one can also create synchronization primitives by using a manager -object -- see :ref:`multiprocessing-managers`. - -.. class:: Barrier(parties[, action[, timeout]]) - - A barrier object: a clone of :class:`threading.Barrier`. - - .. versionadded:: 3.3 - -.. class:: BoundedSemaphore([value]) - - A bounded semaphore object: a close analog of - :class:`threading.BoundedSemaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. - - .. note:: - On Mac OS X, this is indistinguishable from :class:`Semaphore` because - ``sem_getvalue()`` is not implemented on that platform. - -.. class:: Condition([lock]) - - A condition variable: an alias for :class:`threading.Condition`. - - If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. - - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - -.. class:: Event() - - A clone of :class:`threading.Event`. - - -.. class:: Lock() - - A non-recursive lock object: a close analog of :class:`threading.Lock`. - Once a process or thread has acquired a lock, subsequent attempts to - acquire it from any process or thread will block until it is released; - any process or thread may release it. The concepts and behaviors of - :class:`threading.Lock` as it applies to threads are replicated here in - :class:`multiprocessing.Lock` as it applies to either processes or threads, - except as noted. - - Note that :class:`Lock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.Lock`` initialized with a - default context. - - :class:`Lock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - With the *block* argument set to ``True`` (the default), the method call - will block until the lock is in an unlocked state, then set it to locked - and return ``True``. Note that the name of this first argument differs - from that in :meth:`threading.Lock.acquire`. - - With the *block* argument set to ``False``, the method call does not - block. If the lock is currently in a locked state, return ``False``; - otherwise set the lock to a locked state and return ``True``. - - When invoked with a positive, floating-point value for *timeout*, block - for at most the number of seconds specified by *timeout* as long as - the lock can not be acquired. Invocations with a negative value for - *timeout* are equivalent to a *timeout* of zero. Invocations with a - *timeout* value of ``None`` (the default) set the timeout period to - infinite. Note that the treatment of negative or ``None`` values for - *timeout* differs from the implemented behavior in - :meth:`threading.Lock.acquire`. The *timeout* argument has no practical - implications if the *block* argument is set to ``False`` and is thus - ignored. Returns ``True`` if the lock has been acquired or ``False`` if - the timeout period has elapsed. - - - .. method:: release() - - Release a lock. This can be called from any process or thread, not only - the process or thread which originally acquired the lock. - - Behavior is the same as in :meth:`threading.Lock.release` except that - when invoked on an unlocked lock, a :exc:`ValueError` is raised. - - -.. class:: RLock() - - A recursive lock object: a close analog of :class:`threading.RLock`. A - recursive lock must be released by the process or thread that acquired it. - Once a process or thread has acquired a recursive lock, the same process - or thread may acquire it again without blocking; that process or thread - must release it once for each time it has been acquired. - - Note that :class:`RLock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.RLock`` initialized with a - default context. - - :class:`RLock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - When invoked with the *block* argument set to ``True``, block until the - lock is in an unlocked state (not owned by any process or thread) unless - the lock is already owned by the current process or thread. The current - process or thread then takes ownership of the lock (if it does not - already have ownership) and the recursion level inside the lock increments - by one, resulting in a return value of ``True``. Note that there are - several differences in this first argument's behavior compared to the - implementation of :meth:`threading.RLock.acquire`, starting with the name - of the argument itself. - - When invoked with the *block* argument set to ``False``, do not block. - If the lock has already been acquired (and thus is owned) by another - process or thread, the current process or thread does not take ownership - and the recursion level within the lock is not changed, resulting in - a return value of ``False``. If the lock is in an unlocked state, the - current process or thread takes ownership and the recursion level is - incremented, resulting in a return value of ``True``. - - Use and behaviors of the *timeout* argument are the same as in - :meth:`Lock.acquire`. Note that some of these behaviors of *timeout* - differ from the implemented behaviors in :meth:`threading.RLock.acquire`. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the - decrement the recursion level is zero, reset the lock to unlocked (not - owned by any process or thread) and if any other processes or threads - are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling process or - thread. - - Only call this method when the calling process or thread owns the lock. - An :exc:`AssertionError` is raised if this method is called by a process - or thread other than the owner or if the lock is in an unlocked (unowned) - state. Note that the type of exception raised in this situation - differs from the implemented behavior in :meth:`threading.RLock.release`. - - -.. class:: Semaphore([value]) - - A semaphore object: a close analog of :class:`threading.Semaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. - -.. note:: - - On Mac OS X, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with - a timeout will emulate that function's behavior using a sleeping loop. - -.. note:: - - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is - blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, - :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` - or :meth:`Condition.wait` then the call will be immediately interrupted and - :exc:`KeyboardInterrupt` will be raised. - - This differs from the behaviour of :mod:`threading` where SIGINT will be - ignored while the equivalent blocking calls are in progress. - -.. note:: - - Some of this package's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - :mod:`multiprocessing.synchronize` module will be disabled, and attempts to - import it will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. - - -Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is possible to create shared objects using shared memory which can be -inherited by child processes. - -.. function:: Value(typecode_or_type, *args, lock=True) - - Return a :mod:`ctypes` object allocated from shared memory. By default the - return value is actually a synchronized wrapper for the object. The object - itself can be accessed via the *value* attribute of a :class:`Value`. - - *typecode_or_type* determines the type of the returned object: it is either a - ctypes type or a one character typecode of the kind used by the :mod:`array` - module. *\*args* is passed on to the constructor for the type. - - If *lock* is ``True`` (the default) then a new recursive lock - object is created to synchronize access to the value. If *lock* is - a :class:`Lock` or :class:`RLock` object then that will be used to - synchronize access to the value. If *lock* is ``False`` then - access to the returned object will not be automatically protected - by a lock, so it will not necessarily be "process-safe". - - Operations like ``+=`` which involve a read and write are not - atomic. So if, for instance, you want to atomically increment a - shared value it is insufficient to just do :: - - counter.value += 1 - - Assuming the associated lock is recursive (which it is by default) - you can instead do :: - - with counter.get_lock(): - counter.value += 1 - - Note that *lock* is a keyword-only argument. - -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) - - Return a ctypes array allocated from shared memory. By default the return - value is actually a synchronized wrapper for the array. - - *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword only argument. - - Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. - - -The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - -.. module:: multiprocessing.sharedctypes - :synopsis: Allocate ctypes objects from shared memory. - -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating -:mod:`ctypes` objects from shared memory which can be inherited by child -processes. - -.. note:: - - Although it is possible to store a pointer in shared memory remember that - this will refer to a location in the address space of a specific process. - However, the pointer is quite likely to be invalid in the context of a second - process and trying to dereference the pointer from the second process may - cause a crash. - -.. function:: RawArray(typecode_or_type, size_or_initializer) - - Return a ctypes array allocated from shared memory. - - *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer then it - determines the length of the array, and the array will be initially zeroed. - Otherwise *size_or_initializer* is a sequence which is used to initialize the - array and whose length determines the length of the array. - - Note that setting and getting an element is potentially non-atomic -- use - :func:`Array` instead to make sure that access is automatically synchronized - using a lock. - -.. function:: RawValue(typecode_or_type, *args) - - Return a ctypes object allocated from shared memory. - - *typecode_or_type* determines the type of the returned object: it is either a - ctypes type or a one character typecode of the kind used by the :mod:`array` - module. *\*args* is passed on to the constructor for the type. - - Note that setting and getting the value is potentially non-atomic -- use - :func:`Value` instead to make sure that access is automatically synchronized - using a lock. - - Note that an array of :data:`ctypes.c_char` has ``value`` and ``raw`` - attributes which allow one to use it to store and retrieve strings -- see - documentation for :mod:`ctypes`. - -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) - - The same as :func:`RawArray` except that depending on the value of *lock* a - process-safe synchronization wrapper may be returned instead of a raw ctypes - array. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a - :class:`~multiprocessing.Lock` or :class:`~multiprocessing.RLock` object - then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword-only argument. - -.. function:: Value(typecode_or_type, *args, lock=True) - - The same as :func:`RawValue` except that depending on the value of *lock* a - process-safe synchronization wrapper may be returned instead of a raw ctypes - object. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`~multiprocessing.Lock` or - :class:`~multiprocessing.RLock` object then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword-only argument. - -.. function:: copy(obj) - - Return a ctypes object allocated from shared memory which is a copy of the - ctypes object *obj*. - -.. function:: synchronized(obj[, lock]) - - Return a process-safe wrapper object for a ctypes object which uses *lock* to - synchronize access. If *lock* is ``None`` (the default) then a - :class:`multiprocessing.RLock` object is created automatically. - - A synchronized wrapper will have two methods in addition to those of the - object it wraps: :meth:`get_obj` returns the wrapped object and - :meth:`get_lock` returns the lock object used for synchronization. - - Note that accessing the ctypes object through the wrapper can be a lot slower - than accessing the raw ctypes object. - - .. versionchanged:: 3.5 - Synchronized objects support the :term:`context manager` protocol. - - -The table below compares the syntax for creating shared ctypes objects from -shared memory with the normal ctypes syntax. (In the table ``MyStruct`` is some -subclass of :class:`ctypes.Structure`.) - -==================== ========================== =========================== -ctypes sharedctypes using type sharedctypes using typecode -==================== ========================== =========================== -c_double(2.4) RawValue(c_double, 2.4) RawValue('d', 2.4) -MyStruct(4, 6) RawValue(MyStruct, 4, 6) -(c_short * 7)() RawArray(c_short, 7) RawArray('h', 7) -(c_int * 3)(9, 2, 8) RawArray(c_int, (9, 2, 8)) RawArray('i', (9, 2, 8)) -==================== ========================== =========================== - - -Below is an example where a number of ctypes objects are modified by a child -process:: - - from multiprocessing import Process, Lock - from multiprocessing.sharedctypes import Value, Array - from ctypes import Structure, c_double - - class Point(Structure): - _fields_ = [('x', c_double), ('y', c_double)] - - def modify(n, x, s, A): - n.value **= 2 - x.value **= 2 - s.value = s.value.upper() - for a in A: - a.x **= 2 - a.y **= 2 - - if __name__ == '__main__': - lock = Lock() - - n = Value('i', 7) - x = Value(c_double, 1.0/3.0, lock=False) - s = Array('c', b'hello world', lock=lock) - A = Array(Point, [(1.875,-6.25), (-5.75,2.0), (2.375,9.5)], lock=lock) - - p = Process(target=modify, args=(n, x, s, A)) - p.start() - p.join() - - print(n.value) - print(x.value) - print(s.value) - print([(a.x, a.y) for a in A]) - - -.. highlight:: none - -The results printed are :: - - 49 - 0.1111111111111111 - HELLO WORLD - [(3.515625, 39.0625), (33.0625, 4.0), (5.640625, 90.25)] - -.. highlight:: python3 - - -.. _multiprocessing-managers: - -Managers -~~~~~~~~ - -Managers provide a way to create data which can be shared between different -processes, including sharing over a network between processes running on -different machines. A manager object controls a server process which manages -*shared objects*. Other processes can access the shared objects by using -proxies. - -.. function:: multiprocessing.Manager() - - Returns a started :class:`~multiprocessing.managers.SyncManager` object which - can be used for sharing objects between processes. The returned manager - object corresponds to a spawned child process and has methods which will - create shared objects and return corresponding proxies. - -.. module:: multiprocessing.managers - :synopsis: Share data between process with shared objects. - -Manager processes will be shutdown as soon as they are garbage collected or -their parent process exits. The manager classes are defined in the -:mod:`multiprocessing.managers` module: - -.. class:: BaseManager([address[, authkey]]) - - Create a BaseManager object. - - Once created one should call :meth:`start` or ``get_server().serve_forever()`` to ensure - that the manager object refers to a started manager process. - - *address* is the address on which the manager process listens for new - connections. If *address* is ``None`` then an arbitrary one is chosen. - - *authkey* is the authentication key which will be used to check the - validity of incoming connections to the server process. If - *authkey* is ``None`` then ``current_process().authkey`` is used. - Otherwise *authkey* is used and it must be a byte string. - - .. method:: start([initializer[, initargs]]) - - Start a subprocess to start the manager. If *initializer* is not ``None`` - then the subprocess will call ``initializer(*initargs)`` when it starts. - - .. method:: get_server() - - Returns a :class:`Server` object which represents the actual server under - the control of the Manager. The :class:`Server` object supports the - :meth:`serve_forever` method:: - - >>> from multiprocessing.managers import BaseManager - >>> manager = BaseManager(address=('', 50000), authkey=b'abc') - >>> server = manager.get_server() - >>> server.serve_forever() - - :class:`Server` additionally has an :attr:`address` attribute. - - .. method:: connect() - - Connect a local manager object to a remote manager process:: - - >>> from multiprocessing.managers import BaseManager - >>> m = BaseManager(address=('127.0.0.1', 5000), authkey=b'abc') - >>> m.connect() - - .. method:: shutdown() - - Stop the process used by the manager. This is only available if - :meth:`start` has been used to start the server process. - - This can be called multiple times. - - .. method:: register(typeid[, callable[, proxytype[, exposed[, method_to_typeid[, create_method]]]]]) - - A classmethod which can be used for registering a type or callable with - the manager class. - - *typeid* is a "type identifier" which is used to identify a particular - type of shared object. This must be a string. - - *callable* is a callable used for creating objects for this type - identifier. If a manager instance will be connected to the - server using the :meth:`connect` method, or if the - *create_method* argument is ``False`` then this can be left as - ``None``. - - *proxytype* is a subclass of :class:`BaseProxy` which is used to create - proxies for shared objects with this *typeid*. If ``None`` then a proxy - class is created automatically. - - *exposed* is used to specify a sequence of method names which proxies for - this typeid should be allowed to access using - :meth:`BaseProxy._callmethod`. (If *exposed* is ``None`` then - :attr:`proxytype._exposed_` is used instead if it exists.) In the case - where no exposed list is specified, all "public methods" of the shared - object will be accessible. (Here a "public method" means any attribute - which has a :meth:`~object.__call__` method and whose name does not begin - with ``'_'``.) - - *method_to_typeid* is a mapping used to specify the return type of those - exposed methods which should return a proxy. It maps method names to - typeid strings. (If *method_to_typeid* is ``None`` then - :attr:`proxytype._method_to_typeid_` is used instead if it exists.) If a - method's name is not a key of this mapping or if the mapping is ``None`` - then the object returned by the method will be copied by value. - - *create_method* determines whether a method should be created with name - *typeid* which can be used to tell the server process to create a new - shared object and return a proxy for it. By default it is ``True``. - - :class:`BaseManager` instances also have one read-only property: - - .. attribute:: address - - The address used by the manager. - - .. versionchanged:: 3.3 - Manager objects support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` starts the - server process (if it has not already started) and then returns the - manager object. :meth:`~contextmanager.__exit__` calls :meth:`shutdown`. - - In previous versions :meth:`~contextmanager.__enter__` did not start the - manager's server process if it was not already started. - -.. class:: SyncManager - - A subclass of :class:`BaseManager` which can be used for the synchronization - of processes. Objects of this type are returned by - :func:`multiprocessing.Manager`. - - Its methods create and return :ref:`multiprocessing-proxy_objects` for a - number of commonly used data types to be synchronized across processes. - This notably includes shared lists and dictionaries. - - .. method:: Barrier(parties[, action[, timeout]]) - - Create a shared :class:`threading.Barrier` object and return a - proxy for it. - - .. versionadded:: 3.3 - - .. method:: BoundedSemaphore([value]) - - Create a shared :class:`threading.BoundedSemaphore` object and return a - proxy for it. - - .. method:: Condition([lock]) - - Create a shared :class:`threading.Condition` object and return a proxy for - it. - - If *lock* is supplied then it should be a proxy for a - :class:`threading.Lock` or :class:`threading.RLock` object. - - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - - .. method:: Event() - - Create a shared :class:`threading.Event` object and return a proxy for it. - - .. method:: Lock() - - Create a shared :class:`threading.Lock` object and return a proxy for it. - - .. method:: Namespace() - - Create a shared :class:`Namespace` object and return a proxy for it. - - .. method:: Queue([maxsize]) - - Create a shared :class:`queue.Queue` object and return a proxy for it. - - .. method:: RLock() - - Create a shared :class:`threading.RLock` object and return a proxy for it. - - .. method:: Semaphore([value]) - - Create a shared :class:`threading.Semaphore` object and return a proxy for - it. - - .. method:: Array(typecode, sequence) - - Create an array and return a proxy for it. - - .. method:: Value(typecode, value) - - Create an object with a writable ``value`` attribute and return a proxy - for it. - - .. method:: dict() - dict(mapping) - dict(sequence) - - Create a shared :class:`dict` object and return a proxy for it. - - .. method:: list() - list(sequence) - - Create a shared :class:`list` object and return a proxy for it. - - .. versionchanged:: 3.6 - Shared objects are capable of being nested. For example, a shared - container object such as a shared list can contain other shared objects - which will all be managed and synchronized by the :class:`SyncManager`. - -.. class:: Namespace - - A type that can register with :class:`SyncManager`. - - A namespace object has no public methods, but does have writable attributes. - Its representation shows the values of its attributes. - - However, when using a proxy for a namespace object, an attribute beginning - with ``'_'`` will be an attribute of the proxy and not an attribute of the - referent: - - .. doctest:: - - >>> manager = multiprocessing.Manager() - >>> Global = manager.Namespace() - >>> Global.x = 10 - >>> Global.y = 'hello' - >>> Global._z = 12.3 # this is an attribute of the proxy - >>> print(Global) - Namespace(x=10, y='hello') - - -Customized managers ->>>>>>>>>>>>>>>>>>> - -To create one's own manager, one creates a subclass of :class:`BaseManager` and -uses the :meth:`~BaseManager.register` classmethod to register new types or -callables with the manager class. For example:: - - from multiprocessing.managers import BaseManager - - class MathsClass: - def add(self, x, y): - return x + y - def mul(self, x, y): - return x * y - - class MyManager(BaseManager): - pass - - MyManager.register('Maths', MathsClass) - - if __name__ == '__main__': - with MyManager() as manager: - maths = manager.Maths() - print(maths.add(4, 3)) # prints 7 - print(maths.mul(7, 8)) # prints 56 - - -Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> - -It is possible to run a manager server on one machine and have clients use it -from other machines (assuming that the firewalls involved allow it). - -Running the following commands creates a server for a single shared queue which -remote clients can access:: - - >>> from multiprocessing.managers import BaseManager - >>> from queue import Queue - >>> queue = Queue() - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue', callable=lambda:queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') - >>> s = m.get_server() - >>> s.serve_forever() - -One client can access the server as follows:: - - >>> from multiprocessing.managers import BaseManager - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') - >>> m.connect() - >>> queue = m.get_queue() - >>> queue.put('hello') - -Another client can also use it:: - - >>> from multiprocessing.managers import BaseManager - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') - >>> m.connect() - >>> queue = m.get_queue() - >>> queue.get() - 'hello' - -Local processes can also access that queue, using the code from above on the -client to access it remotely:: - - >>> from multiprocessing import Process, Queue - >>> from multiprocessing.managers import BaseManager - >>> class Worker(Process): - ... def __init__(self, q): - ... self.q = q - ... super(Worker, self).__init__() - ... def run(self): - ... self.q.put('local hello') - ... - >>> queue = Queue() - >>> w = Worker(queue) - >>> w.start() - >>> class QueueManager(BaseManager): pass - ... - >>> QueueManager.register('get_queue', callable=lambda: queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') - >>> s = m.get_server() - >>> s.serve_forever() - -.. _multiprocessing-proxy_objects: - -Proxy Objects -~~~~~~~~~~~~~ - -A proxy is an object which *refers* to a shared object which lives (presumably) -in a different process. The shared object is said to be the *referent* of the -proxy. Multiple proxy objects may have the same referent. - -A proxy object has methods which invoke corresponding methods of its referent -(although not every method of the referent will necessarily be available through -the proxy). In this way, a proxy can be used just like its referent can: - -.. doctest:: - - >>> from multiprocessing import Manager - >>> manager = Manager() - >>> l = manager.list([i*i for i in range(10)]) - >>> print(l) - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - >>> print(repr(l)) - - >>> l[4] - 16 - >>> l[2:5] - [4, 9, 16] - -Notice that applying :func:`str` to a proxy will return the representation of -the referent, whereas applying :func:`repr` will return the representation of -the proxy. - -An important feature of proxy objects is that they are picklable so they can be -passed between processes. As such, a referent can contain -:ref:`multiprocessing-proxy_objects`. This permits nesting of these managed -lists, dicts, and other :ref:`multiprocessing-proxy_objects`: - -.. doctest:: - - >>> a = manager.list() - >>> b = manager.list() - >>> a.append(b) # referent of a now contains referent of b - >>> print(a, b) - [] [] - >>> b.append('hello') - >>> print(a[0], b) - ['hello'] ['hello'] - -Similarly, dict and list proxies may be nested inside one another:: - - >>> l_outer = manager.list([ manager.dict() for i in range(2) ]) - >>> d_first_inner = l_outer[0] - >>> d_first_inner['a'] = 1 - >>> d_first_inner['b'] = 2 - >>> l_outer[1]['c'] = 3 - >>> l_outer[1]['z'] = 26 - >>> print(l_outer[0]) - {'a': 1, 'b': 2} - >>> print(l_outer[1]) - {'c': 3, 'z': 26} - -If standard (non-proxy) :class:`list` or :class:`dict` objects are contained -in a referent, modifications to those mutable values will not be propagated -through the manager because the proxy has no way of knowing when the values -contained within are modified. However, storing a value in a container proxy -(which triggers a ``__setitem__`` on the proxy object) does propagate through -the manager and so to effectively modify such an item, one could re-assign the -modified value to the container proxy:: - - # create a list proxy and append a mutable object (a dictionary) - lproxy = manager.list() - lproxy.append({}) - # now mutate the dictionary - d = lproxy[0] - d['a'] = 1 - d['b'] = 2 - # at this point, the changes to d are not yet synced, but by - # updating the dictionary, the proxy is notified of the change - lproxy[0] = d - -This approach is perhaps less convenient than employing nested -:ref:`multiprocessing-proxy_objects` for most use cases but also -demonstrates a level of control over the synchronization. - -.. note:: - - The proxy types in :mod:`multiprocessing` do nothing to support comparisons - by value. So, for instance, we have: - - .. doctest:: - - >>> manager.list([1,2,3]) == [1,2,3] - False - - One should just use a copy of the referent instead when making comparisons. - -.. class:: BaseProxy - - Proxy objects are instances of subclasses of :class:`BaseProxy`. - - .. method:: _callmethod(methodname[, args[, kwds]]) - - Call and return the result of a method of the proxy's referent. - - If ``proxy`` is a proxy whose referent is ``obj`` then the expression :: - - proxy._callmethod(methodname, args, kwds) - - will evaluate the expression :: - - getattr(obj, methodname)(*args, **kwds) - - in the manager's process. - - The returned value will be a copy of the result of the call or a proxy to - a new shared object -- see documentation for the *method_to_typeid* - argument of :meth:`BaseManager.register`. - - If an exception is raised by the call, then is re-raised by - :meth:`_callmethod`. If some other exception is raised in the manager's - process then this is converted into a :exc:`RemoteError` exception and is - raised by :meth:`_callmethod`. - - Note in particular that an exception will be raised if *methodname* has - not been *exposed*. - - An example of the usage of :meth:`_callmethod`: - - .. doctest:: - - >>> l = manager.list(range(10)) - >>> l._callmethod('__len__') - 10 - >>> l._callmethod('__getitem__', (slice(2, 7),)) # equivalent to l[2:7] - [2, 3, 4, 5, 6] - >>> l._callmethod('__getitem__', (20,)) # equivalent to l[20] - Traceback (most recent call last): - ... - IndexError: list index out of range - - .. method:: _getvalue() - - Return a copy of the referent. - - If the referent is unpicklable then this will raise an exception. - - .. method:: __repr__ - - Return a representation of the proxy object. - - .. method:: __str__ - - Return the representation of the referent. - - -Cleanup ->>>>>>> - -A proxy object uses a weakref callback so that when it gets garbage collected it -deregisters itself from the manager which owns its referent. - -A shared object gets deleted from the manager process when there are no longer -any proxies referring to it. - - -Process Pools -~~~~~~~~~~~~~ - -.. module:: multiprocessing.pool - :synopsis: Create pools of processes. - -One can create a pool of processes which will carry out tasks submitted to it -with the :class:`Pool` class. - -.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild [, context]]]]]) - - A process pool object which controls a pool of worker processes to which jobs - can be submitted. It supports asynchronous results with timeouts and - callbacks and has a parallel map implementation. - - *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. - - If *initializer* is not ``None`` then each worker process will call - ``initializer(*initargs)`` when it starts. - - *maxtasksperchild* is the number of tasks a worker process can complete - before it will exit and be replaced with a fresh worker process, to enable - unused resources to be freed. The default *maxtasksperchild* is ``None``, which - means worker processes will live as long as the pool. - - *context* can be used to specify the context used for starting - the worker processes. Usually a pool is created using the - function :func:`multiprocessing.Pool` or the :meth:`Pool` method - of a context object. In both cases *context* is set - appropriately. - - Note that the methods of the pool object should only be called by - the process which created the pool. - - .. versionadded:: 3.2 - *maxtasksperchild* - - .. versionadded:: 3.4 - *context* - - .. note:: - - Worker processes within a :class:`Pool` typically live for the complete - duration of the Pool's work queue. A frequent pattern found in other - systems (such as Apache, mod_wsgi, etc) to free resources held by - workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new - process spawned to replace the old one. The *maxtasksperchild* - argument to the :class:`Pool` exposes this ability to the end user. - - .. method:: apply(func[, args[, kwds]]) - - Call *func* with arguments *args* and keyword arguments *kwds*. It blocks - until the result is ready. Given this blocks, :meth:`apply_async` is - better suited for performing work in parallel. Additionally, *func* - is only executed in one of the workers of the pool. - - .. method:: apply_async(func[, args[, kwds[, callback[, error_callback]]]]) - - A variant of the :meth:`apply` method which returns a result object. - - If *callback* is specified then it should be a callable which accepts a - single argument. When the result becomes ready *callback* is applied to - it, that is unless the call failed, in which case the *error_callback* - is applied instead. - - If *error_callback* is specified then it should be a callable which - accepts a single argument. If the target function fails, then - the *error_callback* is called with the exception instance. - - Callbacks should complete immediately since otherwise the thread which - handles the results will get blocked. - - .. method:: map(func, iterable[, chunksize]) - - A parallel equivalent of the :func:`map` built-in function (it supports only - one *iterable* argument though). It blocks until the result is ready. - - This method chops the iterable into a number of chunks which it submits to - the process pool as separate tasks. The (approximate) size of these - chunks can be specified by setting *chunksize* to a positive integer. - - .. method:: map_async(func, iterable[, chunksize[, callback[, error_callback]]]) - - A variant of the :meth:`.map` method which returns a result object. - - If *callback* is specified then it should be a callable which accepts a - single argument. When the result becomes ready *callback* is applied to - it, that is unless the call failed, in which case the *error_callback* - is applied instead. - - If *error_callback* is specified then it should be a callable which - accepts a single argument. If the target function fails, then - the *error_callback* is called with the exception instance. - - Callbacks should complete immediately since otherwise the thread which - handles the results will get blocked. - - .. method:: imap(func, iterable[, chunksize]) - - A lazier version of :meth:`map`. - - The *chunksize* argument is the same as the one used by the :meth:`.map` - method. For very long iterables using a large value for *chunksize* can - make the job complete **much** faster than using the default value of - ``1``. - - Also if *chunksize* is ``1`` then the :meth:`!next` method of the iterator - returned by the :meth:`imap` method has an optional *timeout* parameter: - ``next(timeout)`` will raise :exc:`multiprocessing.TimeoutError` if the - result cannot be returned within *timeout* seconds. - - .. method:: imap_unordered(func, iterable[, chunksize]) - - The same as :meth:`imap` except that the ordering of the results from the - returned iterator should be considered arbitrary. (Only when there is - only one worker process is the order guaranteed to be "correct".) - - .. method:: starmap(func, iterable[, chunksize]) - - Like :meth:`map` except that the elements of the *iterable* are expected - to be iterables that are unpacked as arguments. - - Hence an *iterable* of ``[(1,2), (3, 4)]`` results in ``[func(1,2), - func(3,4)]``. - - .. versionadded:: 3.3 - - .. method:: starmap_async(func, iterable[, chunksize[, callback[, error_callback]]]) - - A combination of :meth:`starmap` and :meth:`map_async` that iterates over - *iterable* of iterables and calls *func* with the iterables unpacked. - Returns a result object. - - .. versionadded:: 3.3 - - .. method:: close() - - Prevents any more tasks from being submitted to the pool. Once all the - tasks have been completed the worker processes will exit. - - .. method:: terminate() - - Stops the worker processes immediately without completing outstanding - work. When the pool object is garbage collected :meth:`terminate` will be - called immediately. - - .. method:: join() - - Wait for the worker processes to exit. One must call :meth:`close` or - :meth:`terminate` before using :meth:`join`. - - .. versionadded:: 3.3 - Pool objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. - - -.. class:: AsyncResult - - The class of the result returned by :meth:`Pool.apply_async` and - :meth:`Pool.map_async`. - - .. method:: get([timeout]) - - Return the result when it arrives. If *timeout* is not ``None`` and the - result does not arrive within *timeout* seconds then - :exc:`multiprocessing.TimeoutError` is raised. If the remote call raised - an exception then that exception will be reraised by :meth:`get`. - - .. method:: wait([timeout]) - - Wait until the result is available or until *timeout* seconds pass. - - .. method:: ready() - - Return whether the call has completed. - - .. method:: successful() - - Return whether the call completed without raising an exception. Will - raise :exc:`AssertionError` if the result is not ready. - -The following example demonstrates the use of a pool:: - - from multiprocessing import Pool - import time - - def f(x): - return x*x - - if __name__ == '__main__': - with Pool(processes=4) as pool: # start 4 worker processes - result = pool.apply_async(f, (10,)) # evaluate "f(10)" asynchronously in a single process - print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow - - print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" - - it = pool.imap(f, range(10)) - print(next(it)) # prints "0" - print(next(it)) # prints "1" - print(it.next(timeout=1)) # prints "4" unless your computer is *very* slow - - result = pool.apply_async(time.sleep, (10,)) - print(result.get(timeout=1)) # raises multiprocessing.TimeoutError - - -.. _multiprocessing-listeners-clients: - -Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ - -.. module:: multiprocessing.connection - :synopsis: API for dealing with sockets. - -Usually message passing between processes is done using queues or by using -:class:`~Connection` objects returned by -:func:`~multiprocessing.Pipe`. - -However, the :mod:`multiprocessing.connection` module allows some extra -flexibility. It basically gives a high level message oriented API for dealing -with sockets or Windows named pipes. It also has support for *digest -authentication* using the :mod:`hmac` module, and for polling -multiple connections at the same time. - - -.. function:: deliver_challenge(connection, authkey) - - Send a randomly generated message to the other end of the connection and wait - for a reply. - - If the reply matches the digest of the message using *authkey* as the key - then a welcome message is sent to the other end of the connection. Otherwise - :exc:`~multiprocessing.AuthenticationError` is raised. - -.. function:: answer_challenge(connection, authkey) - - Receive a message, calculate the digest of the message using *authkey* as the - key, and then send the digest back. - - If a welcome message is not received, then - :exc:`~multiprocessing.AuthenticationError` is raised. - -.. function:: Client(address[, family[, authkey]]) - - Attempt to set up a connection to the listener which is using address - *address*, returning a :class:`~Connection`. - - The type of the connection is determined by *family* argument, but this can - generally be omitted since it can usually be inferred from the format of - *address*. (See :ref:`multiprocessing-address-formats`) - - If *authkey* is given and not None, it should be a byte string and will be - used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. - :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. - See :ref:`multiprocessing-auth-keys`. - -.. class:: Listener([address[, family[, backlog[, authkey]]]]) - - A wrapper for a bound socket or Windows named pipe which is 'listening' for - connections. - - *address* is the address to be used by the bound socket or named pipe of the - listener object. - - .. note:: - - If an address of '0.0.0.0' is used, the address will not be a connectable - end point on Windows. If you require a connectable end-point, - you should use '127.0.0.1'. - - *family* is the type of socket (or named pipe) to use. This can be one of - the strings ``'AF_INET'`` (for a TCP socket), ``'AF_UNIX'`` (for a Unix - domain socket) or ``'AF_PIPE'`` (for a Windows named pipe). Of these only - the first is guaranteed to be available. If *family* is ``None`` then the - family is inferred from the format of *address*. If *address* is also - ``None`` then a default is chosen. This default is the family which is - assumed to be the fastest available. See - :ref:`multiprocessing-address-formats`. Note that if *family* is - ``'AF_UNIX'`` and address is ``None`` then the socket will be created in a - private temporary directory created using :func:`tempfile.mkstemp`. - - If the listener object uses a socket then *backlog* (1 by default) is passed - to the :meth:`~socket.socket.listen` method of the socket once it has been - bound. - - If *authkey* is given and not None, it should be a byte string and will be - used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. - :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. - See :ref:`multiprocessing-auth-keys`. - - .. method:: accept() - - Accept a connection on the bound socket or named pipe of the listener - object and return a :class:`~Connection` object. - If authentication is attempted and fails, then - :exc:`~multiprocessing.AuthenticationError` is raised. - - .. method:: close() - - Close the bound socket or named pipe of the listener object. This is - called automatically when the listener is garbage collected. However it - is advisable to call it explicitly. - - Listener objects have the following read-only properties: - - .. attribute:: address - - The address which is being used by the Listener object. - - .. attribute:: last_accepted - - The address from which the last accepted connection came. If this is - unavailable then it is ``None``. - - .. versionadded:: 3.3 - Listener objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. - -.. function:: wait(object_list, timeout=None) - - Wait till an object in *object_list* is ready. Returns the list of - those objects in *object_list* which are ready. If *timeout* is a - float then the call blocks for at most that many seconds. If - *timeout* is ``None`` then it will block for an unlimited period. - A negative timeout is equivalent to a zero timeout. - - For both Unix and Windows, an object can appear in *object_list* if - it is - - * a readable :class:`~multiprocessing.connection.Connection` object; - * a connected and readable :class:`socket.socket` object; or - * the :attr:`~multiprocessing.Process.sentinel` attribute of a - :class:`~multiprocessing.Process` object. - - A connection or socket object is ready when there is data available - to be read from it, or the other end has been closed. - - **Unix**: ``wait(object_list, timeout)`` almost equivalent - ``select.select(object_list, [], [], timeout)``. The difference is - that, if :func:`select.select` is interrupted by a signal, it can - raise :exc:`OSError` with an error number of ``EINTR``, whereas - :func:`wait` will not. - - **Windows**: An item in *object_list* must either be an integer - handle which is waitable (according to the definition used by the - documentation of the Win32 function ``WaitForMultipleObjects()``) - or it can be an object with a :meth:`fileno` method which returns a - socket handle or pipe handle. (Note that pipe handles and socket - handles are **not** waitable handles.) - - .. versionadded:: 3.3 - - -**Examples** - -The following server code creates a listener which uses ``'secret password'`` as -an authentication key. It then waits for a connection and sends some data to -the client:: - - from multiprocessing.connection import Listener - from array import array - - address = ('localhost', 6000) # family is deduced to be 'AF_INET' - - with Listener(address, authkey=b'secret password') as listener: - with listener.accept() as conn: - print('connection accepted from', listener.last_accepted) - - conn.send([2.25, None, 'junk', float]) - - conn.send_bytes(b'hello') - - conn.send_bytes(array('i', [42, 1729])) - -The following code connects to the server and receives some data from the -server:: - - from multiprocessing.connection import Client - from array import array - - address = ('localhost', 6000) - - with Client(address, authkey=b'secret password') as conn: - print(conn.recv()) # => [2.25, None, 'junk', float] - - print(conn.recv_bytes()) # => 'hello' - - arr = array('i', [0, 0, 0, 0, 0]) - print(conn.recv_bytes_into(arr)) # => 8 - print(arr) # => array('i', [42, 1729, 0, 0, 0]) - -The following code uses :func:`~multiprocessing.connection.wait` to -wait for messages from multiple processes at once:: - - import time, random - from multiprocessing import Process, Pipe, current_process - from multiprocessing.connection import wait - - def foo(w): - for i in range(10): - w.send((i, current_process().name)) - w.close() - - if __name__ == '__main__': - readers = [] - - for i in range(4): - r, w = Pipe(duplex=False) - readers.append(r) - p = Process(target=foo, args=(w,)) - p.start() - # We close the writable end of the pipe now to be sure that - # p is the only process which owns a handle for it. This - # ensures that when p closes its handle for the writable end, - # wait() will promptly report the readable end as being ready. - w.close() - - while readers: - for r in wait(readers): - try: - msg = r.recv() - except EOFError: - readers.remove(r) - else: - print(msg) - - -.. _multiprocessing-address-formats: - -Address Formats ->>>>>>>>>>>>>>> - -* An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where - *hostname* is a string and *port* is an integer. - -* An ``'AF_UNIX'`` address is a string representing a filename on the - filesystem. - -* An ``'AF_PIPE'`` address is a string of the form - :samp:`r'\\\\.\\pipe\\{PipeName}'`. To use :func:`Client` to connect to a named - pipe on a remote computer called *ServerName* one should use an address of the - form :samp:`r'\\\\{ServerName}\\pipe\\{PipeName}'` instead. - -Note that any string beginning with two backslashes is assumed by default to be -an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. - - -.. _multiprocessing-auth-keys: - -Authentication keys -~~~~~~~~~~~~~~~~~~~ - -When one uses :meth:`Connection.recv `, the -data received is automatically -unpickled. Unfortunately unpickling data from an untrusted source is a security -risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module -to provide digest authentication. - -An authentication key is a byte string which can be thought of as a -password: once a connection is established both ends will demand proof -that the other knows the authentication key. (Demonstrating that both -ends are using the same key does **not** involve sending the key over -the connection.) - -If authentication is requested but no authentication key is specified then the -return value of ``current_process().authkey`` is used (see -:class:`~multiprocessing.Process`). This value will be automatically inherited by -any :class:`~multiprocessing.Process` object that the current process creates. -This means that (by default) all processes of a multi-process program will share -a single authentication key which can be used when setting up connections -between themselves. - -Suitable authentication keys can also be generated by using :func:`os.urandom`. - - -Logging -~~~~~~~ - -Some support for logging is available. Note, however, that the :mod:`logging` -package does not use process shared locks so it is possible (depending on the -handler type) for messages from different processes to get mixed up. - -.. currentmodule:: multiprocessing -.. function:: get_logger() - - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one - will be created. - - When first created the logger has level :data:`logging.NOTSET` and no - default handler. Messages sent to this logger will not by default propagate - to the root logger. - - Note that on Windows child processes will only inherit the level of the - parent process's logger -- any other customization of the logger will not be - inherited. - -.. currentmodule:: multiprocessing -.. function:: log_to_stderr() - - This function performs a call to :func:`get_logger` but in addition to - returning the logger created by get_logger, it adds a handler which sends - output to :data:`sys.stderr` using format - ``'[%(levelname)s/%(processName)s] %(message)s'``. - -Below is an example session with logging turned on:: - - >>> import multiprocessing, logging - >>> logger = multiprocessing.log_to_stderr() - >>> logger.setLevel(logging.INFO) - >>> logger.warning('doomed') - [WARNING/MainProcess] doomed - >>> m = multiprocessing.Manager() - [INFO/SyncManager-...] child process calling self.run() - [INFO/SyncManager-...] created temp directory /.../pymp-... - [INFO/SyncManager-...] manager serving at '/.../listener-...' - >>> del m - [INFO/MainProcess] sending shutdown message to manager - [INFO/SyncManager-...] manager exiting with exitcode 0 - -For a full table of logging levels, see the :mod:`logging` module. - - -The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. module:: multiprocessing.dummy - :synopsis: Dumb wrapper around threading. - -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is -no more than a wrapper around the :mod:`threading` module. - - -.. _multiprocessing-programming: - -Programming guidelines ----------------------- - -There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. - - -All start methods -~~~~~~~~~~~~~~~~~ - -The following applies to all start methods. - -Avoid shared state - - As far as possible one should try to avoid shifting large amounts of data - between processes. - - It is probably best to stick to using queues or pipes for communication - between processes rather than using the lower level synchronization - primitives. - -Picklability - - Ensure that the arguments to the methods of proxies are picklable. - -Thread safety of proxies - - Do not use a proxy object from more than one thread unless you protect it - with a lock. - - (There is never a problem with different processes using the *same* proxy.) - -Joining zombie processes - - On Unix when a process finishes but has not been joined it becomes a zombie. - There should never be very many because each time a new process starts (or - :func:`~multiprocessing.active_children` is called) all completed processes - which have not yet been joined will be joined. Also calling a finished - process's :meth:`Process.is_alive ` will - join the process. Even so it is probably good - practice to explicitly join all the processes that you start. - -Better to inherit than pickle/unpickle - - When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child - processes can use them. However, one should generally avoid - sending shared objects to other processes using pipes or queues. - Instead you should arrange the program so that a process which - needs access to a shared resource created elsewhere can inherit it - from an ancestor process. - -Avoid terminating processes - - Using the :meth:`Process.terminate ` - method to stop a process is liable to - cause any shared resources (such as locks, semaphores, pipes and queues) - currently being used by the process to become broken or unavailable to other - processes. - - Therefore it is probably best to only consider using - :meth:`Process.terminate ` on processes - which never use any shared resources. - -Joining processes that use queues - - Bear in mind that a process that has put items in a queue will wait before - terminating until all the buffered items are fed by the "feeder" thread to - the underlying pipe. (The child process can call the - :meth:`Queue.cancel_join_thread ` - method of the queue to avoid this behaviour.) - - This means that whenever you use a queue you need to make sure that all - items which have been put on the queue will eventually be removed before the - process is joined. Otherwise you cannot be sure that processes which have - put items on the queue will terminate. Remember also that non-daemonic - processes will be joined automatically. - - An example which will deadlock is the following:: - - from multiprocessing import Process, Queue - - def f(q): - q.put('X' * 1000000) - - if __name__ == '__main__': - queue = Queue() - p = Process(target=f, args=(queue,)) - p.start() - p.join() # this deadlocks - obj = queue.get() - - A fix here would be to swap the last two lines (or simply remove the - ``p.join()`` line). - -Explicitly pass resources to child processes - - On Unix using the *fork* start method, a child process can make - use of a shared resource created in a parent process using a - global resource. However, it is better to pass the object as an - argument to the constructor for the child process. - - Apart from making the code (potentially) compatible with Windows - and the other start methods this also ensures that as long as the - child process is still alive the object will not be garbage - collected in the parent process. This might be important if some - resource is freed when the object is garbage collected in the - parent process. - - So for instance :: - - from multiprocessing import Process, Lock - - def f(): - ... do something using "lock" ... - - if __name__ == '__main__': - lock = Lock() - for i in range(10): - Process(target=f).start() - - should be rewritten as :: - - from multiprocessing import Process, Lock - - def f(l): - ... do something using "l" ... - - if __name__ == '__main__': - lock = Lock() - for i in range(10): - Process(target=f, args=(lock,)).start() - -Beware of replacing :data:`sys.stdin` with a "file like object" - - :mod:`multiprocessing` originally unconditionally called:: - - os.close(sys.stdin.fileno()) - - in the :meth:`multiprocessing.Process._bootstrap` method --- this resulted - in issues with processes-in-processes. This has been changed to:: - - sys.stdin.close() - sys.stdin = open(os.open(os.devnull, os.O_RDONLY), closefd=False) - - Which solves the fundamental issue of processes colliding with each other - resulting in a bad file descriptor error, but introduces a potential danger - to applications which replace :func:`sys.stdin` with a "file-like object" - with output buffering. This danger is that if multiple processes call - :meth:`~io.IOBase.close()` on this file-like object, it could result in the same - data being flushed to the object multiple times, resulting in corruption. - - If you write a file-like object and implement your own caching, you can - make it fork-safe by storing the pid whenever you append to the cache, - and discarding the cache when the pid changes. For example:: - - @property - def cache(self): - pid = os.getpid() - if pid != self._pid: - self._pid = pid - self._cache = [] - return self._cache - - For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` - -The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are a few extra restriction which don't apply to the *fork* -start method. - -More picklability - - Ensure that all arguments to :meth:`Process.__init__` are picklable. - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. - -Global variables - - Bear in mind that if code run in a child process tries to access a global - variable, then the value it sees (if any) may not be the same as the value - in the parent process at the time that :meth:`Process.start - ` was called. - - However, global variables which are just module level constants cause no - problems. - -Safe importing of main module - - Make sure that the main module can be safely imported by a new Python - interpreter without causing unintended side effects (such a starting a new - process). - - For example, using the *spawn* or *forkserver* start method - running the following module would fail with a - :exc:`RuntimeError`:: - - from multiprocessing import Process - - def foo(): - print('hello') - - p = Process(target=foo) - p.start() - - Instead one should protect the "entry point" of the program by using ``if - __name__ == '__main__':`` as follows:: - - from multiprocessing import Process, freeze_support, set_start_method - - def foo(): - print('hello') - - if __name__ == '__main__': - freeze_support() - set_start_method('spawn') - p = Process(target=foo) - p.start() - - (The ``freeze_support()`` line can be omitted if the program will be run - normally instead of frozen.) - - This allows the newly spawned Python interpreter to safely import the module - and then run the module's ``foo()`` function. - - Similar restrictions apply if a pool or manager is created in the main - module. - - -.. _multiprocessing-examples: - -Examples --------- - -Demonstration of how to create and use customized managers and proxies: - -.. literalinclude:: ../includes/mp_newtype.py - :language: python3 - - -Using :class:`~multiprocessing.pool.Pool`: - -.. literalinclude:: ../includes/mp_pool.py - :language: python3 - - -An example showing how to use queues to feed tasks to a collection of worker -processes and collect the results: - -.. literalinclude:: ../includes/mp_workers.py diff --git a/third_party/python/Doc/library/netdata.rst b/third_party/python/Doc/library/netdata.rst deleted file mode 100644 index 491501665..000000000 --- a/third_party/python/Doc/library/netdata.rst +++ /dev/null @@ -1,23 +0,0 @@ - -.. _netdata: - -********************** -Internet Data Handling -********************** - -This chapter describes modules which support handling data formats commonly used -on the Internet. - - -.. toctree:: - - email.rst - json.rst - mailcap.rst - mailbox.rst - mimetypes.rst - base64.rst - binhex.rst - binascii.rst - quopri.rst - uu.rst diff --git a/third_party/python/Doc/library/netrc.rst b/third_party/python/Doc/library/netrc.rst deleted file mode 100644 index 64aa3ac7c..000000000 --- a/third_party/python/Doc/library/netrc.rst +++ /dev/null @@ -1,85 +0,0 @@ - -:mod:`netrc` --- netrc file processing -====================================== - -.. module:: netrc - :synopsis: Loading of .netrc files. - -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/netrc.py` - --------------- - -The :class:`~netrc.netrc` class parses and encapsulates the netrc file format used by -the Unix :program:`ftp` program and other FTP clients. - - -.. class:: netrc([file]) - - A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc - file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory will - be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic - information including the file name, line number, and terminating token. - If no argument is specified on a POSIX system, the presence of passwords in - the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file - ownership or permissions are insecure (owned by a user other than the user - running the process, or accessible for read or write by any other user). - This implements security behavior equivalent to that of ftp and other - programs that use :file:`.netrc`. - - .. versionchanged:: 3.4 Added the POSIX permission check. - - -.. exception:: NetrcParseError - - Exception raised by the :class:`~netrc.netrc` class when syntactical errors are - encountered in source text. Instances of this exception provide three - interesting attributes: :attr:`msg` is a textual explanation of the error, - :attr:`filename` is the name of the source file, and :attr:`lineno` gives the - line number on which the error was found. - - -.. _netrc-objects: - -netrc Objects -------------- - -A :class:`~netrc.netrc` instance has the following methods: - - -.. method:: netrc.authenticators(host) - - Return a 3-tuple ``(login, account, password)`` of authenticators for *host*. - If the netrc file did not contain an entry for the given host, return the tuple - associated with the 'default' entry. If neither matching host nor default entry - is available, return ``None``. - - -.. method:: netrc.__repr__() - - Dump the class data as a string in the format of a netrc file. (This discards - comments and may reorder the entries.) - -Instances of :class:`~netrc.netrc` have public instance variables: - - -.. attribute:: netrc.hosts - - Dictionary mapping host names to ``(login, account, password)`` tuples. The - 'default' entry, if any, is represented as a pseudo-host by that name. - - -.. attribute:: netrc.macros - - Dictionary mapping macro names to string lists. - -.. note:: - - Passwords are limited to a subset of the ASCII character set. All ASCII - punctuation is allowed in passwords, however, note that whitespace and - non-printable characters are not allowed in passwords. This is a limitation - of the way the .netrc file is parsed and may be removed in the future. - diff --git a/third_party/python/Doc/library/nis.rst b/third_party/python/Doc/library/nis.rst deleted file mode 100644 index 10c67cbb8..000000000 --- a/third_party/python/Doc/library/nis.rst +++ /dev/null @@ -1,65 +0,0 @@ - -:mod:`nis` --- Interface to Sun's NIS (Yellow Pages) -==================================================== - -.. module:: nis - :platform: Unix - :synopsis: Interface to Sun's NIS (Yellow Pages) library. - -.. moduleauthor:: Fred Gansevles -.. sectionauthor:: Moshe Zadka - --------------- - -The :mod:`nis` module gives a thin wrapper around the NIS library, useful for -central administration of several hosts. - -Because NIS exists only on Unix systems, this module is only available for Unix. - -The :mod:`nis` module defines the following functions: - - -.. function:: match(key, mapname, domain=default_domain) - - Return the match for *key* in map *mapname*, or raise an error - (:exc:`nis.error`) if there is none. Both should be strings, *key* is 8-bit - clean. Return value is an arbitrary array of bytes (may contain ``NULL`` and - other joys). - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: cat(mapname, domain=default_domain) - - Return a dictionary mapping *key* to *value* such that ``match(key, - mapname)==value``. Note that both keys and values of the dictionary are - arbitrary arrays of bytes. - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: maps(domain=default_domain) - - Return a list of all valid maps. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: get_default_domain() - - Return the system default NIS domain. - - -The :mod:`nis` module defines the following exception: - -.. exception:: error - - An error raised when a NIS function returns an error code. - diff --git a/third_party/python/Doc/library/nntplib.rst b/third_party/python/Doc/library/nntplib.rst deleted file mode 100644 index 56188c7ef..000000000 --- a/third_party/python/Doc/library/nntplib.rst +++ /dev/null @@ -1,567 +0,0 @@ -:mod:`nntplib` --- NNTP protocol client -======================================= - -.. module:: nntplib - :synopsis: NNTP protocol client (requires sockets). - -**Source code:** :source:`Lib/nntplib.py` - -.. index:: - pair: NNTP; protocol - single: Network News Transfer Protocol - --------------- - -This module defines the class :class:`NNTP` which implements the client side of -the Network News Transfer Protocol. It can be used to implement a news reader -or poster, or automated news processors. It is compatible with :rfc:`3977` -as well as the older :rfc:`977` and :rfc:`2980`. - -Here are two small examples of how it can be used. To list some statistics -about a newsgroup and print the subjects of the last 10 articles:: - - >>> s = nntplib.NNTP('news.gmane.org') - >>> resp, count, first, last, name = s.group('gmane.comp.python.committers') - >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) - Group gmane.comp.python.committers has 1096 articles, range 1 to 1096 - >>> resp, overviews = s.over((last - 9, last)) - >>> for id, over in overviews: - ... print(id, nntplib.decode_header(over['subject'])) - ... - 1087 Re: Commit privileges for Łukasz Langa - 1088 Re: 3.2 alpha 2 freeze - 1089 Re: 3.2 alpha 2 freeze - 1090 Re: Commit privileges for Łukasz Langa - 1091 Re: Commit privileges for Łukasz Langa - 1092 Updated ssh key - 1093 Re: Updated ssh key - 1094 Re: Updated ssh key - 1095 Hello fellow committers! - 1096 Re: Hello fellow committers! - >>> s.quit() - '205 Bye!' - -To post an article from a binary file (this assumes that the article has valid -headers, and that you have right to post on the particular newsgroup):: - - >>> s = nntplib.NNTP('news.gmane.org') - >>> f = open('article.txt', 'rb') - >>> s.post(f) - '240 Article posted successfully.' - >>> s.quit() - '205 Bye!' - -The module itself defines the following classes: - - -.. class:: NNTP(host, port=119, user=None, password=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP` object, representing a connection - to the NNTP server running on host *host*, listening at port *port*. - An optional *timeout* can be specified for the socket connection. - If the optional *user* and *password* are provided, or if suitable - credentials are present in :file:`/.netrc` and the optional flag *usenetrc* - is true, the ``AUTHINFO USER`` and ``AUTHINFO PASS`` commands are used - to identify and authenticate the user to the server. If the optional - flag *readermode* is true, then a ``mode reader`` command is sent before - authentication is performed. Reader mode is sometimes necessary if you are - connecting to an NNTP server on the local machine and intend to call - reader-specific commands, such as ``group``. If you get unexpected - :exc:`NNTPPermanentError`\ s, you might need to set *readermode*. - The :class:`NNTP` class supports the :keyword:`with` statement to - unconditionally consume :exc:`OSError` exceptions and to close the NNTP - connection when done, e.g.: - - >>> from nntplib import NNTP - >>> with NNTP('news.gmane.org') as n: - ... n.group('gmane.comp.python.committers') - ... # doctest: +SKIP - ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') - >>> - - - .. versionchanged:: 3.2 - *usenetrc* is now ``False`` by default. - - .. versionchanged:: 3.3 - Support for the :keyword:`with` statement was added. - -.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP_SSL` object, representing an encrypted - connection to the NNTP server running on host *host*, listening at - port *port*. :class:`NNTP_SSL` objects have the same methods as - :class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used. - *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object. - Please read :ref:`ssl-security` for best practices. - All other parameters behave the same as for :class:`NNTP`. - - Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of - STARTTLS as described below. However, some servers only support the - former. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - -.. exception:: NNTPError - - Derived from the standard exception :exc:`Exception`, this is the base - class for all exceptions raised by the :mod:`nntplib` module. Instances - of this class have the following attribute: - - .. attribute:: response - - The response of the server if available, as a :class:`str` object. - - -.. exception:: NNTPReplyError - - Exception raised when an unexpected reply is received from the server. - - -.. exception:: NNTPTemporaryError - - Exception raised when a response code in the range 400--499 is received. - - -.. exception:: NNTPPermanentError - - Exception raised when a response code in the range 500--599 is received. - - -.. exception:: NNTPProtocolError - - Exception raised when a reply is received from the server that does not begin - with a digit in the range 1--5. - - -.. exception:: NNTPDataError - - Exception raised when there is some error in the response data. - - -.. _nntp-objects: - -NNTP Objects ------------- - -When connected, :class:`NNTP` and :class:`NNTP_SSL` objects support the -following methods and attributes. - -Attributes -^^^^^^^^^^ - -.. attribute:: NNTP.nntp_version - - An integer representing the version of the NNTP protocol supported by the - server. In practice, this should be ``2`` for servers advertising - :rfc:`3977` compliance and ``1`` for others. - - .. versionadded:: 3.2 - -.. attribute:: NNTP.nntp_implementation - - A string describing the software name and version of the NNTP server, - or :const:`None` if not advertised by the server. - - .. versionadded:: 3.2 - -Methods -^^^^^^^ - -The *response* that is returned as the first item in the return tuple of almost -all methods is the server's response: a string beginning with a three-digit -code. If the server's response indicates an error, the method raises one of -the above exceptions. - -Many of the following methods take an optional keyword-only argument *file*. -When the *file* argument is supplied, it must be either a :term:`file object` -opened for binary writing, or the name of an on-disk file to be written to. -The method will then write any data returned by the server (except for the -response line and the terminating dot) to the file; any list of lines, -tuples or objects that the method normally returns will be empty. - -.. versionchanged:: 3.2 - Many of the following methods have been reworked and fixed, which makes - them incompatible with their 3.1 counterparts. - - -.. method:: NNTP.quit() - - Send a ``QUIT`` command and close the connection. Once this method has been - called, no other methods of the NNTP object should be called. - - -.. method:: NNTP.getwelcome() - - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) - - -.. method:: NNTP.getcapabilities() - - Return the :rfc:`3977` capabilities advertised by the server, as a - :class:`dict` instance mapping capability names to (possibly empty) lists - of values. On legacy servers which don't understand the ``CAPABILITIES`` - command, an empty dictionary is returned instead. - - >>> s = NNTP('news.gmane.org') - >>> 'POST' in s.getcapabilities() - True - - .. versionadded:: 3.2 - - -.. method:: NNTP.login(user=None, password=None, usenetrc=True) - - Send ``AUTHINFO`` commands with the user name and password. If *user* - and *password* are ``None`` and *usenetrc* is true, credentials from - ``~/.netrc`` will be used if possible. - - Unless intentionally delayed, login is normally performed during the - :class:`NNTP` object initialization and separately calling this function - is unnecessary. To force authentication to be delayed, you must not set - *user* or *password* when creating the object, and must set *usenetrc* to - False. - - .. versionadded:: 3.2 - - -.. method:: NNTP.starttls(context=None) - - Send a ``STARTTLS`` command. This will enable encryption on the NNTP - connection. The *context* argument is optional and should be a - :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best - practices. - - Note that this may not be done after authentication information has - been transmitted, and authentication occurs by default if possible during a - :class:`NNTP` object initialization. See :meth:`NNTP.login` for information - on suppressing this behavior. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - -.. method:: NNTP.newgroups(date, *, file=None) - - Send a ``NEWGROUPS`` command. The *date* argument should be a - :class:`datetime.date` or :class:`datetime.datetime` object. - Return a pair ``(response, groups)`` where *groups* is a list representing - the groups that are new since the given *date*. If *file* is supplied, - though, then *groups* will be empty. - - >>> from datetime import date, timedelta - >>> resp, groups = s.newgroups(date.today() - timedelta(days=3)) - >>> len(groups) # doctest: +SKIP - 85 - >>> groups[0] # doctest: +SKIP - GroupInfo(group='gmane.network.tor.devel', last='4', first='1', flag='m') - - -.. method:: NNTP.newnews(group, date, *, file=None) - - Send a ``NEWNEWS`` command. Here, *group* is a group name or ``'*'``, and - *date* has the same meaning as for :meth:`newgroups`. Return a pair - ``(response, articles)`` where *articles* is a list of message ids. - - This command is frequently disabled by NNTP server administrators. - - -.. method:: NNTP.list(group_pattern=None, *, file=None) - - Send a ``LIST`` or ``LIST ACTIVE`` command. Return a pair - ``(response, list)`` where *list* is a list of tuples representing all - the groups available from this NNTP server, optionally matching the - pattern string *group_pattern*. Each tuple has the form - ``(group, last, first, flag)``, where *group* is a group name, *last* - and *first* are the last and first article numbers, and *flag* usually - takes one of these values: - - * ``y``: Local postings and articles from peers are allowed. - * ``m``: The group is moderated and all postings must be approved. - * ``n``: No local postings are allowed, only articles from peers. - * ``j``: Articles from peers are filed in the junk group instead. - * ``x``: No local postings, and articles from peers are ignored. - * ``=foo.bar``: Articles are filed in the ``foo.bar`` group instead. - - If *flag* has another value, then the status of the newsgroup should be - considered unknown. - - This command can return very large results, especially if *group_pattern* - is not specified. It is best to cache the results offline unless you - really need to refresh them. - - .. versionchanged:: 3.2 - *group_pattern* was added. - - -.. method:: NNTP.descriptions(grouppattern) - - Send a ``LIST NEWSGROUPS`` command, where *grouppattern* is a wildmat string as - specified in :rfc:`3977` (it's essentially the same as DOS or UNIX shell wildcard - strings). Return a pair ``(response, descriptions)``, where *descriptions* - is a dictionary mapping group names to textual descriptions. - - >>> resp, descs = s.descriptions('gmane.comp.python.*') - >>> len(descs) # doctest: +SKIP - 295 - >>> descs.popitem() # doctest: +SKIP - ('gmane.comp.python.bio.general', 'BioPython discussion list (Moderated)') - - -.. method:: NNTP.description(group) - - Get a description for a single group *group*. If more than one group matches - (if 'group' is a real wildmat string), return the first match. If no group - matches, return an empty string. - - This elides the response code from the server. If the response code is needed, - use :meth:`descriptions`. - - -.. method:: NNTP.group(name) - - Send a ``GROUP`` command, where *name* is the group name. The group is - selected as the current group, if it exists. Return a tuple - ``(response, count, first, last, name)`` where *count* is the (estimated) - number of articles in the group, *first* is the first article number in - the group, *last* is the last article number in the group, and *name* - is the group name. - - -.. method:: NNTP.over(message_spec, *, file=None) - - Send an ``OVER`` command, or an ``XOVER`` command on legacy servers. - *message_spec* can be either a string representing a message id, or - a ``(first, last)`` tuple of numbers indicating a range of articles in - the current group, or a ``(first, None)`` tuple indicating a range of - articles starting from *first* to the last article in the current group, - or :const:`None` to select the current article in the current group. - - Return a pair ``(response, overviews)``. *overviews* is a list of - ``(article_number, overview)`` tuples, one for each article selected - by *message_spec*. Each *overview* is a dictionary with the same number - of items, but this number depends on the server. These items are either - message headers (the key is then the lower-cased header name) or metadata - items (the key is then the metadata name prepended with ``":"``). The - following items are guaranteed to be present by the NNTP specification: - - * the ``subject``, ``from``, ``date``, ``message-id`` and ``references`` - headers - * the ``:bytes`` metadata: the number of bytes in the entire raw article - (including headers and body) - * the ``:lines`` metadata: the number of lines in the article body - - The value of each item is either a string, or :const:`None` if not present. - - It is advisable to use the :func:`decode_header` function on header - values when they may contain non-ASCII characters:: - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, overviews = s.over((last, last)) - >>> art_num, over = overviews[0] - >>> art_num - 117216 - >>> list(over.keys()) - ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject'] - >>> over['from'] - '=?UTF-8?B?Ik1hcnRpbiB2LiBMw7Z3aXMi?= ' - >>> nntplib.decode_header(over['from']) - '"Martin v. Löwis" ' - - .. versionadded:: 3.2 - - -.. method:: NNTP.help(*, file=None) - - Send a ``HELP`` command. Return a pair ``(response, list)`` where *list* is a - list of help strings. - - -.. method:: NNTP.stat(message_spec=None) - - Send a ``STAT`` command, where *message_spec* is either a message id - (enclosed in ``'<'`` and ``'>'``) or an article number in the current group. - If *message_spec* is omitted or :const:`None`, the current article in the - current group is considered. Return a triple ``(response, number, id)`` - where *number* is the article number and *id* is the message id. - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, number, message_id = s.stat(first) - >>> number, message_id - (9099, '<20030112190404.GE29873@epoch.metaslash.com>') - - -.. method:: NNTP.next() - - Send a ``NEXT`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.last() - - Send a ``LAST`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.article(message_spec=None, *, file=None) - - Send an ``ARTICLE`` command, where *message_spec* has the same meaning as - for :meth:`.stat`. Return a tuple ``(response, info)`` where *info* - is a :class:`~collections.namedtuple` with three attributes *number*, - *message_id* and *lines* (in that order). *number* is the article number - in the group (or 0 if the information is not available), *message_id* the - message id as a string, and *lines* a list of lines (without terminating - newlines) comprising the raw message including headers and body. - - >>> resp, info = s.article('<20030112190404.GE29873@epoch.metaslash.com>') - >>> info.number - 0 - >>> info.message_id - '<20030112190404.GE29873@epoch.metaslash.com>' - >>> len(info.lines) - 65 - >>> info.lines[0] - b'Path: main.gmane.org!not-for-mail' - >>> info.lines[1] - b'From: Neal Norwitz ' - >>> info.lines[-3:] - [b'There is a patch for 2.3 as well as 2.2.', b'', b'Neal'] - - -.. method:: NNTP.head(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``HEAD`` command. The *lines* - returned (or written to *file*) will only contain the message headers, not - the body. - - -.. method:: NNTP.body(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``BODY`` command. The *lines* - returned (or written to *file*) will only contain the message body, not the - headers. - - -.. method:: NNTP.post(data) - - Post an article using the ``POST`` command. The *data* argument is either - a :term:`file object` opened for binary reading, or any iterable of bytes - objects (representing raw lines of the article to be posted). It should - represent a well-formed news article, including the required headers. The - :meth:`post` method automatically escapes lines beginning with ``.`` and - appends the termination line. - - If the method succeeds, the server's response is returned. If the server - refuses posting, a :class:`NNTPReplyError` is raised. - - -.. method:: NNTP.ihave(message_id, data) - - Send an ``IHAVE`` command. *message_id* is the id of the message to send - to the server (enclosed in ``'<'`` and ``'>'``). The *data* parameter - and the return value are the same as for :meth:`post()`. - - -.. method:: NNTP.date() - - Return a pair ``(response, date)``. *date* is a :class:`~datetime.datetime` - object containing the current date and time of the server. - - -.. method:: NNTP.slave() - - Send a ``SLAVE`` command. Return the server's *response*. - - -.. method:: NNTP.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request or response. A value of ``2`` or higher produces the maximum amount - of debugging output, logging each line sent and received on the connection - (including message text). - - -The following are optional NNTP extensions defined in :rfc:`2980`. Some of -them have been superseded by newer commands in :rfc:`3977`. - - -.. method:: NNTP.xhdr(hdr, str, *, file=None) - - Send an ``XHDR`` command. The *hdr* argument is a header keyword, e.g. - ``'subject'``. The *str* argument should have the form ``'first-last'`` - where *first* and *last* are the first and last article numbers to search. - Return a pair ``(response, list)``, where *list* is a list of pairs ``(id, - text)``, where *id* is an article number (as a string) and *text* is the text of - the requested header for that article. If the *file* parameter is supplied, then - the output of the ``XHDR`` command is stored in a file. If *file* is a string, - then the method will open a file with that name, write to it then close it. - If *file* is a :term:`file object`, then it will start calling :meth:`write` on - it to store the lines of the command output. If *file* is supplied, then the - returned *list* is an empty list. - - -.. method:: NNTP.xover(start, end, *, file=None) - - Send an ``XOVER`` command. *start* and *end* are article numbers - delimiting the range of articles to select. The return value is the - same of for :meth:`over()`. It is recommended to use :meth:`over()` - instead, since it will automatically use the newer ``OVER`` command - if available. - - -.. method:: NNTP.xpath(id) - - Return a pair ``(resp, path)``, where *path* is the directory path to the - article with message ID *id*. Most of the time, this extension is not - enabled by NNTP server administrators. - - .. deprecated:: 3.3 - The XPATH extension is not actively used. - - -.. XXX deprecated: - - .. method:: NNTP.xgtitle(name, *, file=None) - - Process an ``XGTITLE`` command, returning a pair ``(response, list)``, where - *list* is a list of tuples containing ``(name, title)``. If the *file* parameter - is supplied, then the output of the ``XGTITLE`` command is stored in a file. - If *file* is a string, then the method will open a file with that name, write - to it then close it. If *file* is a :term:`file object`, then it will start - calling :meth:`write` on it to store the lines of the command output. If *file* - is supplied, then the returned *list* is an empty list. This is an optional NNTP - extension, and may not be supported by all servers. - - :rfc:`2980` says "It is suggested that this extension be deprecated". Use - :meth:`descriptions` or :meth:`description` instead. - - -Utility functions ------------------ - -The module also defines the following utility function: - - -.. function:: decode_header(header_str) - - Decode a header value, un-escaping any escaped non-ASCII characters. - *header_str* must be a :class:`str` object. The unescaped value is - returned. Using this function is recommended to display some headers - in a human readable form:: - - >>> decode_header("Some subject") - 'Some subject' - >>> decode_header("=?ISO-8859-15?Q?D=E9buter_en_Python?=") - 'Débuter en Python' - >>> decode_header("Re: =?UTF-8?B?cHJvYmzDqG1lIGRlIG1hdHJpY2U=?=") - 'Re: problème de matrice' diff --git a/third_party/python/Doc/library/numbers.rst b/third_party/python/Doc/library/numbers.rst deleted file mode 100644 index 1b594952e..000000000 --- a/third_party/python/Doc/library/numbers.rst +++ /dev/null @@ -1,223 +0,0 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ - -.. module:: numbers - :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). - -**Source code:** :source:`Lib/numbers.py` - --------------- - -The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric -:term:`abstract base classes ` which progressively define -more operations. None of the types defined in this module can be instantiated. - - -.. class:: Number - - The root of the numeric hierarchy. If you just want to check if an argument - *x* is a number, without caring what kind, use ``isinstance(x, Number)``. - - -The numeric tower ------------------ - -.. class:: Complex - - Subclasses of this type describe complex numbers and include the operations - that work on the built-in :class:`complex` type. These are: conversions to - :class:`complex` and :class:`bool`, :attr:`.real`, :attr:`.imag`, ``+``, - ``-``, ``*``, ``/``, :func:`abs`, :meth:`conjugate`, ``==``, and ``!=``. All - except ``-`` and ``!=`` are abstract. - - .. attribute:: real - - Abstract. Retrieves the real component of this number. - - .. attribute:: imag - - Abstract. Retrieves the imaginary component of this number. - - .. abstractmethod:: conjugate() - - Abstract. Returns the complex conjugate. For example, ``(1+3j).conjugate() - == (1-3j)``. - -.. class:: Real - - To :class:`Complex`, :class:`Real` adds the operations that work on real - numbers. - - In short, those are: a conversion to :class:`float`, :func:`math.trunc`, - :func:`round`, :func:`math.floor`, :func:`math.ceil`, :func:`divmod`, ``//``, - ``%``, ``<``, ``<=``, ``>``, and ``>=``. - - Real also provides defaults for :func:`complex`, :attr:`~Complex.real`, - :attr:`~Complex.imag`, and :meth:`~Complex.conjugate`. - - -.. class:: Rational - - Subtypes :class:`Real` and adds - :attr:`~Rational.numerator` and :attr:`~Rational.denominator` properties, which - should be in lowest terms. With these, it provides a default for - :func:`float`. - - .. attribute:: numerator - - Abstract. - - .. attribute:: denominator - - Abstract. - - -.. class:: Integral - - Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides - defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`. Adds abstract methods for ``**`` and - bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``. - - -Notes for type implementors ---------------------------- - -Implementors should be careful to make equal numbers equal and hash -them to the same values. This may be subtle if there are two different -extensions of the real numbers. For example, :class:`fractions.Fraction` -implements :func:`hash` as follows:: - - def __hash__(self): - if self.denominator == 1: - # Get integers right. - return hash(self.numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self.numerator, self.denominator)) - - -Adding More Numeric ABCs -~~~~~~~~~~~~~~~~~~~~~~~~ - -There are, of course, more possible ABCs for numbers, and this would -be a poor hierarchy if it precluded the possibility of adding -those. You can add ``MyFoo`` between :class:`Complex` and -:class:`Real` with:: - - class MyFoo(Complex): ... - MyFoo.register(Real) - - -.. _implementing-the-arithmetic-operations: - -Implementing the arithmetic operations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We want to implement the arithmetic operations so that mixed-mode -operations either call an implementation whose author knew about the -types of both arguments, or convert both to the nearest built in type -and do the operation there. For subtypes of :class:`Integral`, this -means that :meth:`__add__` and :meth:`__radd__` should be defined as:: - - class MyIntegral(Integral): - - def __add__(self, other): - if isinstance(other, MyIntegral): - return do_my_adding_stuff(self, other) - elif isinstance(other, OtherTypeIKnowAbout): - return do_my_other_adding_stuff(self, other) - else: - return NotImplemented - - def __radd__(self, other): - if isinstance(other, MyIntegral): - return do_my_adding_stuff(other, self) - elif isinstance(other, OtherTypeIKnowAbout): - return do_my_other_adding_stuff(other, self) - elif isinstance(other, Integral): - return int(other) + int(self) - elif isinstance(other, Real): - return float(other) + float(self) - elif isinstance(other, Complex): - return complex(other) + complex(self) - else: - return NotImplemented - - -There are 5 different cases for a mixed-type operation on subclasses -of :class:`Complex`. I'll refer to all of the above code that doesn't -refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as -"boilerplate". ``a`` will be an instance of ``A``, which is a subtype -of :class:`Complex` (``a : A <: Complex``), and ``b : B <: -Complex``. I'll consider ``a + b``: - - 1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is - well. - 2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the - boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at - all.) - 3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts - ``a``, all is well. - 4. If it falls back to the boilerplate, there are no more possible - methods to try, so this is where the default implementation - should live. - 5. If ``B <: A``, Python tries ``B.__radd__`` before - ``A.__add__``. This is ok, because it was implemented with - knowledge of ``A``, so it can handle those instances before - delegating to :class:`Complex`. - -If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, -then the appropriate shared operation is the one involving the built -in :class:`complex`, and both :meth:`__radd__` s land there, so ``a+b -== b+a``. - -Because most of the operations on any given type will be very similar, -it can be useful to define a helper function which generates the -forward and reverse instances of any given operator. For example, -:class:`fractions.Fraction` uses:: - - def _operator_fallbacks(monomorphic_operator, fallback_operator): - def forward(a, b): - if isinstance(b, (int, Fraction)): - return monomorphic_operator(a, b) - elif isinstance(b, float): - return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) - else: - return NotImplemented - forward.__name__ = '__' + fallback_operator.__name__ + '__' - forward.__doc__ = monomorphic_operator.__doc__ - - def reverse(b, a): - if isinstance(a, Rational): - # Includes ints. - return monomorphic_operator(a, b) - elif isinstance(a, numbers.Real): - return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) - else: - return NotImplemented - reverse.__name__ = '__r' + fallback_operator.__name__ + '__' - reverse.__doc__ = monomorphic_operator.__doc__ - - return forward, reverse - - def _add(a, b): - """a + b""" - return Fraction(a.numerator * b.denominator + - b.numerator * a.denominator, - a.denominator * b.denominator) - - __add__, __radd__ = _operator_fallbacks(_add, operator.add) - - # ... diff --git a/third_party/python/Doc/library/numeric.rst b/third_party/python/Doc/library/numeric.rst deleted file mode 100644 index 7c76a479d..000000000 --- a/third_party/python/Doc/library/numeric.rst +++ /dev/null @@ -1,26 +0,0 @@ - -.. _numeric: - -******************************** -Numeric and Mathematical Modules -******************************** - -The modules described in this chapter provide numeric and math-related functions -and data types. The :mod:`numbers` module defines an abstract hierarchy of -numeric types. The :mod:`math` and :mod:`cmath` modules contain various -mathematical functions for floating-point and complex numbers. The :mod:`decimal` -module supports exact representations of decimal numbers, using arbitrary precision -arithmetic. - -The following modules are documented in this chapter: - - -.. toctree:: - - numbers.rst - math.rst - cmath.rst - decimal.rst - fractions.rst - random.rst - statistics.rst diff --git a/third_party/python/Doc/library/operator.rst b/third_party/python/Doc/library/operator.rst deleted file mode 100644 index 60bf23a54..000000000 --- a/third_party/python/Doc/library/operator.rst +++ /dev/null @@ -1,554 +0,0 @@ -:mod:`operator` --- Standard operators as functions -=================================================== - -.. module:: operator - :synopsis: Functions corresponding to the standard operators. - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/operator.py` - -.. testsetup:: - - import operator - from operator import itemgetter, iadd - --------------- - -The :mod:`operator` module exports a set of efficient functions corresponding to -the intrinsic operators of Python. For example, ``operator.add(x, y)`` is -equivalent to the expression ``x+y``. Many function names are those used for -special methods, without the double underscores. For backward compatibility, -many of these have a variant with the double underscores kept. The variants -without the double underscores are preferred for clarity. - -The functions fall into categories that perform object comparisons, logical -operations, mathematical operations and sequence operations. - -The object comparison functions are useful for all objects, and are named after -the rich comparison operators they support: - - -.. function:: lt(a, b) - le(a, b) - eq(a, b) - ne(a, b) - ge(a, b) - gt(a, b) - __lt__(a, b) - __le__(a, b) - __eq__(a, b) - __ne__(a, b) - __ge__(a, b) - __gt__(a, b) - - Perform "rich comparisons" between *a* and *b*. Specifically, ``lt(a, b)`` is - equivalent to ``a < b``, ``le(a, b)`` is equivalent to ``a <= b``, ``eq(a, - b)`` is equivalent to ``a == b``, ``ne(a, b)`` is equivalent to ``a != b``, - ``gt(a, b)`` is equivalent to ``a > b`` and ``ge(a, b)`` is equivalent to ``a - >= b``. Note that these functions can return any value, which may - or may not be interpretable as a Boolean value. See - :ref:`comparisons` for more information about rich comparisons. - - -The logical operations are also generally applicable to all objects, and support -truth tests, identity tests, and boolean operations: - - -.. function:: not_(obj) - __not__(obj) - - Return the outcome of :keyword:`not` *obj*. (Note that there is no - :meth:`__not__` method for object instances; only the interpreter core defines - this operation. The result is affected by the :meth:`__bool__` and - :meth:`__len__` methods.) - - -.. function:: truth(obj) - - Return :const:`True` if *obj* is true, and :const:`False` otherwise. This is - equivalent to using the :class:`bool` constructor. - - -.. function:: is_(a, b) - - Return ``a is b``. Tests object identity. - - -.. function:: is_not(a, b) - - Return ``a is not b``. Tests object identity. - - -The mathematical and bitwise operations are the most numerous: - - -.. function:: abs(obj) - __abs__(obj) - - Return the absolute value of *obj*. - - -.. function:: add(a, b) - __add__(a, b) - - Return ``a + b``, for *a* and *b* numbers. - - -.. function:: and_(a, b) - __and__(a, b) - - Return the bitwise and of *a* and *b*. - - -.. function:: floordiv(a, b) - __floordiv__(a, b) - - Return ``a // b``. - - -.. function:: index(a) - __index__(a) - - Return *a* converted to an integer. Equivalent to ``a.__index__()``. - - -.. function:: inv(obj) - invert(obj) - __inv__(obj) - __invert__(obj) - - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. - - -.. function:: lshift(a, b) - __lshift__(a, b) - - Return *a* shifted left by *b*. - - -.. function:: mod(a, b) - __mod__(a, b) - - Return ``a % b``. - - -.. function:: mul(a, b) - __mul__(a, b) - - Return ``a * b``, for *a* and *b* numbers. - - -.. function:: matmul(a, b) - __matmul__(a, b) - - Return ``a @ b``. - - .. versionadded:: 3.5 - - -.. function:: neg(obj) - __neg__(obj) - - Return *obj* negated (``-obj``). - - -.. function:: or_(a, b) - __or__(a, b) - - Return the bitwise or of *a* and *b*. - - -.. function:: pos(obj) - __pos__(obj) - - Return *obj* positive (``+obj``). - - -.. function:: pow(a, b) - __pow__(a, b) - - Return ``a ** b``, for *a* and *b* numbers. - - -.. function:: rshift(a, b) - __rshift__(a, b) - - Return *a* shifted right by *b*. - - -.. function:: sub(a, b) - __sub__(a, b) - - Return ``a - b``. - - -.. function:: truediv(a, b) - __truediv__(a, b) - - Return ``a / b`` where 2/3 is .66 rather than 0. This is also known as - "true" division. - - -.. function:: xor(a, b) - __xor__(a, b) - - Return the bitwise exclusive or of *a* and *b*. - - -Operations which work with sequences (some of them with mappings too) include: - -.. function:: concat(a, b) - __concat__(a, b) - - Return ``a + b`` for *a* and *b* sequences. - - -.. function:: contains(a, b) - __contains__(a, b) - - Return the outcome of the test ``b in a``. Note the reversed operands. - - -.. function:: countOf(a, b) - - Return the number of occurrences of *b* in *a*. - - -.. function:: delitem(a, b) - __delitem__(a, b) - - Remove the value of *a* at index *b*. - - -.. function:: getitem(a, b) - __getitem__(a, b) - - Return the value of *a* at index *b*. - - -.. function:: indexOf(a, b) - - Return the index of the first of occurrence of *b* in *a*. - - -.. function:: setitem(a, b, c) - __setitem__(a, b, c) - - Set the value of *a* at index *b* to *c*. - - -.. function:: length_hint(obj, default=0) - - Return an estimated length for the object *o*. First try to return its - actual length, then an estimate using :meth:`object.__length_hint__`, and - finally return the default value. - - .. versionadded:: 3.4 - -The :mod:`operator` module also defines tools for generalized attribute and item -lookups. These are useful for making fast field extractors as arguments for -:func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that -expect a function argument. - - -.. function:: attrgetter(attr) - attrgetter(*attrs) - - Return a callable object that fetches *attr* from its operand. - If more than one attribute is requested, returns a tuple of attributes. - The attribute names can also contain dots. For example: - - * After ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. - - * After ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns - ``(b.name, b.date)``. - - * After ``f = attrgetter('name.first', 'name.last')``, the call ``f(b)`` - returns ``(b.name.first, b.name.last)``. - - Equivalent to:: - - def attrgetter(*items): - if any(not isinstance(item, str) for item in items): - raise TypeError('attribute name must be a string') - if len(items) == 1: - attr = items[0] - def g(obj): - return resolve_attr(obj, attr) - else: - def g(obj): - return tuple(resolve_attr(obj, attr) for attr in items) - return g - - def resolve_attr(obj, attr): - for name in attr.split("."): - obj = getattr(obj, name) - return obj - - -.. function:: itemgetter(item) - itemgetter(*items) - - Return a callable object that fetches *item* from its operand using the - operand's :meth:`__getitem__` method. If multiple items are specified, - returns a tuple of lookup values. For example: - - * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. - - * After ``g = itemgetter(2, 5, 3)``, the call ``g(r)`` returns - ``(r[2], r[5], r[3])``. - - Equivalent to:: - - def itemgetter(*items): - if len(items) == 1: - item = items[0] - def g(obj): - return obj[item] - else: - def g(obj): - return tuple(obj[item] for item in items) - return g - - The items can be any type accepted by the operand's :meth:`__getitem__` - method. Dictionaries accept any hashable value. Lists, tuples, and - strings accept an index or a slice: - - >>> itemgetter(1)('ABCDEFG') - 'B' - >>> itemgetter(1,3,5)('ABCDEFG') - ('B', 'D', 'F') - >>> itemgetter(slice(2,None))('ABCDEFG') - 'CDEFG' - - - Example of using :func:`itemgetter` to retrieve specific fields from a - tuple record: - - >>> inventory = [('apple', 3), ('banana', 2), ('pear', 5), ('orange', 1)] - >>> getcount = itemgetter(1) - >>> list(map(getcount, inventory)) - [3, 2, 5, 1] - >>> sorted(inventory, key=getcount) - [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] - - -.. function:: methodcaller(name[, args...]) - - Return a callable object that calls the method *name* on its operand. If - additional arguments and/or keyword arguments are given, they will be given - to the method as well. For example: - - * After ``f = methodcaller('name')``, the call ``f(b)`` returns ``b.name()``. - - * After ``f = methodcaller('name', 'foo', bar=1)``, the call ``f(b)`` - returns ``b.name('foo', bar=1)``. - - Equivalent to:: - - def methodcaller(name, *args, **kwargs): - def caller(obj): - return getattr(obj, name)(*args, **kwargs) - return caller - - -.. _operator-map: - -Mapping Operators to Functions ------------------------------- - -This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. - -+-----------------------+-------------------------+---------------------------------------+ -| Operation | Syntax | Function | -+=======================+=========================+=======================================+ -| Addition | ``a + b`` | ``add(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Concatenation | ``seq1 + seq2`` | ``concat(seq1, seq2)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Containment Test | ``obj in seq`` | ``contains(seq, obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Division | ``a / b`` | ``truediv(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Division | ``a // b`` | ``floordiv(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Exponentiation | ``a ** b`` | ``pow(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Identity | ``a is b`` | ``is_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Identity | ``a is not b`` | ``is_not(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexed Assignment | ``obj[k] = v`` | ``setitem(obj, k, v)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexed Deletion | ``del obj[k]`` | ``delitem(obj, k)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexing | ``obj[k]`` | ``getitem(obj, k)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Left Shift | ``a << b`` | ``lshift(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Modulo | ``a % b`` | ``mod(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Multiplication | ``a * b`` | ``mul(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Matrix Multiplication | ``a @ b`` | ``matmul(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Negation (Arithmetic) | ``- a`` | ``neg(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Negation (Logical) | ``not a`` | ``not_(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Positive | ``+ a`` | ``pos(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Right Shift | ``a >> b`` | ``rshift(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slice Assignment | ``seq[i:j] = values`` | ``setitem(seq, slice(i, j), values)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slice Deletion | ``del seq[i:j]`` | ``delitem(seq, slice(i, j))`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slicing | ``seq[i:j]`` | ``getitem(seq, slice(i, j))`` | -+-----------------------+-------------------------+---------------------------------------+ -| String Formatting | ``s % obj`` | ``mod(s, obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Subtraction | ``a - b`` | ``sub(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Truth Test | ``obj`` | ``truth(obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a < b`` | ``lt(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a <= b`` | ``le(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Equality | ``a == b`` | ``eq(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Difference | ``a != b`` | ``ne(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a >= b`` | ``ge(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a > b`` | ``gt(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ - -Inplace Operators ------------------ - -Many operations have an "in-place" version. Listed below are functions -providing a more primitive access to in-place operators than the usual syntax -does; for example, the :term:`statement` ``x += y`` is equivalent to -``x = operator.iadd(x, y)``. Another way to put it is to say that -``z = operator.iadd(x, y)`` is equivalent to the compound statement -``z = x; z += y``. - -In those examples, note that when an in-place method is called, the computation -and assignment are performed in two separate steps. The in-place functions -listed below only do the first step, calling the in-place method. The second -step, assignment, is not handled. - -For immutable targets such as strings, numbers, and tuples, the updated -value is computed, but not assigned back to the input variable: - ->>> a = 'hello' ->>> iadd(a, ' world') -'hello world' ->>> a -'hello' - -For mutable targets such as lists and dictionaries, the inplace method -will perform the update, so no subsequent assignment is necessary: - ->>> s = ['h', 'e', 'l', 'l', 'o'] ->>> iadd(s, [' ', 'w', 'o', 'r', 'l', 'd']) -['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'] ->>> s -['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'] - -.. function:: iadd(a, b) - __iadd__(a, b) - - ``a = iadd(a, b)`` is equivalent to ``a += b``. - - -.. function:: iand(a, b) - __iand__(a, b) - - ``a = iand(a, b)`` is equivalent to ``a &= b``. - - -.. function:: iconcat(a, b) - __iconcat__(a, b) - - ``a = iconcat(a, b)`` is equivalent to ``a += b`` for *a* and *b* sequences. - - -.. function:: ifloordiv(a, b) - __ifloordiv__(a, b) - - ``a = ifloordiv(a, b)`` is equivalent to ``a //= b``. - - -.. function:: ilshift(a, b) - __ilshift__(a, b) - - ``a = ilshift(a, b)`` is equivalent to ``a <<= b``. - - -.. function:: imod(a, b) - __imod__(a, b) - - ``a = imod(a, b)`` is equivalent to ``a %= b``. - - -.. function:: imul(a, b) - __imul__(a, b) - - ``a = imul(a, b)`` is equivalent to ``a *= b``. - - -.. function:: imatmul(a, b) - __imatmul__(a, b) - - ``a = imatmul(a, b)`` is equivalent to ``a @= b``. - - .. versionadded:: 3.5 - - -.. function:: ior(a, b) - __ior__(a, b) - - ``a = ior(a, b)`` is equivalent to ``a |= b``. - - -.. function:: ipow(a, b) - __ipow__(a, b) - - ``a = ipow(a, b)`` is equivalent to ``a **= b``. - - -.. function:: irshift(a, b) - __irshift__(a, b) - - ``a = irshift(a, b)`` is equivalent to ``a >>= b``. - - -.. function:: isub(a, b) - __isub__(a, b) - - ``a = isub(a, b)`` is equivalent to ``a -= b``. - - -.. function:: itruediv(a, b) - __itruediv__(a, b) - - ``a = itruediv(a, b)`` is equivalent to ``a /= b``. - - -.. function:: ixor(a, b) - __ixor__(a, b) - - ``a = ixor(a, b)`` is equivalent to ``a ^= b``. diff --git a/third_party/python/Doc/library/optparse.rst b/third_party/python/Doc/library/optparse.rst deleted file mode 100644 index 3afc77bf9..000000000 --- a/third_party/python/Doc/library/optparse.rst +++ /dev/null @@ -1,2037 +0,0 @@ -:mod:`optparse` --- Parser for command line options -=================================================== - -.. module:: optparse - :synopsis: Command-line option parsing library. - :deprecated: - -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - -**Source code:** :source:`Lib/optparse.py` - -.. deprecated:: 3.2 - The :mod:`optparse` module is deprecated and will not be developed further; - development will continue with the :mod:`argparse` module. - --------------- - -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional -GNU/POSIX syntax, and additionally generates usage and help messages for you. - -Here's an example of using :mod:`optparse` in a simple script:: - - from optparse import OptionParser - ... - parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", - help="write report to FILE", metavar="FILE") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print status messages to stdout") - - (options, args) = parser.parse_args() - -With these few lines of code, users of your script can now do the "usual thing" -on the command-line, for example:: - - --file=outfile -q - -As it parses the command line, :mod:`optparse` sets attributes of the -``options`` object returned by :meth:`parse_args` based on user-supplied -command-line values. When :meth:`parse_args` returns from parsing this command -line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short -options to be merged together, and allows options to be associated with their -arguments in a variety of ways. Thus, the following command lines are all -equivalent to the above example:: - - -f outfile --quiet - --quiet --file outfile - -q -foutfile - -qfoutfile - -Additionally, users can run one of :: - - -h - --help - -and :mod:`optparse` will print out a brief summary of your script's options: - -.. code-block:: text - - Usage: [options] - - Options: - -h, --help show this help message and exit - -f FILE, --file=FILE write report to FILE - -q, --quiet don't print status messages to stdout - -where the value of *yourscript* is determined at runtime (normally from -``sys.argv[0]``). - - -.. _optparse-background: - -Background ----------- - -:mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. - - -.. _optparse-terminology: - -Terminology -^^^^^^^^^^^ - -argument - a string entered on the command-line, and passed by the shell to ``execl()`` - or ``execv()``. In Python, arguments are elements of ``sys.argv[1:]`` - (``sys.argv[0]`` is the name of the program being executed). Unix shells - also use the term "word". - - It is occasionally desirable to substitute an argument list other than - ``sys.argv[1:]``, so you should read "argument" as "an element of - ``sys.argv[1:]``, or of some other list provided as a substitute for - ``sys.argv[1:]``". - -option - an argument used to supply extra information to guide or customize the - execution of a program. There are many different syntaxes for options; the - traditional Unix syntax is a hyphen ("-") followed by a single letter, - e.g. ``-x`` or ``-F``. Also, traditional Unix syntax allows multiple - options to be merged into a single argument, e.g. ``-x -F`` is equivalent - to ``-xF``. The GNU project introduced ``--`` followed by a series of - hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. - - Some other option syntaxes that the world has seen include: - - * a hyphen followed by a few letters, e.g. ``-pf`` (this is *not* the same - as multiple options merged into a single argument) - - * a hyphen followed by a whole word, e.g. ``-file`` (this is technically - equivalent to the previous syntax, but they aren't usually seen in the same - program) - - * a plus sign followed by a single letter, or a few letters, or a word, e.g. - ``+f``, ``+rgb`` - - * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, - ``/file`` - - These option syntaxes are not supported by :mod:`optparse`, and they never - will be. This is deliberate: the first three are non-standard on any - environment, and the last only makes sense if you're exclusively targeting - VMS, MS-DOS, and/or Windows. - -option argument - an argument that follows an option, is closely associated with that option, - and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from - their option: - - .. code-block:: text - - -f foo - --file foo - - or included in the same argument: - - .. code-block:: text - - -ffoo - --file=foo - - Typically, a given option either takes an argument or it doesn't. Lots of - people want an "optional option arguments" feature, meaning that some options - will take an argument if they see it, and won't if they don't. This is - somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes - an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not - support this feature. - -positional argument - something leftover in the argument list after options have been parsed, i.e. - after options and their arguments have been parsed and removed from the - argument list. - -required option - an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't - prevent you from implementing required options, but doesn't give you much - help at it either. - -For example, consider this hypothetical command-line:: - - prog -v --report report.txt foo bar - -``-v`` and ``--report`` are both options. Assuming that ``--report`` -takes one argument, ``report.txt`` is an option argument. ``foo`` and -``bar`` are positional arguments. - - -.. _optparse-what-options-for: - -What are options for? -^^^^^^^^^^^^^^^^^^^^^ - -Options are used to provide extra information to tune or customize the execution -of a program. In case it wasn't clear, options are usually *optional*. A -program should be able to run just fine with no options whatsoever. (Pick a -random program from the Unix or GNU toolsets. Can it run without any options at -all and still make sense? The main exceptions are ``find``, ``tar``, and -``dd``\ ---all of which are mutant oddballs that have been rightly criticized -for their non-standard syntax and confusing interfaces.) - -Lots of people want their programs to have "required options". Think about it. -If it's required, then it's *not optional*! If there is a piece of information -that your program absolutely requires in order to run successfully, that's what -positional arguments are for. - -As an example of good command-line interface design, consider the humble ``cp`` -utility, for copying files. It doesn't make much sense to try to copy files -without supplying a destination and at least one source. Hence, ``cp`` fails if -you run it with no arguments. However, it has a flexible, useful syntax that -does not require any options at all:: - - cp SOURCE DEST - cp SOURCE ... DEST-DIR - -You can get pretty far with just that. Most ``cp`` implementations provide a -bunch of options to tweak exactly how the files are copied: you can preserve -mode and modification time, avoid following symlinks, ask before clobbering -existing files, etc. But none of this distracts from the core mission of -``cp``, which is to copy either one file to another, or several files to another -directory. - - -.. _optparse-what-positional-arguments-for: - -What are positional arguments for? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Positional arguments are for those pieces of information that your program -absolutely, positively requires to run. - -A good user interface should have as few absolute requirements as possible. If -your program requires 17 distinct pieces of information in order to run -successfully, it doesn't much matter *how* you get that information from the -user---most people will give up and walk away before they successfully run the -program. This applies whether the user interface is a command-line, a -configuration file, or a GUI: if you make that many demands on your users, most -of them will simply give up. - -In short, try to minimize the amount of information that users are absolutely -required to supply---use sensible defaults whenever possible. Of course, you -also want to make your programs reasonably flexible. That's what options are -for. Again, it doesn't matter if they are entries in a config file, widgets in -the "Preferences" dialog of a GUI, or command-line options---the more options -you implement, the more flexible your program is, and the more complicated its -implementation becomes. Too much flexibility has drawbacks as well, of course; -too many options can overwhelm users and make your code much harder to maintain. - - -.. _optparse-tutorial: - -Tutorial --------- - -While :mod:`optparse` is quite flexible and powerful, it's also straightforward -to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. - -First, you need to import the OptionParser class; then, early in the main -program, create an OptionParser instance:: - - from optparse import OptionParser - ... - parser = OptionParser() - -Then you can start defining options. The basic syntax is:: - - parser.add_option(opt_str, ..., - attr=value, ...) - -Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what -to do when it encounters that option on the command line. - -Typically, each option will have one short option string and one long option -string, e.g.:: - - parser.add_option("-f", "--file", ...) - -You're free to define as many short option strings and as many long option -strings as you like (including zero), as long as there is at least one option -string overall. - -The option strings passed to :meth:`OptionParser.add_option` are effectively -labels for the -option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` -encounters *option strings* and looks up options from them. - -Once all of your options are defined, instruct :mod:`optparse` to parse your -program's command line:: - - (options, args) = parser.parse_args() - -(If you like, you can pass a custom argument list to :meth:`parse_args`, but -that's rarely necessary: by default it uses ``sys.argv[1:]``.) - -:meth:`parse_args` returns two values: - -* ``options``, an object containing values for all of your options---e.g. if - ``--file`` takes a single string argument, then ``options.file`` will be the - filename supplied by the user, or ``None`` if the user did not supply that - option - -* ``args``, the list of positional arguments leftover after parsing options - -This tutorial section only covers the four most important option attributes: -:attr:`~Option.action`, :attr:`~Option.type`, :attr:`~Option.dest` -(destination), and :attr:`~Option.help`. Of these, :attr:`~Option.action` is the -most fundamental. - - -.. _optparse-understanding-option-actions: - -Understanding option actions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; -adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store -a value in some variable---for example, take a string from the command line and -store it in an attribute of ``options``. - -If you don't specify an option action, :mod:`optparse` defaults to ``store``. - - -.. _optparse-store-action: - -The store action -^^^^^^^^^^^^^^^^ - -The most common option action is ``store``, which tells :mod:`optparse` to take -the next argument (or the remainder of the current argument), ensure that it is -of the correct type, and store it to your chosen destination. - -For example:: - - parser.add_option("-f", "--file", - action="store", type="string", dest="filename") - -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: - - args = ["-f", "foo.txt"] - (options, args) = parser.parse_args(args) - -When :mod:`optparse` sees the option string ``-f``, it consumes the next -argument, ``foo.txt``, and stores it in ``options.filename``. So, after this -call to :meth:`parse_args`, ``options.filename`` is ``"foo.txt"``. - -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. -Here's an option that expects an integer argument:: - - parser.add_option("-n", type="int", dest="num") - -Note that this option has no long option string, which is perfectly acceptable. -Also, there's no explicit action, since the default is ``store``. - -Let's parse another fake command-line. This time, we'll jam the option argument -right up against the option: since ``-n42`` (one argument) is equivalent to -``-n 42`` (two arguments), the code :: - - (options, args) = parser.parse_args(["-n42"]) - print(options.num) - -will print ``42``. - -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with -the fact that the default action is ``store``, that means our first example can -be a lot shorter:: - - parser.add_option("-f", "--file", dest="filename") - -If you don't supply a destination, :mod:`optparse` figures out a sensible -default from the option strings: if the first long option string is -``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the -default destination for ``-f`` is ``f``. - -:mod:`optparse` also includes the built-in ``complex`` type. Adding -types is covered in section :ref:`optparse-extending-optparse`. - - -.. _optparse-handling-boolean-options: - -Handling boolean (flag) options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, -``store_true`` and ``store_false``. For example, you might have a ``verbose`` -flag that is turned on with ``-v`` and off with ``-q``:: - - parser.add_option("-v", action="store_true", dest="verbose") - parser.add_option("-q", action="store_false", dest="verbose") - -Here we have two different options with the same destination, which is perfectly -OK. (It just means you have to be a bit careful when setting default -values---see below.) - -When :mod:`optparse` encounters ``-v`` on the command line, it sets -``options.verbose`` to ``True``; when it encounters ``-q``, -``options.verbose`` is set to ``False``. - - -.. _optparse-other-actions: - -Other actions -^^^^^^^^^^^^^ - -Some other actions supported by :mod:`optparse` are: - -``"store_const"`` - store a constant value - -``"append"`` - append this option's argument to a list - -``"count"`` - increment a counter by one - -``"callback"`` - call a specified function - -These are covered in section :ref:`optparse-reference-guide`, Reference Guide -and section :ref:`optparse-option-callbacks`. - - -.. _optparse-default-values: - -Default values -^^^^^^^^^^^^^^ - -All of the above examples involve setting some variable (the "destination") when -certain command-line options are seen. What happens if those options are never -seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you -supply a default value for each destination, which is assigned before the -command line is parsed. - -First, consider the verbose/quiet example. If we want :mod:`optparse` to set -``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: - - parser.add_option("-v", action="store_true", dest="verbose", default=True) - parser.add_option("-q", action="store_false", dest="verbose") - -Since default values apply to the *destination* rather than to any particular -option, and these two options happen to have the same destination, this is -exactly equivalent:: - - parser.add_option("-v", action="store_true", dest="verbose") - parser.add_option("-q", action="store_false", dest="verbose", default=True) - -Consider this:: - - parser.add_option("-v", action="store_true", dest="verbose", default=False) - parser.add_option("-q", action="store_false", dest="verbose", default=True) - -Again, the default value for ``verbose`` will be ``True``: the last default -value supplied for any particular destination is the one that counts. - -A clearer way to specify default values is the :meth:`set_defaults` method of -OptionParser, which you can call at any time before calling :meth:`parse_args`:: - - parser.set_defaults(verbose=True) - parser.add_option(...) - (options, args) = parser.parse_args() - -As before, the last value specified for a given option destination is the one -that counts. For clarity, try to use one method or the other of setting default -values, not both. - - -.. _optparse-generating-help: - -Generating help -^^^^^^^^^^^^^^^ - -:mod:`optparse`'s ability to generate help and usage text automatically is -useful for creating user-friendly command-line interfaces. All you have to do -is supply a :attr:`~Option.help` value for each option, and optionally a short -usage message for your whole program. Here's an OptionParser populated with -user-friendly (documented) options:: - - usage = "usage: %prog [options] arg1 arg2" - parser = OptionParser(usage=usage) - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=True, - help="make lots of noise [default]") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", - help="be vewwy quiet (I'm hunting wabbits)") - parser.add_option("-f", "--filename", - metavar="FILE", help="write output to FILE") - parser.add_option("-m", "--mode", - default="intermediate", - help="interaction mode: novice, intermediate, " - "or expert [default: %default]") - -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the -command-line, or if you just call :meth:`parser.print_help`, it prints the -following to standard output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or - expert [default: intermediate] - -(If the help output is triggered by a help option, :mod:`optparse` exits after -printing the help text.) - -There's a lot going on here to help :mod:`optparse` generate the best possible -help message: - -* the script defines its own usage message:: - - usage = "usage: %prog [options] arg1 arg2" - - :mod:`optparse` expands ``%prog`` in the usage string to the name of the - current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string - is then printed before the detailed option help. - - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible - default: ``"Usage: %prog [options]"``, which is fine if your script doesn't - take any positional arguments. - -* every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making - the help output look good. - -* options that take a value indicate this fact in their automatically-generated - help message, e.g. for the "mode" option:: - - -m MODE, --mode=MODE - - Here, "MODE" is called the meta-variable: it stands for the argument that the - user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses - that for the meta-variable. Sometimes, that's not what you want---for - example, the ``--filename`` option explicitly sets ``metavar="FILE"``, - resulting in this automatically-generated option description:: - - -f FILE, --filename=FILE - - This is important for more than just saving space, though: the manually - written help text uses the meta-variable ``FILE`` to clue the user in that - there's a connection between the semi-formal syntax ``-f FILE`` and the informal - semantic description "write output to FILE". This is a simple but effective - way to make your help text a lot clearer and more useful for end users. - -* options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's - default value. If an option has no default value (or the default value is - ``None``), ``%default`` expands to ``none``. - -Grouping Options -++++++++++++++++ - -When dealing with many options, it is convenient to group these options for -better help output. An :class:`OptionParser` can contain several option groups, -each of which can contain several options. - -An option group is obtained using the class :class:`OptionGroup`: - -.. class:: OptionGroup(parser, title, description=None) - - where - - * parser is the :class:`OptionParser` instance the group will be inserted in - to - * title is the group title - * description, optional, is a long description of the group - -:class:`OptionGroup` inherits from :class:`OptionContainer` (like -:class:`OptionParser`) and so the :meth:`add_option` method can be used to add -an option to the group. - -Once all the options are declared, using the :class:`OptionParser` method -:meth:`add_option_group` the group is added to the previously defined parser. - -Continuing with the parser defined in the previous section, adding an -:class:`OptionGroup` to a parser is easy:: - - group = OptionGroup(parser, "Dangerous Options", - "Caution: use these options at your own risk. " - "It is believed that some of them bite.") - group.add_option("-g", action="store_true", help="Group option.") - parser.add_option_group(group) - -This would result in the following help output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or - expert [default: intermediate] - - Dangerous Options: - Caution: use these options at your own risk. It is believed that some - of them bite. - - -g Group option. - -A bit more complete example might involve using more than one group: still -extending the previous example:: - - group = OptionGroup(parser, "Dangerous Options", - "Caution: use these options at your own risk. " - "It is believed that some of them bite.") - group.add_option("-g", action="store_true", help="Group option.") - parser.add_option_group(group) - - group = OptionGroup(parser, "Debug Options") - group.add_option("-d", "--debug", action="store_true", - help="Print debug information") - group.add_option("-s", "--sql", action="store_true", - help="Print all SQL statements executed") - group.add_option("-e", action="store_true", help="Print every action done") - parser.add_option_group(group) - -that results in the following output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert - [default: intermediate] - - Dangerous Options: - Caution: use these options at your own risk. It is believed that some - of them bite. - - -g Group option. - - Debug Options: - -d, --debug Print debug information - -s, --sql Print all SQL statements executed - -e Print every action done - -Another interesting method, in particular when working programmatically with -option groups is: - -.. method:: OptionParser.get_option_group(opt_str) - - Return the :class:`OptionGroup` to which the short or long option - string *opt_str* (e.g. ``'-o'`` or ``'--option'``) belongs. If - there's no such :class:`OptionGroup`, return ``None``. - -.. _optparse-printing-version-string: - -Printing a version string -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Similar to the brief usage string, :mod:`optparse` can also print a version -string for your program. You have to supply the string as the ``version`` -argument to OptionParser:: - - parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") - -``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` -automatically adds a ``--version`` option to your parser. If it encounters -this option on the command line, it expands your ``version`` string (by -replacing ``%prog``), prints it to stdout, and exits. - -For example, if your script is called ``/usr/bin/foo``: - -.. code-block:: shell-session - - $ /usr/bin/foo --version - foo 1.0 - -The following two methods can be used to print and get the ``version`` string: - -.. method:: OptionParser.print_version(file=None) - - Print the version message for the current program (``self.version``) to - *file* (default stdout). As with :meth:`print_usage`, any occurrence - of ``%prog`` in ``self.version`` is replaced with the name of the current - program. Does nothing if ``self.version`` is empty or undefined. - -.. method:: OptionParser.get_version() - - Same as :meth:`print_version` but returns the version string instead of - printing it. - - -.. _optparse-how-optparse-handles-errors: - -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are two broad classes of errors that :mod:`optparse` has to worry about: -programmer errors and user errors. Programmer errors are usually erroneous -calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown -option attributes, missing option attributes, etc. These are dealt with in the -usual way: raise an exception (either :exc:`optparse.OptionError` or -:exc:`TypeError`) and let the program crash. - -Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect -some user errors, such as bad option arguments (passing ``-n 4x`` where -``-n`` takes an integer argument), missing arguments (``-n`` at the end -of the command line, where ``-n`` takes an argument of any type). Also, -you can call :func:`OptionParser.error` to signal an application-defined error -condition:: - - (options, args) = parser.parse_args() - ... - if options.a and options.b: - parser.error("options -a and -b are mutually exclusive") - -In either case, :mod:`optparse` handles the error the same way: it prints the -program's usage message and an error message to standard error and exits with -error status 2. - -Consider the first example above, where the user passes ``4x`` to an option -that takes an integer: - -.. code-block:: shell-session - - $ /usr/bin/foo -n 4x - Usage: foo [options] - - foo: error: option -n: invalid integer value: '4x' - -Or, where the user fails to pass a value at all: - -.. code-block:: shell-session - - $ /usr/bin/foo -n - Usage: foo [options] - - foo: error: -n option requires an argument - -:mod:`optparse`\ -generated error messages take care always to mention the -option involved in the error; be sure to do the same when calling -:func:`OptionParser.error` from your application code. - -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, -you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` -and/or :meth:`~OptionParser.error` methods. - - -.. _optparse-putting-it-all-together: - -Putting it all together -^^^^^^^^^^^^^^^^^^^^^^^ - -Here's what :mod:`optparse`\ -based scripts usually look like:: - - from optparse import OptionParser - ... - def main(): - usage = "usage: %prog [options] arg" - parser = OptionParser(usage) - parser.add_option("-f", "--file", dest="filename", - help="read data from FILENAME") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose") - ... - (options, args) = parser.parse_args() - if len(args) != 1: - parser.error("incorrect number of arguments") - if options.verbose: - print("reading %s..." % options.filename) - ... - - if __name__ == "__main__": - main() - - -.. _optparse-reference-guide: - -Reference Guide ---------------- - - -.. _optparse-creating-parser: - -Creating the parser -^^^^^^^^^^^^^^^^^^^ - -The first step in using :mod:`optparse` is to create an OptionParser instance. - -.. class:: OptionParser(...) - - The OptionParser constructor has no required arguments, but a number of - optional keyword arguments. You should always pass them as keyword - arguments, i.e. do not rely on the order in which the arguments are declared. - - ``usage`` (default: ``"%prog [options]"``) - The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands - ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you - passed that keyword argument). To suppress a usage message, pass the - special value :data:`optparse.SUPPRESS_USAGE`. - - ``option_list`` (default: ``[]``) - A list of Option objects to populate the parser with. The options in - ``option_list`` are added after any options in ``standard_option_list`` (a - class attribute that may be set by OptionParser subclasses), but before - any version or help options. Deprecated; use :meth:`add_option` after - creating the parser instead. - - ``option_class`` (default: optparse.Option) - Class to use when adding options to the parser in :meth:`add_option`. - - ``version`` (default: ``None``) - A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a - version option with the single option string ``--version``. The - substring ``%prog`` is expanded the same as for ``usage``. - - ``conflict_handler`` (default: ``"error"``) - Specifies what to do when options with conflicting option strings are - added to the parser; see section - :ref:`optparse-conflicts-between-options`. - - ``description`` (default: ``None``) - A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width - and prints it when the user requests help (after ``usage``, but before the - list of options). - - ``formatter`` (default: a new :class:`IndentedHelpFormatter`) - An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: - IndentedHelpFormatter and TitledHelpFormatter. - - ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` - and ``--help``) to the parser. - - ``prog`` - The string to use when expanding ``%prog`` in ``usage`` and ``version`` - instead of ``os.path.basename(sys.argv[0])``. - - ``epilog`` (default: ``None``) - A paragraph of help text to print after the option help. - -.. _optparse-populating-parser: - -Populating the parser -^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to populate the parser with options. The preferred way -is by using :meth:`OptionParser.add_option`, as shown in section -:ref:`optparse-tutorial`. :meth:`add_option` can be called in one of two ways: - -* pass it an Option instance (as returned by :func:`make_option`) - -* pass it any combination of positional and keyword arguments that are - acceptable to :func:`make_option` (i.e., to the Option constructor), and it - will create the Option instance for you - -The other alternative is to pass a list of pre-constructed Option instances to -the OptionParser constructor, as in:: - - option_list = [ - make_option("-f", "--filename", - action="store", type="string", dest="filename"), - make_option("-q", "--quiet", - action="store_false", dest="verbose"), - ] - parser = OptionParser(option_list=option_list) - -(:func:`make_option` is a factory function for creating Option instances; -currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` -will pick the right class to instantiate. Do not instantiate Option directly.) - - -.. _optparse-defining-options: - -Defining options -^^^^^^^^^^^^^^^^ - -Each Option instance represents a set of synonymous command-line option strings, -e.g. ``-f`` and ``--file``. You can specify any number of short or -long option strings, but you must specify at least one overall option string. - -The canonical way to create an :class:`Option` instance is with the -:meth:`add_option` method of :class:`OptionParser`. - -.. method:: OptionParser.add_option(option) - OptionParser.add_option(*opt_str, attr=value, ...) - - To define an option with only a short option string:: - - parser.add_option("-f", attr=value, ...) - - And to define an option with only a long option string:: - - parser.add_option("--foo", attr=value, ...) - - The keyword arguments define attributes of the new Option object. The most - important option attribute is :attr:`~Option.action`, and it largely - determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` - raises an :exc:`OptionError` exception explaining your mistake. - - An option's *action* determines what :mod:`optparse` does when it encounters - this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: - - ``"store"`` - store this option's argument (default) - - ``"store_const"`` - store a constant value - - ``"store_true"`` - store a true value - - ``"store_false"`` - store a false value - - ``"append"`` - append this option's argument to a list - - ``"append_const"`` - append a constant value to a list - - ``"count"`` - increment a counter by one - - ``"callback"`` - call a specified function - - ``"help"`` - print a usage message including all options and the documentation for them - - (If you don't supply an action, the default is ``"store"``. For this action, - you may also supply :attr:`~Option.type` and :attr:`~Option.dest` option - attributes; see :ref:`optparse-standard-option-actions`.) - -As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called -``options`` (it happens to be an instance of :class:`optparse.Values`). Option -arguments (and various other values) are stored as attributes of this object, -according to the :attr:`~Option.dest` (destination) option attribute. - -For example, when you call :: - - parser.parse_args() - -one of the first things :mod:`optparse` does is create the ``options`` object:: - - options = Values() - -If one of the options in this parser is defined with :: - - parser.add_option("-f", "--file", action="store", type="string", dest="filename") - -and the command-line being parsed includes any of the following:: - - -ffoo - -f foo - --file=foo - --file foo - -then :mod:`optparse`, on seeing this option, will do the equivalent of :: - - options.filename = "foo" - -The :attr:`~Option.type` and :attr:`~Option.dest` option attributes are almost -as important as :attr:`~Option.action`, but :attr:`~Option.action` is the only -one that makes sense for *all* options. - - -.. _optparse-option-attributes: - -Option attributes -^^^^^^^^^^^^^^^^^ - -The following option attributes may be passed as keyword arguments to -:meth:`OptionParser.add_option`. If you pass an option attribute that is not -relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. - -.. attribute:: Option.action - - (default: ``"store"``) - - Determines :mod:`optparse`'s behaviour when this option is seen on the - command line; the available options are documented :ref:`here - `. - -.. attribute:: Option.type - - (default: ``"string"``) - - The argument type expected by this option (e.g., ``"string"`` or ``"int"``); - the available option types are documented :ref:`here - `. - -.. attribute:: Option.dest - - (default: derived from option strings) - - If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses - the command line. - -.. attribute:: Option.default - - The value to use for this option's destination if the option is not seen on - the command line. See also :meth:`OptionParser.set_defaults`. - -.. attribute:: Option.nargs - - (default: 1) - - How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to - :attr:`~Option.dest`. - -.. attribute:: Option.const - - For actions that store a constant value, the constant value to store. - -.. attribute:: Option.choices - - For options of type ``"choice"``, the list of strings the user may choose - from. - -.. attribute:: Option.callback - - For options with action ``"callback"``, the callable to call when this option - is seen. See section :ref:`optparse-option-callbacks` for detail on the - arguments passed to the callable. - -.. attribute:: Option.callback_args - Option.callback_kwargs - - Additional positional and keyword arguments to pass to ``callback`` after the - four standard callback arguments. - -.. attribute:: Option.help - - Help text to print for this option when listing all available options after - the user supplies a :attr:`~Option.help` option (such as ``--help``). If - no help text is supplied, the option will be listed without help text. To - hide this option, use the special value :data:`optparse.SUPPRESS_HELP`. - -.. attribute:: Option.metavar - - (default: derived from option strings) - - Stand-in for the option argument(s) to use when printing help text. See - section :ref:`optparse-tutorial` for an example. - - -.. _optparse-standard-option-actions: - -Standard option actions -^^^^^^^^^^^^^^^^^^^^^^^ - -The various option actions all have slightly different requirements and effects. -Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you -must specify for any option using that action. - -* ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, - :attr:`~Option.nargs`, :attr:`~Option.choices`] - - The option must be followed by an argument, which is converted to a value - according to :attr:`~Option.type` and stored in :attr:`~Option.dest`. If - :attr:`~Option.nargs` > 1, multiple arguments will be consumed from the - command line; all will be converted according to :attr:`~Option.type` and - stored to :attr:`~Option.dest` as a tuple. See the - :ref:`optparse-standard-option-types` section. - - If :attr:`~Option.choices` is supplied (a list or tuple of strings), the type - defaults to ``"choice"``. - - If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination - from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a - destination from the first short option string (e.g., ``-f`` implies ``f``). - - Example:: - - parser.add_option("-f") - parser.add_option("-p", type="float", nargs=3, dest="point") - - As it parses the command line :: - - -f foo.txt -p 1 -3.5 4 -fbar.txt - - :mod:`optparse` will set :: - - options.f = "foo.txt" - options.point = (1.0, -3.5, 4.0) - options.f = "bar.txt" - -* ``"store_const"`` [required: :attr:`~Option.const`; relevant: - :attr:`~Option.dest`] - - The value :attr:`~Option.const` is stored in :attr:`~Option.dest`. - - Example:: - - parser.add_option("-q", "--quiet", - action="store_const", const=0, dest="verbose") - parser.add_option("-v", "--verbose", - action="store_const", const=1, dest="verbose") - parser.add_option("--noisy", - action="store_const", const=2, dest="verbose") - - If ``--noisy`` is seen, :mod:`optparse` will set :: - - options.verbose = 2 - -* ``"store_true"`` [relevant: :attr:`~Option.dest`] - - A special case of ``"store_const"`` that stores a true value to - :attr:`~Option.dest`. - -* ``"store_false"`` [relevant: :attr:`~Option.dest`] - - Like ``"store_true"``, but stores a false value. - - Example:: - - parser.add_option("--clobber", action="store_true", dest="clobber") - parser.add_option("--no-clobber", action="store_false", dest="clobber") - -* ``"append"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, - :attr:`~Option.nargs`, :attr:`~Option.choices`] - - The option must be followed by an argument, which is appended to the list in - :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first - encounters this option on the command-line. If :attr:`~Option.nargs` > 1, - multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` - is appended to :attr:`~Option.dest`. - - The defaults for :attr:`~Option.type` and :attr:`~Option.dest` are the same as - for the ``"store"`` action. - - Example:: - - parser.add_option("-t", "--tracks", action="append", type="int") - - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent - of:: - - options.tracks = [] - options.tracks.append(int("3")) - - If, a little later on, ``--tracks=4`` is seen, it does:: - - options.tracks.append(int("4")) - - The ``append`` action calls the ``append`` method on the current value of the - option. This means that any default value specified must have an ``append`` - method. It also means that if the default value is non-empty, the default - elements will be present in the parsed value for the option, with any values - from the command line appended after those default values:: - - >>> parser.add_option("--files", action="append", default=['~/.mypkg/defaults']) - >>> opts, args = parser.parse_args(['--files', 'overrides.mypkg']) - >>> opts.files - ['~/.mypkg/defaults', 'overrides.mypkg'] - -* ``"append_const"`` [required: :attr:`~Option.const`; relevant: - :attr:`~Option.dest`] - - Like ``"store_const"``, but the value :attr:`~Option.const` is appended to - :attr:`~Option.dest`; as with ``"append"``, :attr:`~Option.dest` defaults to - ``None``, and an empty list is automatically created the first time the option - is encountered. - -* ``"count"`` [relevant: :attr:`~Option.dest`] - - Increment the integer stored at :attr:`~Option.dest`. If no default value is - supplied, :attr:`~Option.dest` is set to zero before being incremented the - first time. - - Example:: - - parser.add_option("-v", action="count", dest="verbosity") - - The first time ``-v`` is seen on the command line, :mod:`optparse` does the - equivalent of:: - - options.verbosity = 0 - options.verbosity += 1 - - Every subsequent occurrence of ``-v`` results in :: - - options.verbosity += 1 - -* ``"callback"`` [required: :attr:`~Option.callback`; relevant: - :attr:`~Option.type`, :attr:`~Option.nargs`, :attr:`~Option.callback_args`, - :attr:`~Option.callback_kwargs`] - - Call the function specified by :attr:`~Option.callback`, which is called as :: - - func(option, opt_str, value, parser, *args, **kwargs) - - See section :ref:`optparse-option-callbacks` for more detail. - -* ``"help"`` - - Prints a complete help message for all the options in the current option - parser. The help message is constructed from the ``usage`` string passed to - OptionParser's constructor and the :attr:`~Option.help` string passed to every - option. - - If no :attr:`~Option.help` string is supplied for an option, it will still be - listed in the help message. To omit an option entirely, use the special value - :data:`optparse.SUPPRESS_HELP`. - - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all - OptionParsers, so you do not normally need to create one. - - Example:: - - from optparse import OptionParser, SUPPRESS_HELP - - # usually, a help option is added automatically, but that can - # be suppressed using the add_help_option argument - parser = OptionParser(add_help_option=False) - - parser.add_option("-h", "--help", action="help") - parser.add_option("-v", action="store_true", dest="verbose", - help="Be moderately verbose") - parser.add_option("--file", dest="filename", - help="Input file to read data from") - parser.add_option("--secret", help=SUPPRESS_HELP) - - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, - it will print something like the following help message to stdout (assuming - ``sys.argv[0]`` is ``"foo.py"``): - - .. code-block:: text - - Usage: foo.py [options] - - Options: - -h, --help Show this help message and exit - -v Be moderately verbose - --file=FILENAME Input file to read data from - - After printing the help message, :mod:`optparse` terminates your process with - ``sys.exit(0)``. - -* ``"version"`` - - Prints the version number supplied to the OptionParser to stdout and exits. - The version number is actually formatted and printed by the - ``print_version()`` method of OptionParser. Generally only relevant if the - ``version`` argument is supplied to the OptionParser constructor. As with - :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. - - -.. _optparse-standard-option-types: - -Standard option types -^^^^^^^^^^^^^^^^^^^^^ - -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, -``"choice"``, ``"float"`` and ``"complex"``. If you need to add new -option types, see section :ref:`optparse-extending-optparse`. - -Arguments to string options are not checked or converted in any way: the text on -the command line is stored in the destination (or passed to the callback) as-is. - -Integer arguments (type ``"int"``) are parsed as follows: - -* if the number starts with ``0x``, it is parsed as a hexadecimal number - -* if the number starts with ``0``, it is parsed as an octal number - -* if the number starts with ``0b``, it is parsed as a binary number - -* otherwise, the number is parsed as a decimal number - - -The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful -error message. - -``"float"`` and ``"complex"`` option arguments are converted directly with -:func:`float` and :func:`complex`, with similar error-handling. - -``"choice"`` options are a subtype of ``"string"`` options. The -:attr:`~Option.choices` option attribute (a sequence of strings) defines the -set of allowed option arguments. :func:`optparse.check_choice` compares -user-supplied option arguments against this master list and raises -:exc:`OptionValueError` if an invalid string is given. - - -.. _optparse-parsing-arguments: - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -The whole point of creating and populating an OptionParser is to call its -:meth:`parse_args` method:: - - (options, args) = parser.parse_args(args=None, values=None) - -where the input parameters are - -``args`` - the list of arguments to process (default: ``sys.argv[1:]``) - -``values`` - an :class:`optparse.Values` object to store option arguments in (default: a - new instance of :class:`Values`) -- if you give an existing object, the - option defaults will not be initialized on it - -and the return values are - -``options`` - the same object that was passed in as ``values``, or the optparse.Values - instance created by :mod:`optparse` - -``args`` - the leftover positional arguments after all options have been processed - -The most common usage is to supply neither keyword argument. If you supply -``values``, it will be modified with repeated :func:`setattr` calls (roughly one -for every option argument stored to an option destination) and returned by -:meth:`parse_args`. - -If :meth:`parse_args` encounters any errors in the argument list, it calls the -OptionParser's :meth:`error` method with an appropriate end-user error message. -This ultimately terminates your process with an exit status of 2 (the -traditional Unix exit status for command-line errors). - - -.. _optparse-querying-manipulating-option-parser: - -Querying and manipulating your option parser -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The default behavior of the option parser can be customized slightly, and you -can also poke around your option parser and see what's there. OptionParser -provides several methods to help you out: - -.. method:: OptionParser.disable_interspersed_args() - - Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` - normally accepts this syntax:: - - prog -a arg1 -b arg2 - - and treats it as equivalent to :: - - prog -a -b arg1 arg2 - - To disable this feature, call :meth:`disable_interspersed_args`. This - restores traditional Unix syntax, where option parsing stops with the first - non-option argument. - - Use this if you have a command processor which runs another command which has - options of its own and you want to make sure these options don't get - confused. For example, each command might have a different set of options. - -.. method:: OptionParser.enable_interspersed_args() - - Set parsing to not stop on the first non-option, allowing interspersing - switches with command arguments. This is the default behavior. - -.. method:: OptionParser.get_option(opt_str) - - Returns the Option instance with the option string *opt_str*, or ``None`` if - no options have that option string. - -.. method:: OptionParser.has_option(opt_str) - - Return true if the OptionParser has an option with option string *opt_str* - (e.g., ``-q`` or ``--verbose``). - -.. method:: OptionParser.remove_option(opt_str) - - If the :class:`OptionParser` has an option corresponding to *opt_str*, that - option is removed. If that option provided any other option strings, all of - those option strings become invalid. If *opt_str* does not occur in any - option belonging to this :class:`OptionParser`, raises :exc:`ValueError`. - - -.. _optparse-conflicts-between-options: - -Conflicts between options -^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you're not careful, it's easy to define options with conflicting option -strings:: - - parser.add_option("-n", "--dry-run", ...) - ... - parser.add_option("-n", "--noisy", ...) - -(This is particularly true if you've defined your own OptionParser subclass with -some standard options.) - -Every time you add an option, :mod:`optparse` checks for conflicts with existing -options. If it finds any, it invokes the current conflict-handling mechanism. -You can set the conflict-handling mechanism either in the constructor:: - - parser = OptionParser(..., conflict_handler=handler) - -or with a separate call:: - - parser.set_conflict_handler(handler) - -The available conflict handlers are: - - ``"error"`` (default) - assume option conflicts are a programming error and raise - :exc:`OptionConflictError` - - ``"resolve"`` - resolve option conflicts intelligently (see below) - - -As an example, let's define an :class:`OptionParser` that resolves conflicts -intelligently and add conflicting options to it:: - - parser = OptionParser(conflict_handler="resolve") - parser.add_option("-n", "--dry-run", ..., help="do no harm") - parser.add_option("-n", "--noisy", ..., help="be noisy") - -At this point, :mod:`optparse` detects that a previously-added option is already -using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, -it resolves the situation by removing ``-n`` from the earlier option's list of -option strings. Now ``--dry-run`` is the only way for the user to activate -that option. If the user asks for help, the help message will reflect that:: - - Options: - --dry-run do no harm - ... - -n, --noisy be noisy - -It's possible to whittle away the option strings for a previously-added option -until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, -so it doesn't show up in help text or anywhere else. Carrying on with our -existing OptionParser:: - - parser.add_option("--dry-run", ..., help="new dry-run option") - -At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: - - Options: - ... - -n, --noisy be noisy - --dry-run new dry-run option - - -.. _optparse-cleanup: - -Cleanup -^^^^^^^ - -OptionParser instances have several cyclic references. This should not be a -problem for Python's garbage collector, but you may wish to break the cyclic -references explicitly by calling :meth:`~OptionParser.destroy` on your -OptionParser once you are done with it. This is particularly useful in -long-running applications where large object graphs are reachable from your -OptionParser. - - -.. _optparse-other-methods: - -Other methods -^^^^^^^^^^^^^ - -OptionParser supports several other public methods: - -.. method:: OptionParser.set_usage(usage) - - Set the usage string according to the rules described above for the ``usage`` - constructor keyword argument. Passing ``None`` sets the default usage - string; use :data:`optparse.SUPPRESS_USAGE` to suppress a usage message. - -.. method:: OptionParser.print_usage(file=None) - - Print the usage message for the current program (``self.usage``) to *file* - (default stdout). Any occurrence of the string ``%prog`` in ``self.usage`` - is replaced with the name of the current program. Does nothing if - ``self.usage`` is empty or not defined. - -.. method:: OptionParser.get_usage() - - Same as :meth:`print_usage` but returns the usage string instead of - printing it. - -.. method:: OptionParser.set_defaults(dest=value, ...) - - Set default values for several option destinations at once. Using - :meth:`set_defaults` is the preferred way to set default values for options, - since multiple options can share the same destination. For example, if - several "mode" options all set the same destination, any one of them can set - the default, and the last one wins:: - - parser.add_option("--advanced", action="store_const", - dest="mode", const="advanced", - default="novice") # overridden below - parser.add_option("--novice", action="store_const", - dest="mode", const="novice", - default="advanced") # overrides above setting - - To avoid this confusion, use :meth:`set_defaults`:: - - parser.set_defaults(mode="advanced") - parser.add_option("--advanced", action="store_const", - dest="mode", const="advanced") - parser.add_option("--novice", action="store_const", - dest="mode", const="novice") - - -.. _optparse-option-callbacks: - -Option Callbacks ----------------- - -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple -cases. Quite often a simple callback is all you need. - -There are two steps to defining a callback option: - -* define the option itself using the ``"callback"`` action - -* write the callback; this is a function (or method) that takes at least four - arguments, as described below - - -.. _optparse-defining-callback-option: - -Defining a callback option -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As always, the easiest way to define a callback option is by using the -:meth:`OptionParser.add_option` method. Apart from :attr:`~Option.action`, the -only option attribute you must specify is ``callback``, the function to call:: - - parser.add_option("-c", action="callback", callback=my_callback) - -``callback`` is a function (or other callable object), so you must have already -defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, -which usually means that the option takes no arguments---the mere presence of -``-c`` on the command-line is all it needs to know. In some -circumstances, though, you might want your callback to consume an arbitrary -number of command-line arguments. This is where writing callbacks gets tricky; -it's covered later in this section. - -:mod:`optparse` always passes four particular arguments to your callback, and it -will only pass additional arguments if you specify them via -:attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the -minimal callback function signature is:: - - def my_callback(option, opt, value, parser): - -The four arguments to a callback are described below. - -There are several other option attributes that you can supply when you define a -callback option: - -:attr:`~Option.type` - has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to - :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. - -:attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will - consume :attr:`~Option.nargs` arguments, each of which must be convertible to - :attr:`~Option.type`. It then passes a tuple of converted values to your - callback. - -:attr:`~Option.callback_args` - a tuple of extra positional arguments to pass to the callback - -:attr:`~Option.callback_kwargs` - a dictionary of extra keyword arguments to pass to the callback - - -.. _optparse-how-callbacks-called: - -How callbacks are called -^^^^^^^^^^^^^^^^^^^^^^^^ - -All callbacks are called as follows:: - - func(option, opt_str, value, parser, *args, **kwargs) - -where - -``option`` - is the Option instance that's calling the callback - -``opt_str`` - is the option string seen on the command-line that's triggering the callback. - (If an abbreviated long option was used, ``opt_str`` will be the full, - canonical option string---e.g. if the user puts ``--foo`` on the - command-line as an abbreviation for ``--foobar``, then ``opt_str`` will be - ``"--foobar"``.) - -``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will - only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be - the type implied by the option's type. If :attr:`~Option.type` for this option is - ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` - > 1, ``value`` will be a tuple of values of the appropriate type. - -``parser`` - is the OptionParser instance driving the whole thing, mainly useful because - you can access some other interesting data through its instance attributes: - - ``parser.largs`` - the current list of leftover arguments, ie. arguments that have been - consumed but are neither options nor option arguments. Feel free to modify - ``parser.largs``, e.g. by adding more arguments to it. (This list will - become ``args``, the second return value of :meth:`parse_args`.) - - ``parser.rargs`` - the current list of remaining arguments, ie. with ``opt_str`` and - ``value`` (if applicable) removed, and only the arguments following them - still there. Feel free to modify ``parser.rargs``, e.g. by consuming more - arguments. - - ``parser.values`` - the object where option values are by default stored (an instance of - optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess - around with globals or closures. You can also access or modify the - value(s) of any options already encountered on the command-line. - -``args`` - is a tuple of arbitrary positional arguments supplied via the - :attr:`~Option.callback_args` option attribute. - -``kwargs`` - is a dictionary of arbitrary keyword arguments supplied via - :attr:`~Option.callback_kwargs`. - - -.. _optparse-raising-errors-in-callback: - -Raising errors in a callback -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and -terminates the program, printing the error message you supply to stderr. Your -message should be clear, concise, accurate, and mention the option at fault. -Otherwise, the user will have a hard time figuring out what they did wrong. - - -.. _optparse-callback-example-1: - -Callback example 1: trivial callback -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Here's an example of a callback option that takes no arguments, and simply -records that the option was seen:: - - def record_foo_seen(option, opt_str, value, parser): - parser.values.saw_foo = True - - parser.add_option("--foo", action="callback", callback=record_foo_seen) - -Of course, you could do that with the ``"store_true"`` action. - - -.. _optparse-callback-example-2: - -Callback example 2: check option order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Here's a slightly more interesting example: record the fact that ``-a`` is -seen, but blow up if it comes after ``-b`` in the command-line. :: - - def check_order(option, opt_str, value, parser): - if parser.values.b: - raise OptionValueError("can't use -a after -b") - parser.values.a = 1 - ... - parser.add_option("-a", action="callback", callback=check_order) - parser.add_option("-b", action="store_true", dest="b") - - -.. _optparse-callback-example-3: - -Callback example 3: check option order (generalized) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to re-use this callback for several similar options (set a flag, but -blow up if ``-b`` has already been seen), it needs a bit of work: the error -message and the flag that it sets must be generalized. :: - - def check_order(option, opt_str, value, parser): - if parser.values.b: - raise OptionValueError("can't use %s after -b" % opt_str) - setattr(parser.values, option.dest, 1) - ... - parser.add_option("-a", action="callback", callback=check_order, dest='a') - parser.add_option("-b", action="store_true", dest="b") - parser.add_option("-c", action="callback", callback=check_order, dest='c') - - -.. _optparse-callback-example-4: - -Callback example 4: check arbitrary condition -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Of course, you could put any condition in there---you're not limited to checking -the values of already-defined options. For example, if you have options that -should not be called when the moon is full, all you have to do is this:: - - def check_moon(option, opt_str, value, parser): - if is_moon_full(): - raise OptionValueError("%s option invalid when moon is full" - % opt_str) - setattr(parser.values, option.dest, 1) - ... - parser.add_option("--foo", - action="callback", callback=check_moon, dest="foo") - -(The definition of ``is_moon_full()`` is left as an exercise for the reader.) - - -.. _optparse-callback-example-5: - -Callback example 5: fixed arguments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Things get slightly more interesting when you define callback options that take -a fixed number of arguments. Specifying that a callback option takes arguments -is similar to defining a ``"store"`` or ``"append"`` option: if you define -:attr:`~Option.type`, then the option takes one argument that must be -convertible to that type; if you further define :attr:`~Option.nargs`, then the -option takes :attr:`~Option.nargs` arguments. - -Here's an example that just emulates the standard ``"store"`` action:: - - def store_value(option, opt_str, value, parser): - setattr(parser.values, option.dest, value) - ... - parser.add_option("--foo", - action="callback", callback=store_value, - type="int", nargs=3, dest="foo") - -Note that :mod:`optparse` takes care of consuming 3 arguments and converting -them to integers for you; all you have to do is store them. (Or whatever; -obviously you don't need a callback for this example.) - - -.. _optparse-callback-example-6: - -Callback example 6: variable arguments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any -built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for -you. In particular, callbacks should implement the conventional rules for bare -``--`` and ``-`` arguments: - -* either ``--`` or ``-`` can be option arguments - -* bare ``--`` (if not the argument to some option): halt command-line - processing and discard the ``--`` - -* bare ``-`` (if not the argument to some option): halt command-line - processing but keep the ``-`` (append it to ``parser.largs``) - -If you want an option that takes a variable number of arguments, there are -several subtle, tricky issues to worry about. The exact implementation you -choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing -directly). - -Nevertheless, here's a stab at a callback for an option with variable -arguments:: - - def vararg_callback(option, opt_str, value, parser): - assert value is None - value = [] - - def floatable(str): - try: - float(str) - return True - except ValueError: - return False - - for arg in parser.rargs: - # stop on --foo like options - if arg[:2] == "--" and len(arg) > 2: - break - # stop on -a, but not on -3 or -3.0 - if arg[:1] == "-" and len(arg) > 1 and not floatable(arg): - break - value.append(arg) - - del parser.rargs[:len(value)] - setattr(parser.values, option.dest, value) - - ... - parser.add_option("-c", "--callback", dest="vararg_attr", - action="callback", callback=vararg_callback) - - -.. _optparse-extending-optparse: - -Extending :mod:`optparse` -------------------------- - -Since the two major controlling factors in how :mod:`optparse` interprets -command-line options are the action and type of each option, the most likely -direction of extension is to add new actions and new types. - - -.. _optparse-adding-new-types: - -Adding new types -^^^^^^^^^^^^^^^^ - -To add new types, you need to define your own subclass of :mod:`optparse`'s -:class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. - -.. attribute:: Option.TYPES - - A tuple of type names; in your subclass, simply define a new tuple - :attr:`TYPES` that builds on the standard one. - -.. attribute:: Option.TYPE_CHECKER - - A dictionary mapping type names to type-checking functions. A type-checking - function has the following signature:: - - def check_mytype(option, opt, value) - - where ``option`` is an :class:`Option` instance, ``opt`` is an option string - (e.g., ``-f``), and ``value`` is the string from the command line that must - be checked and converted to your desired type. ``check_mytype()`` should - return an object of the hypothetical type ``mytype``. The value returned by - a type-checking function will wind up in the OptionValues instance returned - by :meth:`OptionParser.parse_args`, or be passed to a callback as the - ``value`` parameter. - - Your type-checking function should raise :exc:`OptionValueError` if it - encounters any problems. :exc:`OptionValueError` takes a single string - argument, which is passed as-is to :class:`OptionParser`'s :meth:`error` - method, which in turn prepends the program name and the string ``"error:"`` - and prints everything to stderr before terminating the process. - -Here's a silly example that demonstrates adding a ``"complex"`` option type to -parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for -complex numbers, but never mind.) - -First, the necessary imports:: - - from copy import copy - from optparse import Option, OptionValueError - -You need to define your type-checker first, since it's referred to later (in the -:attr:`~Option.TYPE_CHECKER` class attribute of your Option subclass):: - - def check_complex(option, opt, value): - try: - return complex(value) - except ValueError: - raise OptionValueError( - "option %s: invalid complex value: %r" % (opt, value)) - -Finally, the Option subclass:: - - class MyOption (Option): - TYPES = Option.TYPES + ("complex",) - TYPE_CHECKER = copy(Option.TYPE_CHECKER) - TYPE_CHECKER["complex"] = check_complex - -(If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s -Option class. This being Python, nothing stops you from doing that except good -manners and common sense.) - -That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your -OptionParser to use MyOption instead of Option:: - - parser = OptionParser(option_class=MyOption) - parser.add_option("-c", type="complex") - -Alternately, you can build your own option list and pass it to OptionParser; if -you don't use :meth:`add_option` in the above way, you don't need to tell -OptionParser which option class to use:: - - option_list = [MyOption("-c", action="store", type="complex", dest="c")] - parser = OptionParser(option_list=option_list) - - -.. _optparse-adding-new-actions: - -Adding new actions -^^^^^^^^^^^^^^^^^^ - -Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: - -"store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the - current OptionValues instance; these options require a :attr:`~Option.dest` - attribute to be supplied to the Option constructor. - -"typed" actions - actions that take a value from the command line and expect it to be of a - certain type; or rather, a string that can be converted to a certain type. - These options require a :attr:`~Option.type` attribute to the Option - constructor. - -These are overlapping sets: some default "store" actions are ``"store"``, -``"store_const"``, ``"append"``, and ``"count"``, while the default "typed" -actions are ``"store"``, ``"append"``, and ``"callback"``. - -When you add an action, you need to categorize it by listing it in at least one -of the following class attributes of Option (all are lists of strings): - -.. attribute:: Option.ACTIONS - - All actions must be listed in ACTIONS. - -.. attribute:: Option.STORE_ACTIONS - - "store" actions are additionally listed here. - -.. attribute:: Option.TYPED_ACTIONS - - "typed" actions are additionally listed here. - -.. attribute:: Option.ALWAYS_TYPED_ACTIONS - - Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` - assigns the default type, ``"string"``, to options with no explicit type - whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. - -In order to actually implement your new action, you must override Option's -:meth:`take_action` method and add a case that recognizes your action. - -For example, let's add an ``"extend"`` action. This is similar to the standard -``"append"`` action, but instead of taking a single value from the command-line -and appending it to an existing list, ``"extend"`` will take multiple values in -a single comma-delimited string, and extend an existing list with them. That -is, if ``--names`` is an ``"extend"`` option of type ``"string"``, the command -line :: - - --names=foo,bar --names blah --names ding,dong - -would result in a list :: - - ["foo", "bar", "blah", "ding", "dong"] - -Again we define a subclass of Option:: - - class MyOption(Option): - - ACTIONS = Option.ACTIONS + ("extend",) - STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) - TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) - ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) - - def take_action(self, action, dest, opt, value, values, parser): - if action == "extend": - lvalue = value.split(",") - values.ensure_value(dest, []).extend(lvalue) - else: - Option.take_action( - self, action, dest, opt, value, values, parser) - -Features of note: - -* ``"extend"`` both expects a value on the command-line and stores that value - somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and - :attr:`~Option.TYPED_ACTIONS`. - -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to - ``"extend"`` actions, we put the ``"extend"`` action in - :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. - -* :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` - actions. - -* ``values`` is an instance of the optparse_parser.Values class, which provides - the very useful :meth:`ensure_value` method. :meth:`ensure_value` is - essentially :func:`getattr` with a safety valve; it is called as :: - - values.ensure_value(attr, value) - - If the ``attr`` attribute of ``values`` doesn't exist or is ``None``, then - ensure_value() first sets it to ``value``, and then returns 'value. This is - very handy for actions like ``"extend"``, ``"append"``, and ``"count"``, all - of which accumulate data in a variable and expect that variable to be of a - certain type (a list for the first two, an integer for the latter). Using - :meth:`ensure_value` means that scripts using your action don't have to worry - about setting a default value for the option destinations in question; they - can just leave the default as ``None`` and :meth:`ensure_value` will take care of - getting it right when it's needed. diff --git a/third_party/python/Doc/library/os.path.rst b/third_party/python/Doc/library/os.path.rst deleted file mode 100644 index 9b655d7ec..000000000 --- a/third_party/python/Doc/library/os.path.rst +++ /dev/null @@ -1,480 +0,0 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ - -.. module:: os.path - :synopsis: Operations on pathnames. - -**Source code:** :source:`Lib/posixpath.py` (for POSIX), -:source:`Lib/ntpath.py` (for Windows NT), -and :source:`Lib/macpath.py` (for Macintosh) - -.. index:: single: path; operations - --------------- - -This module implements some useful functions on pathnames. To read or -write files see :func:`open`, and for accessing the filesystem see the -:mod:`os` module. The path parameters can be passed as either strings, -or bytes. Applications are encouraged to represent file names as -(Unicode) character strings. Unfortunately, some file names may not be -representable as strings on Unix, so applications that need to support -arbitrary file names on Unix should use bytes objects to represent -path names. Vice versa, using bytes objects cannot represent all file -names on Windows (in the standard ``mbcs`` encoding), hence Windows -applications should use string objects to access all files. - -Unlike a unix shell, Python does not do any *automatic* path expansions. -Functions such as :func:`expanduser` and :func:`expandvars` can be invoked -explicitly when an application desires shell-like path expansion. (See also -the :mod:`glob` module.) - - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - -.. note:: - - All of these functions accept either only bytes or only string objects as - their parameters. The result is an object of the same type, if a path or - file name is returned. - - -.. note:: - - Since different operating systems have different path name conventions, there - are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating - system Python is running on, and therefore usable for local paths. However, - you can also import and use the individual modules if you want to manipulate - a path that is *always* in one of the different formats. They all have the - same interface: - - * :mod:`posixpath` for UNIX-style paths - * :mod:`ntpath` for Windows paths - * :mod:`macpath` for old-style MacOS paths - - -.. function:: abspath(path) - - Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: basename(path) - - Return the base name of pathname *path*. This is the second element of the - pair returned by passing *path* to the function :func:`split`. Note that - the result of this function is different - from the Unix :program:`basename` program; where :program:`basename` for - ``'/foo/bar/'`` returns ``'bar'``, the :func:`basename` function returns an - empty string (``''``). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: commonpath(paths) - - Return the longest common sub-path of each pathname in the sequence - *paths*. Raise ValueError if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. - - Availability: Unix, Windows - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - Accepts a sequence of :term:`path-like objects `. - - -.. function:: commonprefix(list) - - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string - (``''``). - - .. note:: - - This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. - - :: - - >>> os.path.commonprefix(['/usr/lib', '/usr/local/lib']) - '/usr/l' - - >>> os.path.commonpath(['/usr/lib', '/usr/local/lib']) - '/usr' - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: dirname(path) - - Return the directory name of pathname *path*. This is the first element of - the pair returned by passing *path* to the function :func:`split`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: exists(path) - - Return ``True`` if *path* refers to an existing path or an open - file descriptor. Returns ``False`` for broken symbolic links. On - some platforms, this function may return ``False`` if permission is - not granted to execute :func:`os.stat` on the requested file, even - if the *path* physically exists. - - .. versionchanged:: 3.3 - *path* can now be an integer: ``True`` is returned if it is an - open file descriptor, ``False`` otherwise. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lexists(path) - - Return ``True`` if *path* refers to an existing path. Returns ``True`` for - broken symbolic links. Equivalent to :func:`exists` on platforms lacking - :func:`os.lstat`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. index:: single: ~ (tilde); home directory expansion - -.. function:: expanduser(path) - - On Unix and Windows, return the argument with an initial component of ``~`` or - ``~user`` replaced by that *user*'s home directory. - - .. index:: module: pwd - - On Unix, an initial ``~`` is replaced by the environment variable :envvar:`HOME` - if it is set; otherwise the current user's home directory is looked up in the - password directory through the built-in module :mod:`pwd`. An initial ``~user`` - is looked up directly in the password directory. - - On Windows, :envvar:`HOME` and :envvar:`USERPROFILE` will be used if set, - otherwise a combination of :envvar:`HOMEPATH` and :envvar:`HOMEDRIVE` will be - used. An initial ``~user`` is handled by stripping the last directory component - from the created user path derived above. - - If the expansion fails or if the path does not begin with a tilde, the path is - returned unchanged. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. index:: - single: $ (dollar); environment variables expansion - single: % (percent); environment variables expansion (Windows) - -.. function:: expandvars(path) - - Return the argument with environment variables expanded. Substrings of the form - ``$name`` or ``${name}`` are replaced by the value of environment variable - *name*. Malformed variable names and references to non-existing variables are - left unchanged. - - On Windows, ``%name%`` expansions are supported in addition to ``$name`` and - ``${name}``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getatime(path) - - Return the time of last access of *path*. The return value is a number giving - the number of seconds since the epoch (see the :mod:`time` module). Raise - :exc:`OSError` if the file does not exist or is inaccessible. - - If :func:`os.stat_float_times` returns ``True``, the result is a floating point - number. - - -.. function:: getmtime(path) - - Return the time of last modification of *path*. The return value is a number - giving the number of seconds since the epoch (see the :mod:`time` module). - Raise :exc:`OSError` if the file does not exist or is inaccessible. - - If :func:`os.stat_float_times` returns ``True``, the result is a floating point - number. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getctime(path) - - Return the system's ctime which, on some systems (like Unix) is the time of the - last metadata change, and, on others (like Windows), is the creation time for *path*. - The return value is a number giving the number of seconds since the epoch (see - the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or - is inaccessible. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getsize(path) - - Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does - not exist or is inaccessible. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isabs(path) - - Return ``True`` if *path* is an absolute pathname. On Unix, that means it - begins with a slash, on Windows that it begins with a (back)slash after chopping - off a potential drive letter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isfile(path) - - Return ``True`` if *path* is an :func:`existing ` regular file. - This follows symbolic links, so both :func:`islink` and :func:`isfile` can - be true for the same path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isdir(path) - - Return ``True`` if *path* is an :func:`existing ` directory. This - follows symbolic links, so both :func:`islink` and :func:`isdir` can be true - for the same path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: islink(path) - - Return ``True`` if *path* refers to an :func:`existing ` directory - entry that is a symbolic link. Always ``False`` if symbolic links are not - supported by the Python runtime. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: ismount(path) - - Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. - - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: join(path, *paths) - - Join one or more path components intelligently. The return value is the - concatenation of *path* and any members of *\*paths* with exactly one - directory separator (``os.sep``) following each non-empty part except the - last, meaning that the result will only end in a separator if the last - part is empty. If a component is an absolute path, all previous - components are thrown away and joining continues from the absolute path - component. - - On Windows, the drive letter is not reset when an absolute path component - (e.g., ``r'\foo'``) is encountered. If a component contains a drive - letter, all previous components are thrown away and the drive letter is - reset. Note that since there is a current directory for each drive, - ``os.path.join("c:", "foo")`` represents a path relative to the current - directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *paths*. - - -.. function:: normcase(path) - - Normalize the case of a pathname. On Unix and Mac OS X, this returns the - path unchanged; on case-insensitive filesystems, it converts the path to - lowercase. On Windows, it also converts forward slashes to backward slashes. - Raise a TypeError if the type of *path* is not ``str`` or ``bytes`` (directly - or indirectly through the :class:`os.PathLike` interface). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: normpath(path) - - Normalize a pathname by collapsing redundant separators and up-level - references so that ``A//B``, ``A/B/``, ``A/./B`` and ``A/foo/../B`` all - become ``A/B``. This string manipulation may change the meaning of a path - that contains symbolic links. On Windows, it converts forward slashes to - backward slashes. To normalize case, use :func:`normcase`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: realpath(path) - - Return the canonical path of the specified filename, eliminating any symbolic - links encountered in the path (if they are supported by the operating system). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: relpath(path, start=os.curdir) - - Return a relative filepath to *path* either from the current directory or - from an optional *start* directory. This is a path computation: the - filesystem is not accessed to confirm the existence or nature of *path* or - *start*. - - *start* defaults to :attr:`os.curdir`. - - Availability: Unix, Windows. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: samefile(path1, path2) - - Return ``True`` if both pathname arguments refer to the same file or directory. - This is determined by the device number and i-node number and raises an - exception if an :func:`os.stat` call on either pathname fails. - - Availability: Unix, Windows. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionchanged:: 3.4 - Windows now uses the same implementation as all other platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: sameopenfile(fp1, fp2) - - Return ``True`` if the file descriptors *fp1* and *fp2* refer to the same file. - - Availability: Unix, Windows. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: samestat(stat1, stat2) - - Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. - These structures may have been returned by :func:`os.fstat`, - :func:`os.lstat`, or :func:`os.stat`. This function implements the - underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - - Availability: Unix, Windows. - - .. versionchanged:: 3.4 - Added Windows support. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: split(path) - - Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the - last pathname component and *head* is everything leading up to that. The - *tail* part will never contain a slash; if *path* ends in a slash, *tail* - will be empty. If there is no slash in *path*, *head* will be empty. If - *path* is empty, both *head* and *tail* are empty. Trailing slashes are - stripped from *head* unless it is the root (one or more slashes only). In - all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: splitdrive(path) - - Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either - a mount point or the empty string. On systems which do not use drive - specifications, *drive* will always be the empty string. In all cases, ``drive - + tail`` will be the same as *path*. - - On Windows, splits a pathname into drive/UNC sharepoint and relative path. - - If the path contains a drive letter, drive will contain everything - up to and including the colon. - e.g. ``splitdrive("c:/dir")`` returns ``("c:", "/dir")`` - - If the path contains a UNC path, drive will contain the host name - and share, up to but not including the fourth separator. - e.g. ``splitdrive("//host/computer/dir")`` returns ``("//host/computer", "/dir")`` - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: splitext(path) - - Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == - path``, and *ext* is empty or begins with a period and contains at most one - period. Leading periods on the basename are ignored; ``splitext('.cshrc')`` - returns ``('.cshrc', '')``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: splitunc(path) - - .. deprecated:: 3.1 - Use *splitdrive* instead. - - Split the pathname *path* into a pair ``(unc, rest)`` so that *unc* is the UNC - mount point (such as ``r'\\host\mount'``), if present, and *rest* the rest of - the path (such as ``r'\path\file.ext'``). For paths containing drive letters, - *unc* will always be the empty string. - - Availability: Windows. - - -.. data:: supports_unicode_filenames - - ``True`` if arbitrary Unicode strings can be used as file names (within limitations - imposed by the file system). diff --git a/third_party/python/Doc/library/os.rst b/third_party/python/Doc/library/os.rst deleted file mode 100644 index a36164ea7..000000000 --- a/third_party/python/Doc/library/os.rst +++ /dev/null @@ -1,4042 +0,0 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= - -.. module:: os - :synopsis: Miscellaneous operating system interfaces. - -**Source code:** :source:`Lib/os.py` - --------------- - -This module provides a portable way of using operating system dependent -functionality. If you just want to read or write a file see :func:`open`, if -you want to manipulate paths, see the :mod:`os.path` module, and if you want to -read all the lines in all the files on the command line see the :mod:`fileinput` -module. For creating temporary files and directories see the :mod:`tempfile` -module, and for high-level file and directory handling see the :mod:`shutil` -module. - -Notes on the availability of these functions: - -* The design of all built-in operating system dependent modules of Python is - such that as long as the same functionality is available, it uses the same - interface; for example, the function ``os.stat(path)`` returns stat - information about *path* in the same format (which happens to have originated - with the POSIX interface). - -* Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to - portability. - -* All functions accepting path or file names accept both bytes and string - objects, and result in an object of the same type, if a path or file name is - returned. - -* An "Availability: Unix" note means that this function is commonly found on - Unix systems. It does not make any claims about its existence on a specific - operating system. - -* If not separately noted, all functions that claim "Availability: Unix" are - supported on Mac OS X, which builds on a Unix core. - -.. Availability notes get their own line and occur at the end of the function -.. documentation. - -.. note:: - - All functions in this module raise :exc:`OSError` in the case of invalid or - inaccessible file names and paths, or other arguments that have the correct - type, but are not accepted by the operating system. - -.. exception:: error - - An alias for the built-in :exc:`OSError` exception. - - -.. data:: name - - The name of the operating system dependent module imported. The following - names have currently been registered: ``'posix'``, ``'nt'``, - ``'java'``. - - .. seealso:: - :attr:`sys.platform` has a finer granularity. :func:`os.uname` gives - system-dependent version information. - - The :mod:`platform` module provides detailed checks for the - system's identity. - - -.. _os-filenames: -.. _filesystem-encoding: - -File Names, Command Line Arguments, and Environment Variables -------------------------------------------------------------- - -In Python, file names, command line arguments, and environment variables are -represented using the string type. On some systems, decoding these strings to -and from bytes is necessary before passing them to the operating system. Python -uses the file system encoding to perform this conversion (see -:func:`sys.getfilesystemencoding`). - -.. versionchanged:: 3.1 - On some systems, conversion using the file system encoding may fail. In this - case, Python uses the :ref:`surrogateescape encoding error handler - `, which means that undecodable bytes are replaced by a - Unicode character U+DCxx on decoding, and these are again translated to the - original byte on encoding. - - -The file system encoding must guarantee to successfully decode all bytes -below 128. If the file system encoding fails to provide this guarantee, API -functions may raise UnicodeErrors. - - -.. _os-procinfo: - -Process Parameters ------------------- - -These functions and data items provide information and operate on the current -process and user. - - -.. function:: ctermid() - - Return the filename corresponding to the controlling terminal of the process. - - Availability: Unix. - - -.. data:: environ - - A :term:`mapping` object representing the string environment. For example, - ``environ['HOME']`` is the pathname of your home directory (on some platforms), - and is equivalent to ``getenv("HOME")`` in C. - - This mapping is captured the first time the :mod:`os` module is imported, - typically during Python startup as part of processing :file:`site.py`. Changes - to the environment made after this time are not reflected in ``os.environ``, - except for changes made by modifying ``os.environ`` directly. - - If the platform supports the :func:`putenv` function, this mapping may be used - to modify the environment as well as query the environment. :func:`putenv` will - be called automatically when the mapping is modified. - - On Unix, keys and values use :func:`sys.getfilesystemencoding` and - ``'surrogateescape'`` error handler. Use :data:`environb` if you would like - to use a different encoding. - - .. note:: - - Calling :func:`putenv` directly does not change ``os.environ``, so it's better - to modify ``os.environ``. - - .. note:: - - On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may - cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. - - If :func:`putenv` is not provided, a modified copy of this mapping may be - passed to the appropriate process-creation functions to cause child processes - to use a modified environment. - - If the platform supports the :func:`unsetenv` function, you can delete items in - this mapping to unset environment variables. :func:`unsetenv` will be called - automatically when an item is deleted from ``os.environ``, and when - one of the :meth:`pop` or :meth:`clear` methods is called. - - -.. data:: environb - - Bytes version of :data:`environ`: a :term:`mapping` object representing the - environment as byte strings. :data:`environ` and :data:`environb` are - synchronized (modify :data:`environb` updates :data:`environ`, and vice - versa). - - :data:`environb` is only available if :data:`supports_bytes_environ` is - True. - - .. versionadded:: 3.2 - - -.. function:: chdir(path) - fchdir(fd) - getcwd() - :noindex: - - These functions are described in :ref:`os-file-dir`. - - -.. function:: fsencode(filename) - - Encode :term:`path-like ` *filename* to the filesystem - encoding with ``'surrogateescape'`` error handler, or ``'strict'`` on - Windows; return :class:`bytes` unchanged. - - :func:`fsdecode` is the reverse function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Support added to accept objects implementing the :class:`os.PathLike` - interface. - - -.. function:: fsdecode(filename) - - Decode the :term:`path-like ` *filename* from the - filesystem encoding with ``'surrogateescape'`` error handler, or ``'strict'`` - on Windows; return :class:`str` unchanged. - - :func:`fsencode` is the reverse function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Support added to accept objects implementing the :class:`os.PathLike` - interface. - - -.. function:: fspath(path) - - Return the file system representation of the path. - - If :class:`str` or :class:`bytes` is passed in, it is returned unchanged. - Otherwise :meth:`~os.PathLike.__fspath__` is called and its value is - returned as long as it is a :class:`str` or :class:`bytes` object. - In all other cases, :exc:`TypeError` is raised. - - .. versionadded:: 3.6 - - -.. class:: PathLike - - An :term:`abstract base class` for objects representing a file system path, - e.g. :class:`pathlib.PurePath`. - - .. versionadded:: 3.6 - - .. abstractmethod:: __fspath__() - - Return the file system path representation of the object. - - The method should only return a :class:`str` or :class:`bytes` object, - with the preference being for :class:`str`. - - -.. function:: getenv(key, default=None) - - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are str. - - On Unix, keys and values are decoded with :func:`sys.getfilesystemencoding` - and ``'surrogateescape'`` error handler. Use :func:`os.getenvb` if you - would like to use a different encoding. - - Availability: most flavors of Unix, Windows. - - -.. function:: getenvb(key, default=None) - - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are bytes. - - :func:`getenvb` is only available if :data:`supports_bytes_environ` - is True. - - Availability: most flavors of Unix. - - .. versionadded:: 3.2 - - -.. function:: get_exec_path(env=None) - - Returns the list of directories that will be searched for a named - executable, similar to a shell, when launching a process. - *env*, when specified, should be an environment variable dictionary - to lookup the PATH in. - By default, when *env* is ``None``, :data:`environ` is used. - - .. versionadded:: 3.2 - - -.. function:: getegid() - - Return the effective group id of the current process. This corresponds to the - "set id" bit on the file being executed in the current process. - - Availability: Unix. - - -.. function:: geteuid() - - .. index:: single: user; effective id - - Return the current process's effective user id. - - Availability: Unix. - - -.. function:: getgid() - - .. index:: single: process; group - - Return the real group id of the current process. - - Availability: Unix. - - -.. function:: getgrouplist(user, group) - - Return list of group ids that *user* belongs to. If *group* is not in the - list, it is included; typically, *group* is specified as the group ID - field from the password record for *user*. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: getgroups() - - Return list of supplemental group ids associated with the current process. - - Availability: Unix. - - .. note:: - - On Mac OS X, :func:`getgroups` behavior differs somewhat from - other Unix platforms. If the Python interpreter was built with a - deployment target of :const:`10.5` or earlier, :func:`getgroups` returns - the list of effective group ids associated with the current user process; - this list is limited to a system-defined number of entries, typically 16, - and may be modified by calls to :func:`setgroups` if suitably privileged. - If built with a deployment target greater than :const:`10.5`, - :func:`getgroups` returns the current group access list for the user - associated with the effective user id of the process; the group access - list may change over the lifetime of the process, it is not affected by - calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. - - -.. function:: getlogin() - - Return the name of the user logged in on the controlling terminal of the - process. For most purposes, it is more useful to use - :func:`getpass.getuser` since the latter checks the environment variables - :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user is, and - falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the - current real user id. - - Availability: Unix, Windows. - - -.. function:: getpgid(pid) - - Return the process group id of the process with process id *pid*. If *pid* is 0, - the process group id of the current process is returned. - - Availability: Unix. - -.. function:: getpgrp() - - .. index:: single: process; group - - Return the id of the current process group. - - Availability: Unix. - - -.. function:: getpid() - - .. index:: single: process; id - - Return the current process id. - - -.. function:: getppid() - - .. index:: single: process; id of parent - - Return the parent's process id. When the parent process has exited, on Unix - the id returned is the one of the init process (1), on Windows it is still - the same id, which may be already reused by another process. - - Availability: Unix, Windows. - - .. versionchanged:: 3.2 - Added support for Windows. - - -.. function:: getpriority(which, who) - - .. index:: single: process; scheduling priority - - Get program scheduling priority. The value *which* is one of - :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who* - is interpreted relative to *which* (a process identifier for - :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a - user ID for :const:`PRIO_USER`). A zero value for *who* denotes - (respectively) the calling process, the process group of the calling process, - or the real user ID of the calling process. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: PRIO_PROCESS - PRIO_PGRP - PRIO_USER - - Parameters for the :func:`getpriority` and :func:`setpriority` functions. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: getresuid() - - Return a tuple (ruid, euid, suid) denoting the current process's - real, effective, and saved user ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: getresgid() - - Return a tuple (rgid, egid, sgid) denoting the current process's - real, effective, and saved group ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: getuid() - - .. index:: single: user; id - - Return the current process's real user id. - - Availability: Unix. - - -.. function:: initgroups(username, gid) - - Call the system initgroups() to initialize the group access list with all of - the groups of which the specified username is a member, plus the specified - group id. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: putenv(key, value) - - .. index:: single: environment variables; setting - - Set the environment variable named *key* to the string *value*. Such - changes to the environment affect subprocesses started with :func:`os.system`, - :func:`popen` or :func:`fork` and :func:`execv`. - - Availability: most flavors of Unix, Windows. - - .. note:: - - On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may - cause memory leaks. Refer to the system documentation for putenv. - - When :func:`putenv` is supported, assignments to items in ``os.environ`` are - automatically translated into corresponding calls to :func:`putenv`; however, - calls to :func:`putenv` don't update ``os.environ``, so it is actually - preferable to assign to items of ``os.environ``. - - -.. function:: setegid(egid) - - Set the current process's effective group id. - - Availability: Unix. - - -.. function:: seteuid(euid) - - Set the current process's effective user id. - - Availability: Unix. - - -.. function:: setgid(gid) - - Set the current process' group id. - - Availability: Unix. - - -.. function:: setgroups(groups) - - Set the list of supplemental group ids associated with the current process to - *groups*. *groups* must be a sequence, and each element must be an integer - identifying a group. This operation is typically available only to the superuser. - - Availability: Unix. - - .. note:: On Mac OS X, the length of *groups* may not exceed the - system-defined maximum number of effective group ids, typically 16. - See the documentation for :func:`getgroups` for cases where it may not - return the same group list set by calling setgroups(). - -.. function:: setpgrp() - - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on - which version is implemented (if any). See the Unix manual for the semantics. - - Availability: Unix. - - -.. function:: setpgid(pid, pgrp) - - Call the system call :c:func:`setpgid` to set the process group id of the - process with id *pid* to the process group with id *pgrp*. See the Unix manual - for the semantics. - - Availability: Unix. - - -.. function:: setpriority(which, who, priority) - - .. index:: single: process; scheduling priority - - Set program scheduling priority. The value *which* is one of - :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who* - is interpreted relative to *which* (a process identifier for - :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a - user ID for :const:`PRIO_USER`). A zero value for *who* denotes - (respectively) the calling process, the process group of the calling process, - or the real user ID of the calling process. - *priority* is a value in the range -20 to 19. The default priority is 0; - lower priorities cause more favorable scheduling. - - Availability: Unix - - .. versionadded:: 3.3 - - -.. function:: setregid(rgid, egid) - - Set the current process's real and effective group ids. - - Availability: Unix. - - -.. function:: setresgid(rgid, egid, sgid) - - Set the current process's real, effective, and saved group ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: setresuid(ruid, euid, suid) - - Set the current process's real, effective, and saved user ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: setreuid(ruid, euid) - - Set the current process's real and effective user ids. - - Availability: Unix. - - -.. function:: getsid(pid) - - Call the system call :c:func:`getsid`. See the Unix manual for the semantics. - - Availability: Unix. - - -.. function:: setsid() - - Call the system call :c:func:`setsid`. See the Unix manual for the semantics. - - Availability: Unix. - - -.. function:: setuid(uid) - - .. index:: single: user; id, setting - - Set the current process's user id. - - Availability: Unix. - - -.. placed in this section since it relates to errno.... a little weak -.. function:: strerror(code) - - Return the error message corresponding to the error code in *code*. - On platforms where :c:func:`strerror` returns ``NULL`` when given an unknown - error number, :exc:`ValueError` is raised. - - -.. data:: supports_bytes_environ - - ``True`` if the native OS type of the environment is bytes (eg. ``False`` on - Windows). - - .. versionadded:: 3.2 - - -.. function:: umask(mask) - - Set the current numeric umask and return the previous umask. - - -.. function:: uname() - - .. index:: - single: gethostname() (in module socket) - single: gethostbyaddr() (in module socket) - - Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. - - Availability: recent flavors of Unix. - - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. - - -.. function:: unsetenv(key) - - .. index:: single: environment variables; deleting - - Unset (delete) the environment variable named *key*. Such changes to the - environment affect subprocesses started with :func:`os.system`, :func:`popen` or - :func:`fork` and :func:`execv`. - - When :func:`unsetenv` is supported, deletion of items in ``os.environ`` is - automatically translated into a corresponding call to :func:`unsetenv`; however, - calls to :func:`unsetenv` don't update ``os.environ``, so it is actually - preferable to delete items of ``os.environ``. - - Availability: most flavors of Unix, Windows. - - -.. _os-newstreams: - -File Object Creation --------------------- - -This function creates new :term:`file objects `. (See also -:func:`~os.open` for opening file descriptors.) - - -.. function:: fdopen(fd, *args, **kwargs) - - Return an open file object connected to the file descriptor *fd*. This is an - alias of the :func:`open` built-in function and accepts the same arguments. - The only difference is that the first argument of :func:`fdopen` must always - be an integer. - - -.. _os-fd-ops: - -File Descriptor Operations --------------------------- - -These functions operate on I/O streams referenced using file descriptors. - -File descriptors are small integers corresponding to a file that has been opened -by the current process. For example, standard input is usually file descriptor -0, standard output is 1, and standard error is 2. Further files opened by a -process will then be assigned 3, 4, 5, and so forth. The name "file descriptor" -is slightly deceptive; on Unix platforms, sockets and pipes are also referenced -by file descriptors. - -The :meth:`~io.IOBase.fileno` method can be used to obtain the file descriptor -associated with a :term:`file object` when required. Note that using the file -descriptor directly will bypass the file object methods, ignoring aspects such -as internal buffering of data. - - -.. function:: close(fd) - - Close file descriptor *fd*. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To close a "file - object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, use its :meth:`~io.IOBase.close` method. - - -.. function:: closerange(fd_low, fd_high) - - Close all file descriptors from *fd_low* (inclusive) to *fd_high* (exclusive), - ignoring errors. Equivalent to (but much faster than):: - - for fd in range(fd_low, fd_high): - try: - os.close(fd) - except OSError: - pass - - -.. function:: device_encoding(fd) - - Return a string describing the encoding of the device associated with *fd* - if it is connected to a terminal; else return :const:`None`. - - -.. function:: dup(fd) - - Return a duplicate of file descriptor *fd*. The new file descriptor is - :ref:`non-inheritable `. - - On Windows, when duplicating a standard stream (0: stdin, 1: stdout, - 2: stderr), the new file descriptor is :ref:`inheritable - `. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - -.. function:: dup2(fd, fd2, inheritable=True) - - Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary. - The file descriptor *fd2* is :ref:`inheritable ` by default, - or non-inheritable if *inheritable* is ``False``. - - .. versionchanged:: 3.4 - Add the optional *inheritable* parameter. - - -.. function:: fchmod(fd, mode) - - Change the mode of the file given by *fd* to the numeric *mode*. See the - docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this - is equivalent to ``os.chmod(fd, mode)``. - - Availability: Unix. - - -.. function:: fchown(fd, uid, gid) - - Change the owner and group id of the file given by *fd* to the numeric *uid* - and *gid*. To leave one of the ids unchanged, set it to -1. See - :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, - gid)``. - - Availability: Unix. - - -.. function:: fdatasync(fd) - - Force write of file with filedescriptor *fd* to disk. Does not force update of - metadata. - - Availability: Unix. - - .. note:: - This function is not available on MacOS. - - -.. function:: fpathconf(fd, name) - - Return system configuration information relevant to an open file. *name* - specifies the configuration value to retrieve; it may be a string which is the - name of a defined system value; these names are specified in a number of - standards (POSIX.1, Unix 95, Unix 98, and others). Some platforms define - additional names as well. The names known to the host operating system are - given in the ``pathconf_names`` dictionary. For configuration variables not - included in that mapping, passing an integer for *name* is also accepted. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``pathconf_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - - Availability: Unix. - - -.. function:: fstat(fd) - - Get the status of the file descriptor *fd*. Return a :class:`stat_result` - object. - - As of Python 3.3, this is equivalent to ``os.stat(fd)``. - - .. seealso:: - - The :func:`.stat` function. - - -.. function:: fstatvfs(fd) - - Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. - - Availability: Unix. - - -.. function:: fsync(fd) - - Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. - - If you're starting with a buffered Python :term:`file object` *f*, first do - ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal - buffers associated with *f* are written to disk. - - Availability: Unix, Windows. - - -.. function:: ftruncate(fd, length) - - Truncate the file corresponding to file descriptor *fd*, so that it is at - most *length* bytes in size. As of Python 3.3, this is equivalent to - ``os.truncate(fd, length)``. - - Availability: Unix, Windows. - - .. versionchanged:: 3.5 - Added support for Windows - -.. function:: get_blocking(fd) - - Get the blocking mode of the file descriptor: ``False`` if the - :data:`O_NONBLOCK` flag is set, ``True`` if the flag is cleared. - - See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. - - Availability: Unix. - - .. versionadded:: 3.5 - -.. function:: isatty(fd) - - Return ``True`` if the file descriptor *fd* is open and connected to a - tty(-like) device, else ``False``. - - -.. function:: lockf(fd, cmd, len) - - Apply, test or remove a POSIX lock on an open file descriptor. - *fd* is an open file descriptor. - *cmd* specifies the command to use - one of :data:`F_LOCK`, :data:`F_TLOCK`, - :data:`F_ULOCK` or :data:`F_TEST`. - *len* specifies the section of the file to lock. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: F_LOCK - F_TLOCK - F_ULOCK - F_TEST - - Flags that specify what action :func:`lockf` will take. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: lseek(fd, pos, how) - - Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. - - -.. data:: SEEK_SET - SEEK_CUR - SEEK_END - - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. - - .. versionadded:: 3.3 - Some operating systems could support additional values, like - :data:`os.SEEK_HOLE` or :data:`os.SEEK_DATA`. - - -.. function:: open(path, flags, mode=0o777, *, dir_fd=None) - - Open the file *path* and set various flags according to *flags* and possibly - its mode according to *mode*. When computing *mode*, the current umask value - is first masked out. Return the file descriptor for the newly opened file. - The new file descriptor is :ref:`non-inheritable `. - - For a description of the flag and mode values, see the C run-time documentation; - flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding - :const:`O_BINARY` is needed to open files in binary mode. - - This function can support :ref:`paths relative to directory descriptors - ` with the *dir_fd* parameter. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - .. note:: - - This function is intended for low-level I/O. For normal usage, use the - built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -The following constants are options for the *flags* parameter to the -:func:`~os.open` function. They can be combined using the bitwise OR operator -``|``. Some of them are not available on all platforms. For descriptions of -their availability and use, consult the :manpage:`open(2)` manual page on Unix -or `the MSDN `_ on Windows. - - -.. data:: O_RDONLY - O_WRONLY - O_RDWR - O_APPEND - O_CREAT - O_EXCL - O_TRUNC - - The above constants are available on Unix and Windows. - - -.. data:: O_DSYNC - O_RSYNC - O_SYNC - O_NDELAY - O_NONBLOCK - O_NOCTTY - O_CLOEXEC - - The above constants are only available on Unix. - - .. versionchanged:: 3.3 - Add :data:`O_CLOEXEC` constant. - -.. data:: O_BINARY - O_NOINHERIT - O_SHORT_LIVED - O_TEMPORARY - O_RANDOM - O_SEQUENTIAL - O_TEXT - - The above constants are only available on Windows. - - -.. data:: O_ASYNC - O_DIRECT - O_DIRECTORY - O_NOFOLLOW - O_NOATIME - O_PATH - O_TMPFILE - O_SHLOCK - O_EXLOCK - - The above constants are extensions and not present if they are not defined by - the C library. - - .. versionchanged:: 3.4 - Add :data:`O_PATH` on systems that support it. - Add :data:`O_TMPFILE`, only available on Linux Kernel 3.11 - or newer. - - -.. function:: openpty() - - .. index:: module: pty - - Open a new pseudo-terminal pair. Return a pair of file descriptors - ``(master, slave)`` for the pty and the tty, respectively. The new file - descriptors are :ref:`non-inheritable `. For a (slightly) more - portable approach, use the :mod:`pty` module. - - Availability: some flavors of Unix. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe() - - Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for - reading and writing, respectively. The new file descriptor is - :ref:`non-inheritable `. - - Availability: Unix, Windows. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe2(flags) - - Create a pipe with *flags* set atomically. - *flags* can be constructed by ORing together one or more of these values: - :data:`O_NONBLOCK`, :data:`O_CLOEXEC`. - Return a pair of file descriptors ``(r, w)`` usable for reading and writing, - respectively. - - Availability: some flavors of Unix. - - .. versionadded:: 3.3 - - -.. function:: posix_fallocate(fd, offset, len) - - Ensures that enough disk space is allocated for the file specified by *fd* - starting from *offset* and continuing for *len* bytes. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: posix_fadvise(fd, offset, len, advice) - - Announces an intention to access data in a specific pattern thus allowing - the kernel to make optimizations. - The advice applies to the region of the file specified by *fd* starting at - *offset* and continuing for *len* bytes. - *advice* is one of :data:`POSIX_FADV_NORMAL`, :data:`POSIX_FADV_SEQUENTIAL`, - :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, - :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: POSIX_FADV_NORMAL - POSIX_FADV_SEQUENTIAL - POSIX_FADV_RANDOM - POSIX_FADV_NOREUSE - POSIX_FADV_WILLNEED - POSIX_FADV_DONTNEED - - Flags that can be used in *advice* in :func:`posix_fadvise` that specify - the access pattern that is likely to be used. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: pread(fd, buffersize, offset) - - Read from a file descriptor, *fd*, at a position of *offset*. It will read up - to *buffersize* number of bytes. The file offset remains unchanged. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: pwrite(fd, str, offset) - - Write *bytestring* to a file descriptor, *fd*, from *offset*, - leaving the file offset unchanged. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: read(fd, n) - - Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the - bytes read. If the end of the file referred to by *fd* has been reached, an - empty bytes object is returned. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To read a - "file object" returned by the built-in function :func:`open` or by - :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: sendfile(out, in, offset, count) - sendfile(out, in, offset, count, [headers], [trailers], flags=0) - - Copy *count* bytes from file descriptor *in* to file descriptor *out* - starting at *offset*. - Return the number of bytes sent. When EOF is reached return 0. - - The first function notation is supported by all platforms that define - :func:`sendfile`. - - On Linux, if *offset* is given as ``None``, the bytes are read from the - current position of *in* and the position of *in* is updated. - - The second case may be used on Mac OS X and FreeBSD where *headers* and - *trailers* are arbitrary sequences of buffers that are written before and - after the data from *in* is written. It returns the same as the first case. - - On Mac OS X and FreeBSD, a value of 0 for *count* specifies to send until - the end of *in* is reached. - - All platforms support sockets as *out* file descriptor, and some platforms - allow other types (e.g. regular file, pipe) as well. - - Cross-platform applications should not use *headers*, *trailers* and *flags* - arguments. - - Availability: Unix. - - .. note:: - - For a higher-level wrapper of :func:`sendfile`, see - :meth:`socket.socket.sendfile`. - - .. versionadded:: 3.3 - - -.. function:: set_blocking(fd, blocking) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - Availability: Unix. - - .. versionadded:: 3.5 - - -.. data:: SF_NODISKIO - SF_MNOWAIT - SF_SYNC - - Parameters to the :func:`sendfile` function, if the implementation supports - them. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: readv(fd, buffers) - - Read from a file descriptor *fd* into a number of mutable :term:`bytes-like - objects ` *buffers*. :func:`~os.readv` will transfer data - into each buffer until it is full and then move on to the next buffer in the - sequence to hold the rest of the data. :func:`~os.readv` returns the total - number of bytes read (which may be less than the total capacity of all the - objects). - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: tcgetpgrp(fd) - - Return the process group associated with the terminal given by *fd* (an open - file descriptor as returned by :func:`os.open`). - - Availability: Unix. - - -.. function:: tcsetpgrp(fd, pg) - - Set the process group associated with the terminal given by *fd* (an open file - descriptor as returned by :func:`os.open`) to *pg*. - - Availability: Unix. - - -.. function:: ttyname(fd) - - Return a string which specifies the terminal device associated with - file descriptor *fd*. If *fd* is not associated with a terminal device, an - exception is raised. - - Availability: Unix. - - -.. function:: write(fd, str) - - Write the bytestring in *str* to file descriptor *fd*. Return the number of - bytes actually written. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file - object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: writev(fd, buffers) - - Write the contents of *buffers* to file descriptor *fd*. *buffers* must be a - sequence of :term:`bytes-like objects `. Buffers are - processed in array order. Entire contents of first buffer is written before - proceeding to second, and so on. The operating system may set a limit - (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. - - :func:`~os.writev` writes the contents of each object to the file descriptor - and returns the total number of bytes written. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. _terminal-size: - -Querying the size of a terminal -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.3 - -.. function:: get_terminal_size(fd=STDOUT_FILENO) - - Return the size of the terminal window as ``(columns, lines)``, - tuple of type :class:`terminal_size`. - - The optional argument ``fd`` (default ``STDOUT_FILENO``, or standard - output) specifies which file descriptor should be queried. - - If the file descriptor is not connected to a terminal, an :exc:`OSError` - is raised. - - :func:`shutil.get_terminal_size` is the high-level function which - should normally be used, ``os.get_terminal_size`` is the low-level - implementation. - - Availability: Unix, Windows. - -.. class:: terminal_size - - A subclass of tuple, holding ``(columns, lines)`` of the terminal window size. - - .. attribute:: columns - - Width of the terminal window in characters. - - .. attribute:: lines - - Height of the terminal window in characters. - - -.. _fd_inheritance: - -Inheritance of File Descriptors -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.4 - -A file descriptor has an "inheritable" flag which indicates if the file descriptor -can be inherited by child processes. Since Python 3.4, file descriptors -created by Python are non-inheritable by default. - -On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. - -On Windows, non-inheritable handles and file descriptors are closed in child -processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout -and stderr), which are always inherited. Using :func:`spawn\* ` functions, -all inheritable handles and all inheritable file descriptors are inherited. -Using the :mod:`subprocess` module, all file descriptors except standard -streams are closed, and inheritable handles are only inherited if the -*close_fds* parameter is ``False``. - -.. function:: get_inheritable(fd) - - Get the "inheritable" flag of the specified file descriptor (a boolean). - -.. function:: set_inheritable(fd, inheritable) - - Set the "inheritable" flag of the specified file descriptor. - -.. function:: get_handle_inheritable(handle) - - Get the "inheritable" flag of the specified handle (a boolean). - - Availability: Windows. - -.. function:: set_handle_inheritable(handle, inheritable) - - Set the "inheritable" flag of the specified handle. - - Availability: Windows. - - -.. _os-file-dir: - -Files and Directories ---------------------- - -On some Unix platforms, many of these functions support one or more of these -features: - -.. _path_fd: - -* **specifying a file descriptor:** - For some functions, the *path* argument can be not only a string giving a path - name, but also a file descriptor. The function will then operate on the file - referred to by the descriptor. (For POSIX systems, Python will call the - ``f...`` version of the function.) - - You can check whether or not *path* can be specified as a file descriptor on - your platform using :data:`os.supports_fd`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. - - If the function also supports *dir_fd* or *follow_symlinks* arguments, it is - an error to specify one of those when supplying *path* as a file descriptor. - -.. _dir_fd: - -* **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it - should be a file descriptor referring to a directory, and the path to operate - on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call - the ``...at`` or ``f...at`` version of the function.) - - You can check whether or not *dir_fd* is supported on your platform using - :data:`os.supports_dir_fd`. If it is unavailable, using it will raise a - :exc:`NotImplementedError`. - -.. _follow_symlinks: - -* **not following symlinks:** If *follow_symlinks* is - ``False``, and the last element of the path to operate on is a symbolic link, - the function will operate on the symbolic link itself instead of the file the - link points to. (For POSIX systems, Python will call the ``l...`` version of - the function.) - - You can check whether or not *follow_symlinks* is supported on your platform - using :data:`os.supports_follow_symlinks`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. - - - -.. function:: access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True) - - Use the real uid/gid to test for access to *path*. Note that most operations - will use the effective uid/gid, therefore this routine can be used in a - suid/sgid environment to test if the invoking user has the specified access to - *path*. *mode* should be :const:`F_OK` to test the existence of *path*, or it - can be the inclusive OR of one or more of :const:`R_OK`, :const:`W_OK`, and - :const:`X_OK` to test permissions. Return :const:`True` if access is allowed, - :const:`False` if not. See the Unix man page :manpage:`access(2)` for more - information. - - This function can support specifying :ref:`paths relative to directory - descriptors ` and :ref:`not following symlinks `. - - If *effective_ids* is ``True``, :func:`access` will perform its access - checks using the effective uid/gid instead of the real uid/gid. - *effective_ids* may not be supported on your platform; you can check whether - or not it is available using :data:`os.supports_effective_ids`. If it is - unavailable, using it will raise a :exc:`NotImplementedError`. - - .. note:: - - Using :func:`access` to check if a user is authorized to e.g. open a file - before actually doing so using :func:`open` creates a security hole, - because the user might exploit the short time interval between checking - and opening the file to manipulate it. It's preferable to use :term:`EAFP` - techniques. For example:: - - if os.access("myfile", os.R_OK): - with open("myfile") as fp: - return fp.read() - return "some default data" - - is better written as:: - - try: - fp = open("myfile") - except PermissionError: - return "some default data" - else: - with fp: - return fp.read() - - .. note:: - - I/O operations may fail even when :func:`access` indicates that they would - succeed, particularly for operations on network filesystems which may have - permissions semantics beyond the usual POSIX permission-bit model. - - .. versionchanged:: 3.3 - Added the *dir_fd*, *effective_ids*, and *follow_symlinks* parameters. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: F_OK - R_OK - W_OK - X_OK - - Values to pass as the *mode* parameter of :func:`access` to test the - existence, readability, writability and executability of *path*, - respectively. - - -.. function:: chdir(path) - - .. index:: single: directory; changing - - Change the current working directory to *path*. - - This function can support :ref:`specifying a file descriptor `. The - descriptor must refer to an opened directory, not an open file. - - .. versionadded:: 3.3 - Added support for specifying *path* as a file descriptor - on some platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chflags(path, flags, *, follow_symlinks=True) - - Set the flags of *path* to the numeric *flags*. *flags* may take a combination - (bitwise OR) of the following values (as defined in the :mod:`stat` module): - - * :data:`stat.UF_NODUMP` - * :data:`stat.UF_IMMUTABLE` - * :data:`stat.UF_APPEND` - * :data:`stat.UF_OPAQUE` - * :data:`stat.UF_NOUNLINK` - * :data:`stat.UF_COMPRESSED` - * :data:`stat.UF_HIDDEN` - * :data:`stat.SF_ARCHIVED` - * :data:`stat.SF_IMMUTABLE` - * :data:`stat.SF_APPEND` - * :data:`stat.SF_NOUNLINK` - * :data:`stat.SF_SNAPSHOT` - - This function can support :ref:`not following symlinks `. - - Availability: Unix. - - .. versionadded:: 3.3 - The *follow_symlinks* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chmod(path, mode, *, dir_fd=None, follow_symlinks=True) - - Change the mode of *path* to the numeric *mode*. *mode* may take one of the - following values (as defined in the :mod:`stat` module) or bitwise ORed - combinations of them: - - * :data:`stat.S_ISUID` - * :data:`stat.S_ISGID` - * :data:`stat.S_ENFMT` - * :data:`stat.S_ISVTX` - * :data:`stat.S_IREAD` - * :data:`stat.S_IWRITE` - * :data:`stat.S_IEXEC` - * :data:`stat.S_IRWXU` - * :data:`stat.S_IRUSR` - * :data:`stat.S_IWUSR` - * :data:`stat.S_IXUSR` - * :data:`stat.S_IRWXG` - * :data:`stat.S_IRGRP` - * :data:`stat.S_IWGRP` - * :data:`stat.S_IXGRP` - * :data:`stat.S_IRWXO` - * :data:`stat.S_IROTH` - * :data:`stat.S_IWOTH` - * :data:`stat.S_IXOTH` - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - .. note:: - - Although Windows supports :func:`chmod`, you can only set the file's - read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` - constants or a corresponding integer value). All other bits are ignored. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor, - and the *dir_fd* and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. To - leave one of the ids unchanged, set it to -1. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - See :func:`shutil.chown` for a higher-level function that accepts names in - addition to numeric ids. - - Availability: Unix. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, - and the *dir_fd* and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Supports a :term:`path-like object`. - - -.. function:: chroot(path) - - Change the root directory of the current process to *path*. - - Availability: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: fchdir(fd) - - Change the current working directory to the directory represented by the file - descriptor *fd*. The descriptor must refer to an opened directory, not an - open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. - - Availability: Unix. - - -.. function:: getcwd() - - Return a string representing the current working directory. - - -.. function:: getcwdb() - - Return a bytestring representing the current working directory. - - -.. function:: lchflags(path, flags) - - Set the flags of *path* to the numeric *flags*, like :func:`chflags`, but do - not follow symbolic links. As of Python 3.3, this is equivalent to - ``os.chflags(path, flags, follow_symlinks=False)``. - - Availability: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lchmod(path, mode) - - Change the mode of *path* to the numeric *mode*. If path is a symlink, this - affects the symlink rather than the target. See the docs for :func:`chmod` - for possible values of *mode*. As of Python 3.3, this is equivalent to - ``os.chmod(path, mode, follow_symlinks=False)``. - - Availability: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: lchown(path, uid, gid) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. This - function will not follow symbolic links. As of Python 3.3, this is equivalent - to ``os.chown(path, uid, gid, follow_symlinks=False)``. - - Availability: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: link(src, dst, *, src_dir_fd=None, dst_dir_fd=None, follow_symlinks=True) - - Create a hard link pointing to *src* named *dst*. - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `, and :ref:`not - following symlinks `. - - Availability: Unix, Windows. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionadded:: 3.3 - Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: listdir(path='.') - - Return a list containing the names of the entries in the directory given by - *path*. The list is in arbitrary order, and does not include the special - entries ``'.'`` and ``'..'`` even if they are present in the directory. - - *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` - (directly or indirectly through the :class:`PathLike` interface), - the filenames returned will also be of type ``bytes``; - in all other circumstances, they will be of type ``str``. - - This function can also support :ref:`specifying a file descriptor - `; the file descriptor must refer to a directory. - - .. note:: - To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. - - .. seealso:: - - The :func:`scandir` function returns directory entries along with - file attribute information, giving better performance for many - common use cases. - - .. versionchanged:: 3.2 - The *path* parameter became optional. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lstat(path, \*, dir_fd=None) - - Perform the equivalent of an :c:func:`lstat` system call on the given path. - Similar to :func:`~os.stat`, but does not follow symbolic links. Return a - :class:`stat_result` object. - - On platforms that do not support symbolic links, this is an alias for - :func:`~os.stat`. - - As of Python 3.3, this is equivalent to ``os.stat(path, dir_fd=dir_fd, - follow_symlinks=False)``. - - This function can also support :ref:`paths relative to directory descriptors - `. - - .. seealso:: - - The :func:`.stat` function. - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionchanged:: 3.3 - Added the *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: mkdir(path, mode=0o777, *, dir_fd=None) - - Create a directory named *path* with numeric mode *mode*. - - If the directory already exists, :exc:`FileExistsError` is raised. - - .. _mkdir_modebits: - - On some systems, *mode* is ignored. Where it is used, the current umask - value is first masked out. If bits other than the last 9 (i.e. the last 3 - digits of the octal representation of the *mode*) are set, their meaning is - platform-dependent. On some platforms, they are ignored and you should call - :func:`chmod` explicitly to set them. - - This function can also support :ref:`paths relative to directory descriptors - `. - - It is also possible to create temporary directories; see the - :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: makedirs(name, mode=0o777, exist_ok=False) - - .. index:: - single: directory; creating - single: UNC paths; and os.makedirs() - - Recursive directory creation function. Like :func:`mkdir`, but makes all - intermediate-level directories needed to contain the leaf directory. - - The *mode* parameter is passed to :func:`mkdir`; see :ref:`the mkdir() - description ` for how it is interpreted. - - If *exist_ok* is ``False`` (the default), an :exc:`OSError` is raised if the - target directory already exists. - - .. note:: - - :func:`makedirs` will become confused if the path elements to create - include :data:`pardir` (eg. ".." on UNIX systems). - - This function handles UNC paths correctly. - - .. versionadded:: 3.2 - The *exist_ok* parameter. - - .. versionchanged:: 3.4.1 - - Before Python 3.4.1, if *exist_ok* was ``True`` and the directory existed, - :func:`makedirs` would still raise an error if *mode* did not match the - mode of the existing directory. Since this behavior was impossible to - implement safely, it was removed in Python 3.4.1. See :issue:`21082`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: mkfifo(path, mode=0o666, *, dir_fd=None) - - Create a FIFO (a named pipe) named *path* with numeric mode *mode*. - The current umask value is first masked out from the mode. - - This function can also support :ref:`paths relative to directory descriptors - `. - - FIFOs are pipes that can be accessed like regular files. FIFOs exist until they - are deleted (for example with :func:`os.unlink`). Generally, FIFOs are used as - rendezvous between "client" and "server" type processes: the server opens the - FIFO for reading, and the client opens it for writing. Note that :func:`mkfifo` - doesn't open the FIFO --- it just creates the rendezvous point. - - Availability: Unix. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: mknod(path, mode=0o600, device=0, *, dir_fd=None) - - Create a filesystem node (file, device special file or named pipe) named - *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using - :func:`os.makedev`), otherwise it is ignored. - - This function can also support :ref:`paths relative to directory descriptors - `. - - Availability: Unix. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: major(device) - - Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:type:`stat`). - - -.. function:: minor(device) - - Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:type:`stat`). - - -.. function:: makedev(major, minor) - - Compose a raw device number from the major and minor device numbers. - - -.. function:: pathconf(path, name) - - Return system configuration information relevant to a named file. *name* - specifies the configuration value to retrieve; it may be a string which is the - name of a defined system value; these names are specified in a number of - standards (POSIX.1, Unix 95, Unix 98, and others). Some platforms define - additional names as well. The names known to the host operating system are - given in the ``pathconf_names`` dictionary. For configuration variables not - included in that mapping, passing an integer for *name* is also accepted. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``pathconf_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - This function can support :ref:`specifying a file descriptor - `. - - Availability: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: pathconf_names - - Dictionary mapping names accepted by :func:`pathconf` and :func:`fpathconf` to - the integer values defined for those names by the host operating system. This - can be used to determine the set of names known to the system. - - Availability: Unix. - - -.. function:: readlink(path, *, dir_fd=None) - - Return a string representing the path to which the symbolic link points. The - result may be either an absolute or relative pathname; if it is relative, it - may be converted to an absolute pathname using - ``os.path.join(os.path.dirname(path), result)``. - - If the *path* is a string object (directly or indirectly through a - :class:`PathLike` interface), the result will also be a string object, - and the call may raise a UnicodeDecodeError. If the *path* is a bytes - object (direct or indirectly), the result will be a bytes object. - - This function can also support :ref:`paths relative to directory descriptors - `. - - Availability: Unix, Windows - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: remove(path, *, dir_fd=None) - - Remove (delete) the file *path*. If *path* is a directory, :exc:`OSError` is - raised. Use :func:`rmdir` to remove directories. - - This function can support :ref:`paths relative to directory descriptors - `. - - On Windows, attempting to remove a file that is in use causes an exception to - be raised; on Unix, the directory entry is removed but the storage allocated - to the file is not made available until the original file is no longer in use. - - This function is semantically identical to :func:`unlink`. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: removedirs(name) - - .. index:: single: directory; deleting - - Remove directories recursively. Works like :func:`rmdir` except that, if the - leaf directory is successfully removed, :func:`removedirs` tries to - successively remove every parent directory mentioned in *path* until an error - is raised (which is ignored, because it generally means that a parent directory - is not empty). For example, ``os.removedirs('foo/bar/baz')`` will first remove - the directory ``'foo/bar/baz'``, and then remove ``'foo/bar'`` and ``'foo'`` if - they are empty. Raises :exc:`OSError` if the leaf directory could not be - successfully removed. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - - Rename the file or directory *src* to *dst*. If *dst* is a directory, - :exc:`OSError` will be raised. On Unix, if *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail on some - Unix flavors if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). On - Windows, if *dst* already exists, :exc:`OSError` will be raised even if it is a - file. - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. - - If you want cross-platform overwriting of the destination, use :func:`replace`. - - .. versionadded:: 3.3 - The *src_dir_fd* and *dst_dir_fd* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: renames(old, new) - - Recursive directory or file renaming function. Works like :func:`rename`, except - creation of any intermediate directories needed to make the new pathname good is - attempted first. After the rename, directories corresponding to rightmost path - segments of the old name will be pruned away using :func:`removedirs`. - - .. note:: - - This function can fail with the new directory structure made if you lack - permissions needed to remove the leaf directory or file. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *old* and *new*. - - -.. function:: replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - - Rename the file or directory *src* to *dst*. If *dst* is a directory, - :exc:`OSError` will be raised. If *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail - if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: rmdir(path, *, dir_fd=None) - - Remove (delete) the directory *path*. Only works when the directory is - empty, otherwise, :exc:`OSError` is raised. In order to remove whole - directory trees, :func:`shutil.rmtree` can be used. - - This function can support :ref:`paths relative to directory descriptors - `. - - .. versionadded:: 3.3 - The *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: scandir(path='.') - - Return an iterator of :class:`os.DirEntry` objects corresponding to the - entries in the directory given by *path*. The entries are yielded in - arbitrary order, and the special entries ``'.'`` and ``'..'`` are not - included. - - Using :func:`scandir` instead of :func:`listdir` can significantly - increase the performance of code that also needs file type or file - attribute information, because :class:`os.DirEntry` objects expose this - information if the operating system provides it when scanning a directory. - All :class:`os.DirEntry` methods may perform a system call, but - :func:`~os.DirEntry.is_dir` and :func:`~os.DirEntry.is_file` usually only - require a system call for symbolic links; :func:`os.DirEntry.stat` - always requires a system call on Unix but only requires one for - symbolic links on Windows. - - *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` - (directly or indirectly through the :class:`PathLike` interface), - the type of the :attr:`~os.DirEntry.name` and :attr:`~os.DirEntry.path` - attributes of each :class:`os.DirEntry` will be ``bytes``; in all other - circumstances, they will be of type ``str``. - - The :func:`scandir` iterator supports the :term:`context manager` protocol - and has the following method: - - .. method:: scandir.close() - - Close the iterator and free acquired resources. - - This is called automatically when the iterator is exhausted or garbage - collected, or when an error happens during iterating. However it - is advisable to call it explicitly or use the :keyword:`with` - statement. - - .. versionadded:: 3.6 - - The following example shows a simple use of :func:`scandir` to display all - the files (excluding directories) in the given *path* that don't start with - ``'.'``. The ``entry.is_file()`` call will generally not make an additional - system call:: - - with os.scandir(path) as it: - for entry in it: - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) - - .. note:: - - On Unix-based systems, :func:`scandir` uses the system's - `opendir() `_ - and - `readdir() `_ - functions. On Windows, it uses the Win32 - `FindFirstFileW `_ - and - `FindNextFileW `_ - functions. - - .. versionadded:: 3.5 - - .. versionadded:: 3.6 - Added support for the :term:`context manager` protocol and the - :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither - exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted - in its destructor. - - The function accepts a :term:`path-like object`. - - -.. class:: DirEntry - - Object yielded by :func:`scandir` to expose the file path and other file - attributes of a directory entry. - - :func:`scandir` will provide as much of this information as possible without - making additional system calls. When a ``stat()`` or ``lstat()`` system call - is made, the ``os.DirEntry`` object will cache the result. - - ``os.DirEntry`` instances are not intended to be stored in long-lived data - structures; if you know the file metadata has changed or if a long time has - elapsed since calling :func:`scandir`, call ``os.stat(entry.path)`` to fetch - up-to-date information. - - Because the ``os.DirEntry`` methods can make operating system calls, they may - also raise :exc:`OSError`. If you need very fine-grained - control over errors, you can catch :exc:`OSError` when calling one of the - ``os.DirEntry`` methods and handle as appropriate. - - To be directly usable as a :term:`path-like object`, ``os.DirEntry`` - implements the :class:`PathLike` interface. - - Attributes and methods on a ``os.DirEntry`` instance are as follows: - - .. attribute:: name - - The entry's base filename, relative to the :func:`scandir` *path* - argument. - - The :attr:`name` attribute will be ``bytes`` if the :func:`scandir` - *path* argument is of type ``bytes`` and ``str`` otherwise. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. attribute:: path - - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. - - The :attr:`path` attribute will be ``bytes`` if the :func:`scandir` - *path* argument is of type ``bytes`` and ``str`` otherwise. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. method:: inode() - - Return the inode number of the entry. - - The result is cached on the ``os.DirEntry`` object. Use - ``os.stat(entry.path, follow_symlinks=False).st_ino`` to fetch up-to-date - information. - - On the first, uncached call, a system call is required on Windows but - not on Unix. - - .. method:: is_dir(\*, follow_symlinks=True) - - Return ``True`` if this entry is a directory or a symbolic link pointing - to a directory; return ``False`` if the entry is or points to any other - kind of file, or if it doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a directory (without following symlinks); return ``False`` if the - entry is any other kind of file or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object, with a separate cache - for *follow_symlinks* ``True`` and ``False``. Call :func:`os.stat` along - with :func:`stat.S_ISDIR` to fetch up-to-date information. - - On the first, uncached call, no system call is required in most cases. - Specifically, for non-symlinks, neither Windows or Unix require a system - call, except on certain Unix file systems, such as network file systems, - that return ``dirent.d_type == DT_UNKNOWN``. If the entry is a symlink, - a system call will be required to follow the symlink unless - *follow_symlinks* is ``False``. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - .. method:: is_file(\*, follow_symlinks=True) - - Return ``True`` if this entry is a file or a symbolic link pointing to a - file; return ``False`` if the entry is or points to a directory or other - non-file entry, or if it doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a file (without following symlinks); return ``False`` if the entry is - a directory or other non-file entry, or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object. Caching, system calls - made, and exceptions raised are as per :func:`~os.DirEntry.is_dir`. - - .. method:: is_symlink() - - Return ``True`` if this entry is a symbolic link (even if broken); - return ``False`` if the entry points to a directory or any kind of file, - or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object. Call - :func:`os.path.islink` to fetch up-to-date information. - - On the first, uncached call, no system call is required in most cases. - Specifically, neither Windows or Unix require a system call, except on - certain Unix file systems, such as network file systems, that return - ``dirent.d_type == DT_UNKNOWN``. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - .. method:: stat(\*, follow_symlinks=True) - - Return a :class:`stat_result` object for this entry. This method - follows symbolic links by default; to stat a symbolic link add the - ``follow_symlinks=False`` argument. - - On Unix, this method always requires a system call. On Windows, it - only requires a system call if *follow_symlinks* is ``True`` and the - entry is a symbolic link. - - On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the - :class:`stat_result` are always set to zero. Call :func:`os.stat` to - get these attributes. - - The result is cached on the ``os.DirEntry`` object, with a separate cache - for *follow_symlinks* ``True`` and ``False``. Call :func:`os.stat` to - fetch up-to-date information. - - Note that there is a nice correspondence between several attributes - and methods of ``os.DirEntry`` and of :class:`pathlib.Path`. In - particular, the ``name`` attribute has the same - meaning, as do the ``is_dir()``, ``is_file()``, ``is_symlink()`` - and ``stat()`` methods. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - Added support for the :class:`~os.PathLike` interface. Added support - for :class:`bytes` paths on Windows. - - -.. function:: stat(path, \*, dir_fd=None, follow_symlinks=True) - - Get the status of a file or a file descriptor. Perform the equivalent of a - :c:func:`stat` system call on the given path. *path* may be specified as - either a string or bytes -- directly or indirectly through the :class:`PathLike` - interface -- or as an open file descriptor. Return a :class:`stat_result` - object. - - This function normally follows symlinks; to stat a symlink add the argument - ``follow_symlinks=False``, or use :func:`lstat`. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. index:: module: stat - - Example:: - - >>> import os - >>> statinfo = os.stat('somefile.txt') - >>> statinfo - os.stat_result(st_mode=33188, st_ino=7876932, st_dev=234881026, - st_nlink=1, st_uid=501, st_gid=501, st_size=264, st_atime=1297230295, - st_mtime=1297230027, st_ctime=1297230027) - >>> statinfo.st_size - 264 - - .. seealso:: - - :func:`fstat` and :func:`lstat` functions. - - .. versionadded:: 3.3 - Added the *dir_fd* and *follow_symlinks* arguments, specifying a file - descriptor instead of a path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. class:: stat_result - - Object whose attributes correspond roughly to the members of the - :c:type:`stat` structure. It is used for the result of :func:`os.stat`, - :func:`os.fstat` and :func:`os.lstat`. - - Attributes: - - .. attribute:: st_mode - - File mode: file type and file mode bits (permissions). - - .. attribute:: st_ino - - Platform dependent, but if non-zero, uniquely identifies the - file for a given value of ``st_dev``. Typically: - - * the inode number on Unix, - * the `file index - `_ on - Windows - - .. attribute:: st_dev - - Identifier of the device on which this file resides. - - .. attribute:: st_nlink - - Number of hard links. - - .. attribute:: st_uid - - User identifier of the file owner. - - .. attribute:: st_gid - - Group identifier of the file owner. - - .. attribute:: st_size - - Size of the file in bytes, if it is a regular file or a symbolic link. - The size of a symbolic link is the length of the pathname it contains, - without a terminating null byte. - - Timestamps: - - .. attribute:: st_atime - - Time of most recent access expressed in seconds. - - .. attribute:: st_mtime - - Time of most recent content modification expressed in seconds. - - .. attribute:: st_ctime - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in seconds. - - .. attribute:: st_atime_ns - - Time of most recent access expressed in nanoseconds as an integer. - - .. attribute:: st_mtime_ns - - Time of most recent content modification expressed in nanoseconds as an - integer. - - .. attribute:: st_ctime_ns - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in nanoseconds as an - integer. - - See also the :func:`stat_float_times` function. - - .. note:: - - The exact meaning and resolution of the :attr:`st_atime`, - :attr:`st_mtime`, and :attr:`st_ctime` attributes depend on the operating - system and the file system. For example, on Windows systems using the FAT - or FAT32 file systems, :attr:`st_mtime` has 2-second resolution, and - :attr:`st_atime` has only 1-day resolution. See your operating system - documentation for details. - - Similarly, although :attr:`st_atime_ns`, :attr:`st_mtime_ns`, - and :attr:`st_ctime_ns` are always expressed in nanoseconds, many - systems do not provide nanosecond precision. On systems that do - provide nanosecond precision, the floating-point object used to - store :attr:`st_atime`, :attr:`st_mtime`, and :attr:`st_ctime` - cannot preserve all of it, and as such will be slightly inexact. - If you need the exact timestamps you should always use - :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and :attr:`st_ctime_ns`. - - On some Unix systems (such as Linux), the following attributes may also be - available: - - .. attribute:: st_blocks - - Number of 512-byte blocks allocated for file. - This may be smaller than :attr:`st_size`/512 when the file has holes. - - .. attribute:: st_blksize - - "Preferred" blocksize for efficient file system I/O. Writing to a file in - smaller chunks may cause an inefficient read-modify-rewrite. - - .. attribute:: st_rdev - - Type of device if an inode device. - - .. attribute:: st_flags - - User defined flags for file. - - On other Unix systems (such as FreeBSD), the following attributes may be - available (but may be only filled out if root tries to use them): - - .. attribute:: st_gen - - File generation number. - - .. attribute:: st_birthtime - - Time of file creation. - - On Mac OS systems, the following attributes may also be available: - - .. attribute:: st_rsize - - Real size of the file. - - .. attribute:: st_creator - - Creator of the file. - - .. attribute:: st_type - - File type. - - On Windows systems, the following attribute is also available: - - .. attribute:: st_file_attributes - - Windows file attributes: ``dwFileAttributes`` member of the - ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` - constants in the :mod:`stat` module. - - The standard module :mod:`stat` defines functions and constants that are - useful for extracting information from a :c:type:`stat` structure. (On - Windows, some items are filled with dummy values.) - - For backward compatibility, a :class:`stat_result` instance is also - accessible as a tuple of at least 10 integers giving the most important (and - portable) members of the :c:type:`stat` structure, in the order - :attr:`st_mode`, :attr:`st_ino`, :attr:`st_dev`, :attr:`st_nlink`, - :attr:`st_uid`, :attr:`st_gid`, :attr:`st_size`, :attr:`st_atime`, - :attr:`st_mtime`, :attr:`st_ctime`. More items may be added at the end by - some implementations. For compatibility with older Python versions, - accessing :class:`stat_result` as a tuple always returns integers. - - .. versionadded:: 3.3 - Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and - :attr:`st_ctime_ns` members. - - .. versionadded:: 3.5 - Added the :attr:`st_file_attributes` member on Windows. - - .. versionchanged:: 3.5 - Windows now returns the file index as :attr:`st_ino` when - available. - - -.. function:: stat_float_times([newvalue]) - - Determine whether :class:`stat_result` represents time stamps as float objects. - If *newvalue* is ``True``, future calls to :func:`~os.stat` return floats, if it is - ``False``, future calls return ints. If *newvalue* is omitted, return the - current setting. - - For compatibility with older Python versions, accessing :class:`stat_result` as - a tuple always returns integers. - - Python now returns float values by default. Applications which do not work - correctly with floating point time stamps can use this function to restore the - old behaviour. - - The resolution of the timestamps (that is the smallest possible fraction) - depends on the system. Some systems only support second resolution; on these - systems, the fraction will always be zero. - - It is recommended that this setting is only changed at program startup time in - the *__main__* module; libraries should never change this setting. If an - application uses a library that works incorrectly if floating point time stamps - are processed, this application should turn the feature off until the library - has been corrected. - - .. deprecated:: 3.3 - - -.. function:: statvfs(path) - - Perform a :c:func:`statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:type:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). - - This function can support :ref:`specifying a file descriptor `. - - Availability: Unix. - - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. - - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: supports_dir_fd - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *dir_fd* parameter. Different platforms - provide different functionality, and an option that might work on one might - be unsupported on another. For consistency's sakes, functions that support - *dir_fd* always allow specifying the parameter, but will raise an exception - if the functionality is not actually available. - - To check whether a particular function permits use of its *dir_fd* - parameter, use the ``in`` operator on ``supports_dir_fd``. As an example, - this expression determines whether the *dir_fd* parameter of :func:`os.stat` - is locally available:: - - os.stat in os.supports_dir_fd - - Currently *dir_fd* parameters only work on Unix platforms; none of them work - on Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_effective_ids - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of the *effective_ids* parameter for - :func:`os.access`. If the local platform supports it, the collection will - contain :func:`os.access`, otherwise it will be empty. - - To check whether you can use the *effective_ids* parameter for - :func:`os.access`, use the ``in`` operator on ``supports_effective_ids``, - like so:: - - os.access in os.supports_effective_ids - - Currently *effective_ids* only works on Unix platforms; it does not work on - Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_fd - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file - descriptor. Different platforms provide different functionality, and an - option that might work on one might be unsupported on another. For - consistency's sakes, functions that support *fd* always allow specifying - the parameter, but will raise an exception if the functionality is not - actually available. - - To check whether a particular function permits specifying an open file - descriptor for its *path* parameter, use the ``in`` operator on - ``supports_fd``. As an example, this expression determines whether - :func:`os.chdir` accepts open file descriptors when called on your local - platform:: - - os.chdir in os.supports_fd - - .. versionadded:: 3.3 - - -.. data:: supports_follow_symlinks - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *follow_symlinks* parameter. Different - platforms provide different functionality, and an option that might work on - one might be unsupported on another. For consistency's sakes, functions that - support *follow_symlinks* always allow specifying the parameter, but will - raise an exception if the functionality is not actually available. - - To check whether a particular function permits use of its *follow_symlinks* - parameter, use the ``in`` operator on ``supports_follow_symlinks``. As an - example, this expression determines whether the *follow_symlinks* parameter - of :func:`os.stat` is locally available:: - - os.stat in os.supports_follow_symlinks - - .. versionadded:: 3.3 - - -.. function:: symlink(src, dst, target_is_directory=False, *, dir_fd=None) - - Create a symbolic link pointing to *src* named *dst*. - - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. - - Symbolic link support was introduced in Windows 6.0 (Vista). :func:`symlink` - will raise a :exc:`NotImplementedError` on Windows versions earlier than 6.0. - - This function can support :ref:`paths relative to directory descriptors - `. - - .. note:: - - On Windows, the *SeCreateSymbolicLinkPrivilege* is required in order to - successfully create symlinks. This privilege is not typically granted to - regular users but is available to accounts which can escalate privileges - to the administrator level. Either obtaining the privilege or running your - application as an administrator are ways to successfully create symlinks. - - - :exc:`OSError` is raised when the function is called by an unprivileged - user. - - Availability: Unix, Windows. - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionadded:: 3.3 - Added the *dir_fd* argument, and now allow *target_is_directory* - on non-Windows platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: sync() - - Force write of everything to disk. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: truncate(path, length) - - Truncate the file corresponding to *path*, so that it is at most - *length* bytes in size. - - This function can support :ref:`specifying a file descriptor `. - - Availability: Unix, Windows. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - Added support for Windows - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: unlink(path, *, dir_fd=None) - - Remove (delete) the file *path*. This function is semantically - identical to :func:`remove`; the ``unlink`` name is its - traditional Unix name. Please see the documentation for - :func:`remove` for further information. - - .. versionadded:: 3.3 - The *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True) - - Set the access and modified times of the file specified by *path*. - - :func:`utime` takes two optional parameters, *times* and *ns*. - These specify the times set on *path* and are used as follows: - - - If *ns* is specified, - it must be a 2-tuple of the form ``(atime_ns, mtime_ns)`` - where each member is an int expressing nanoseconds. - - If *times* is not ``None``, - it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. - - If *times* is ``None`` and *ns* is unspecified, - this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` - where both times are the current time. - - It is an error to specify tuples for both *times* and *ns*. - - Whether a directory can be given for *path* - depends on whether the operating system implements directories as files - (for example, Windows does not). Note that the exact times you set here may - not be returned by a subsequent :func:`~os.stat` call, depending on the - resolution with which your operating system records access and modification - times; see :func:`~os.stat`. The best way to preserve exact times is to - use the *st_atime_ns* and *st_mtime_ns* fields from the :func:`os.stat` - result object with the *ns* parameter to `utime`. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, - and the *dir_fd*, *follow_symlinks*, and *ns* parameters. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: walk(top, topdown=True, onerror=None, followlinks=False) - - .. index:: - single: directory; walking - single: directory; traversal - - Generate the file names in a directory tree by walking the tree - either top-down or bottom-up. For each directory in the tree rooted at directory - *top* (including *top* itself), it yields a 3-tuple ``(dirpath, dirnames, - filenames)``. - - *dirpath* is a string, the path to the directory. *dirnames* is a list of the - names of the subdirectories in *dirpath* (excluding ``'.'`` and ``'..'``). - *filenames* is a list of the names of the non-directory files in *dirpath*. - Note that the names in the lists contain no path components. To get a full path - (which begins with *top*) to a file or directory in *dirpath*, do - ``os.path.join(dirpath, name)``. - - If optional argument *topdown* is ``True`` or not specified, the triple for a - directory is generated before the triples for any of its subdirectories - (directories are generated top-down). If *topdown* is ``False``, the triple - for a directory is generated after the triples for all of its subdirectories - (directories are generated bottom-up). No matter the value of *topdown*, the - list of subdirectories is retrieved before the tuples for the directory and - its subdirectories are generated. - - When *topdown* is ``True``, the caller can modify the *dirnames* list in-place - (perhaps using :keyword:`del` or slice assignment), and :func:`walk` will only - recurse into the subdirectories whose names remain in *dirnames*; this can be - used to prune the search, impose a specific order of visiting, or even to inform - :func:`walk` about directories the caller creates or renames before it resumes - :func:`walk` again. Modifying *dirnames* when *topdown* is ``False`` has - no effect on the behavior of the walk, because in bottom-up mode the directories - in *dirnames* are generated before *dirpath* itself is generated. - - By default, errors from the :func:`scandir` call are ignored. If optional - argument *onerror* is specified, it should be a function; it will be called with - one argument, an :exc:`OSError` instance. It can report the error to continue - with the walk, or raise the exception to abort the walk. Note that the filename - is available as the ``filename`` attribute of the exception object. - - By default, :func:`walk` will not walk down into symbolic links that resolve to - directories. Set *followlinks* to ``True`` to visit directories pointed to by - symlinks, on systems that support them. - - .. note:: - - Be aware that setting *followlinks* to ``True`` can lead to infinite - recursion if a link points to a parent directory of itself. :func:`walk` - does not keep track of the directories it visited already. - - .. note:: - - If you pass a relative pathname, don't change the current working directory - between resumptions of :func:`walk`. :func:`walk` never changes the current - directory, and assumes that its caller doesn't either. - - This example displays the number of bytes taken by non-directory files in each - directory under the starting directory, except that it doesn't look under any - CVS subdirectory:: - - import os - from os.path import join, getsize - for root, dirs, files in os.walk('python/Lib/email'): - print(root, "consumes", end=" ") - print(sum(getsize(join(root, name)) for name in files), end=" ") - print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories - - In the next example (simple implementation of :func:`shutil.rmtree`), - walking the tree bottom-up is essential, :func:`rmdir` doesn't allow - deleting a directory before the directory is empty:: - - # Delete everything reachable from the directory named in "top", - # assuming there are no symbolic links. - # CAUTION: This is dangerous! For example, if top == '/', it - # could delete all your disk files. - import os - for root, dirs, files in os.walk(top, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - - .. versionchanged:: 3.5 - This function now calls :func:`os.scandir` instead of :func:`os.listdir`, - making it faster by reducing the number of calls to :func:`os.stat`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: fwalk(top='.', topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None) - - .. index:: - single: directory; walking - single: directory; traversal - - This behaves exactly like :func:`walk`, except that it yields a 4-tuple - ``(dirpath, dirnames, filenames, dirfd)``, and it supports ``dir_fd``. - - *dirpath*, *dirnames* and *filenames* are identical to :func:`walk` output, - and *dirfd* is a file descriptor referring to the directory *dirpath*. - - This function always supports :ref:`paths relative to directory descriptors - ` and :ref:`not following symlinks `. Note however - that, unlike other functions, the :func:`fwalk` default value for - *follow_symlinks* is ``False``. - - .. note:: - - Since :func:`fwalk` yields file descriptors, those are only valid until - the next iteration step, so you should duplicate them (e.g. with - :func:`dup`) if you want to keep them longer. - - This example displays the number of bytes taken by non-directory files in each - directory under the starting directory, except that it doesn't look under any - CVS subdirectory:: - - import os - for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): - print(root, "consumes", end="") - print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), - end="") - print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories - - In the next example, walking the tree bottom-up is essential: - :func:`rmdir` doesn't allow deleting a directory before the directory is - empty:: - - # Delete everything reachable from the directory named in "top", - # assuming there are no symbolic links. - # CAUTION: This is dangerous! For example, if top == '/', it - # could delete all your disk files. - import os - for root, dirs, files, rootfd in os.fwalk(top, topdown=False): - for name in files: - os.unlink(name, dir_fd=rootfd) - for name in dirs: - os.rmdir(name, dir_fd=rootfd) - - Availability: Unix. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -Linux extended attributes -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.3 - -These functions are all available on Linux only. - -.. function:: getxattr(path, attribute, *, follow_symlinks=True) - - Return the value of the extended filesystem attribute *attribute* for - *path*. *attribute* can be bytes or str (directly or indirectly through the - :class:`PathLike` interface). If it is str, it is encoded with the filesystem - encoding. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. function:: listxattr(path=None, *, follow_symlinks=True) - - Return a list of the extended filesystem attributes on *path*. The - attributes in the list are represented as strings decoded with the filesystem - encoding. If *path* is ``None``, :func:`listxattr` will examine the current - directory. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: removexattr(path, attribute, *, follow_symlinks=True) - - Removes the extended filesystem attribute *attribute* from *path*. - *attribute* should be bytes or str (directly or indirectly through the - :class:`PathLike` interface). If it is a string, it is encoded - with the filesystem encoding. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. function:: setxattr(path, attribute, value, flags=0, *, follow_symlinks=True) - - Set the extended filesystem attribute *attribute* on *path* to *value*. - *attribute* must be a bytes or str with no embedded NULs (directly or - indirectly through the :class:`PathLike` interface). If it is a str, - it is encoded with the filesystem encoding. *flags* may be - :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is - given and the attribute does not exist, ``EEXISTS`` will be raised. - If :data:`XATTR_CREATE` is given and the attribute already exists, the - attribute will not be created and ``ENODATA`` will be raised. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. note:: - - A bug in Linux kernel versions less than 2.6.39 caused the flags argument - to be ignored on some filesystems. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. data:: XATTR_SIZE_MAX - - The maximum size the value of an extended attribute can be. Currently, this - is 64 KiB on Linux. - - -.. data:: XATTR_CREATE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must create an attribute. - - -.. data:: XATTR_REPLACE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must replace an existing attribute. - - -.. _os-process: - -Process Management ------------------- - -These functions may be used to create and manage processes. - -The various :func:`exec\* ` functions take a list of arguments for the new -program loaded into the process. In each case, the first of these arguments is -passed to the new program as its own name rather than as an argument a user may -have typed on a command line. For the C programmer, this is the ``argv[0]`` -passed to a program's :c:func:`main`. For example, ``os.execv('/bin/echo', -['foo', 'bar'])`` will only print ``bar`` on standard output; ``foo`` will seem -to be ignored. - - -.. function:: abort() - - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default - behavior is to produce a core dump; on Windows, the process immediately returns - an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with - :func:`signal.signal`. - - -.. function:: execl(path, arg0, arg1, ...) - execle(path, arg0, arg1, ..., env) - execlp(file, arg0, arg1, ...) - execlpe(file, arg0, arg1, ..., env) - execv(path, args) - execve(path, args, env) - execvp(file, args) - execvpe(file, args, env) - - These functions all execute a new program, replacing the current process; they - do not return. On Unix, the new executable is loaded into the current process, - and will have the same process id as the caller. Errors will be reported as - :exc:`OSError` exceptions. - - The current process is replaced immediately. Open file objects and - descriptors are not flushed, so if there may be data buffered - on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an - :func:`exec\* ` function. - - The "l" and "v" variants of the :func:`exec\* ` functions differ in how - command-line arguments are passed. The "l" variants are perhaps the easiest - to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the :func:`execl\*` - functions. The "v" variants are good when the number of parameters is - variable, with the arguments being passed in a list or tuple as the *args* - parameter. In either case, the arguments to the child process should start with - the name of the command being run, but this is not enforced. - - The variants which include a "p" near the end (:func:`execlp`, - :func:`execlpe`, :func:`execvp`, and :func:`execvpe`) will use the - :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`exec\*e ` variants, - discussed in the next paragraph), the new environment is used as the source of - the :envvar:`PATH` variable. The other variants, :func:`execl`, :func:`execle`, - :func:`execv`, and :func:`execve`, will not use the :envvar:`PATH` variable to - locate the executable; *path* must contain an appropriate absolute or relative - path. - - For :func:`execle`, :func:`execlpe`, :func:`execve`, and :func:`execvpe` (note - that these all end in "e"), the *env* parameter must be a mapping which is - used to define the environment variables for the new process (these are used - instead of the current process' environment); the functions :func:`execl`, - :func:`execlp`, :func:`execv`, and :func:`execvp` all cause the new process to - inherit the environment of the current process. - - For :func:`execve` on some platforms, *path* may also be specified as an open - file descriptor. This functionality may not be supported on your platform; - you can check whether or not it is available using :data:`os.supports_fd`. - If it is unavailable, using it will raise a :exc:`NotImplementedError`. - - Availability: Unix, Windows. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path* - for :func:`execve`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: _exit(n) - - Exit the process with status *n*, without calling cleanup handlers, flushing - stdio buffers, etc. - - .. note:: - - The standard way to exit is ``sys.exit(n)``. :func:`_exit` should - normally only be used in the child process after a :func:`fork`. - -The following exit codes are defined and can be used with :func:`_exit`, -although they are not required. These are typically used for system programs -written in Python, such as a mail server's external command delivery program. - -.. note:: - - Some of these may not be available on all Unix platforms, since there is some - variation. These constants are defined where they are defined by the underlying - platform. - - -.. data:: EX_OK - - Exit code that means no error occurred. - - Availability: Unix. - - -.. data:: EX_USAGE - - Exit code that means the command was used incorrectly, such as when the wrong - number of arguments are given. - - Availability: Unix. - - -.. data:: EX_DATAERR - - Exit code that means the input data was incorrect. - - Availability: Unix. - - -.. data:: EX_NOINPUT - - Exit code that means an input file did not exist or was not readable. - - Availability: Unix. - - -.. data:: EX_NOUSER - - Exit code that means a specified user did not exist. - - Availability: Unix. - - -.. data:: EX_NOHOST - - Exit code that means a specified host did not exist. - - Availability: Unix. - - -.. data:: EX_UNAVAILABLE - - Exit code that means that a required service is unavailable. - - Availability: Unix. - - -.. data:: EX_SOFTWARE - - Exit code that means an internal software error was detected. - - Availability: Unix. - - -.. data:: EX_OSERR - - Exit code that means an operating system error was detected, such as the - inability to fork or create a pipe. - - Availability: Unix. - - -.. data:: EX_OSFILE - - Exit code that means some system file did not exist, could not be opened, or had - some other kind of error. - - Availability: Unix. - - -.. data:: EX_CANTCREAT - - Exit code that means a user specified output file could not be created. - - Availability: Unix. - - -.. data:: EX_IOERR - - Exit code that means that an error occurred while doing I/O on some file. - - Availability: Unix. - - -.. data:: EX_TEMPFAIL - - Exit code that means a temporary failure occurred. This indicates something - that may not really be an error, such as a network connection that couldn't be - made during a retryable operation. - - Availability: Unix. - - -.. data:: EX_PROTOCOL - - Exit code that means that a protocol exchange was illegal, invalid, or not - understood. - - Availability: Unix. - - -.. data:: EX_NOPERM - - Exit code that means that there were insufficient permissions to perform the - operation (but not intended for file system problems). - - Availability: Unix. - - -.. data:: EX_CONFIG - - Exit code that means that some kind of configuration error occurred. - - Availability: Unix. - - -.. data:: EX_NOTFOUND - - Exit code that means something like "an entry was not found". - - Availability: Unix. - - -.. function:: fork() - - Fork a child process. Return ``0`` in the child and the child's process id in the - parent. If an error occurs :exc:`OSError` is raised. - - Note that some platforms including FreeBSD <= 6.3 and Cygwin have - known issues when using fork() from a thread. - - .. warning:: - - See :mod:`ssl` for applications that use the SSL module with fork(). - - Availability: Unix. - - -.. function:: forkpty() - - Fork a child process, using a new pseudo-terminal as the child's controlling - terminal. Return a pair of ``(pid, fd)``, where *pid* is ``0`` in the child, the - new child's process id in the parent, and *fd* is the file descriptor of the - master end of the pseudo-terminal. For a more portable approach, use the - :mod:`pty` module. If an error occurs :exc:`OSError` is raised. - - Availability: some flavors of Unix. - - -.. function:: kill(pid, sig) - - .. index:: - single: process; killing - single: process; signalling - - Send signal *sig* to the process *pid*. Constants for the specific signals - available on the host platform are defined in the :mod:`signal` module. - - Windows: The :data:`signal.CTRL_C_EVENT` and - :data:`signal.CTRL_BREAK_EVENT` signals are special signals which can - only be sent to console processes which share a common console window, - e.g., some subprocesses. Any other value for *sig* will cause the process - to be unconditionally killed by the TerminateProcess API, and the exit code - will be set to *sig*. The Windows version of :func:`kill` additionally takes - process handles to be killed. - - See also :func:`signal.pthread_kill`. - - .. versionadded:: 3.2 - Windows support. - - -.. function:: killpg(pgid, sig) - - .. index:: - single: process; killing - single: process; signalling - - Send the signal *sig* to the process group *pgid*. - - Availability: Unix. - - -.. function:: nice(increment) - - Add *increment* to the process's "niceness". Return the new niceness. - - Availability: Unix. - - -.. function:: plock(op) - - Lock program segments into memory. The value of *op* (defined in - ````) determines which segments are locked. - - Availability: Unix. - - -.. function:: popen(cmd, mode='r', buffering=-1) - - Open a pipe to or from command *cmd*. - The return value is an open file object - connected to the pipe, which can be read or written depending on whether *mode* - is ``'r'`` (default) or ``'w'``. The *buffering* argument has the same meaning as - the corresponding argument to the built-in :func:`open` function. The - returned file object reads or writes text strings rather than bytes. - - The ``close`` method returns :const:`None` if the subprocess exited - successfully, or the subprocess's return code if there was an - error. On POSIX systems, if the return code is positive it - represents the return value of the process left-shifted by one - byte. If the return code is negative, the process was terminated - by the signal given by the negated value of the return code. (For - example, the return value might be ``- signal.SIGKILL`` if the - subprocess was killed.) On Windows systems, the return value - contains the signed integer return code from the child process. - - This is implemented using :class:`subprocess.Popen`; see that class's - documentation for more powerful ways to manage and communicate with - subprocesses. - - -.. function:: spawnl(mode, path, ...) - spawnle(mode, path, ..., env) - spawnlp(mode, file, ...) - spawnlpe(mode, file, ..., env) - spawnv(mode, path, args) - spawnve(mode, path, args, env) - spawnvp(mode, file, args) - spawnvpe(mode, file, args, env) - - Execute the program *path* in a new process. - - (Note that the :mod:`subprocess` module provides more powerful facilities for - spawning new processes and retrieving their results; using that module is - preferable to using these functions. Check especially the - :ref:`subprocess-replacements` section.) - - If *mode* is :const:`P_NOWAIT`, this function returns the process id of the new - process; if *mode* is :const:`P_WAIT`, returns the process's exit code if it - exits normally, or ``-signal``, where *signal* is the signal that killed the - process. On Windows, the process id will actually be the process handle, so can - be used with the :func:`waitpid` function. - - The "l" and "v" variants of the :func:`spawn\* ` functions differ in how - command-line arguments are passed. The "l" variants are perhaps the easiest - to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the - :func:`spawnl\*` functions. The "v" variants are good when the number of - parameters is variable, with the arguments being passed in a list or tuple as - the *args* parameter. In either case, the arguments to the child process must - start with the name of the command being run. - - The variants which include a second "p" near the end (:func:`spawnlp`, - :func:`spawnlpe`, :func:`spawnvp`, and :func:`spawnvpe`) will use the - :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`spawn\*e ` variants, - discussed in the next paragraph), the new environment is used as the source of - the :envvar:`PATH` variable. The other variants, :func:`spawnl`, - :func:`spawnle`, :func:`spawnv`, and :func:`spawnve`, will not use the - :envvar:`PATH` variable to locate the executable; *path* must contain an - appropriate absolute or relative path. - - For :func:`spawnle`, :func:`spawnlpe`, :func:`spawnve`, and :func:`spawnvpe` - (note that these all end in "e"), the *env* parameter must be a mapping - which is used to define the environment variables for the new process (they are - used instead of the current process' environment); the functions - :func:`spawnl`, :func:`spawnlp`, :func:`spawnv`, and :func:`spawnvp` all cause - the new process to inherit the environment of the current process. Note that - keys and values in the *env* dictionary must be strings; invalid keys or - values will cause the function to fail, with a return value of ``127``. - - As an example, the following calls to :func:`spawnlp` and :func:`spawnvpe` are - equivalent:: - - import os - os.spawnlp(os.P_WAIT, 'cp', 'cp', 'index.html', '/dev/null') - - L = ['cp', 'index.html', '/dev/null'] - os.spawnvpe(os.P_WAIT, 'cp', L, os.environ) - - Availability: Unix, Windows. :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` - and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and - :func:`spawnve` are not thread-safe on Windows; we advise you to use the - :mod:`subprocess` module instead. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: P_NOWAIT - P_NOWAITO - - Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. If either of these values is given, the :func:`spawn\*` functions - will return as soon as the new process has been created, with the process id as - the return value. - - Availability: Unix, Windows. - - -.. data:: P_WAIT - - Possible value for the *mode* parameter to the :func:`spawn\* ` family of - functions. If this is given as *mode*, the :func:`spawn\*` functions will not - return until the new process has run to completion and will return the exit code - of the process the run is successful, or ``-signal`` if a signal kills the - process. - - Availability: Unix, Windows. - - -.. data:: P_DETACH - P_OVERLAY - - Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. These are less portable than those listed above. :const:`P_DETACH` - is similar to :const:`P_NOWAIT`, but the new process is detached from the - console of the calling process. If :const:`P_OVERLAY` is used, the current - process will be replaced; the :func:`spawn\* ` function will not return. - - Availability: Windows. - - -.. function:: startfile(path[, operation]) - - Start a file with its associated application. - - When *operation* is not specified or ``'open'``, this acts like double-clicking - the file in Windows Explorer, or giving the file name as an argument to the - :program:`start` command from the interactive command shell: the file is opened - with whatever application (if any) its extension is associated. - - When another *operation* is given, it must be a "command verb" that specifies - what should be done with the file. Common verbs documented by Microsoft are - ``'print'`` and ``'edit'`` (to be used on files) as well as ``'explore'`` and - ``'find'`` (to be used on directories). - - :func:`startfile` returns as soon as the associated application is launched. - There is no option to wait for the application to close, and no way to retrieve - the application's exit status. The *path* parameter is relative to the current - directory. If you want to use an absolute path, make sure the first character - is not a slash (``'/'``); the underlying Win32 :c:func:`ShellExecute` function - doesn't work if it is. Use the :func:`os.path.normpath` function to ensure that - the path is properly encoded for Win32. - - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` - function is not resolved until this function is first called. If the function - cannot be resolved, :exc:`NotImplementedError` will be raised. - - Availability: Windows. - - -.. function:: system(command) - - Execute the command (a string) in a subshell. This is implemented by calling - the Standard C function :c:func:`system`, and has the same limitations. - Changes to :data:`sys.stdin`, etc. are not reflected in the environment of - the executed command. If *command* generates any output, it will be sent to - the interpreter standard output stream. - - On Unix, the return value is the exit status of the process encoded in the - format specified for :func:`wait`. Note that POSIX does not specify the - meaning of the return value of the C :c:func:`system` function, so the return - value of the Python function is system-dependent. - - On Windows, the return value is that returned by the system shell after - running *command*. The shell is given by the Windows environment variable - :envvar:`COMSPEC`: it is usually :program:`cmd.exe`, which returns the exit - status of the command run; on systems using a non-native shell, consult your - shell documentation. - - The :mod:`subprocess` module provides more powerful facilities for spawning - new processes and retrieving their results; using that module is preferable - to using this function. See the :ref:`subprocess-replacements` section in - the :mod:`subprocess` documentation for some helpful recipes. - - Availability: Unix, Windows. - - -.. function:: times() - - Returns the current global process times. - The return value is an object with five attributes: - - * :attr:`user` - user time - * :attr:`system` - system time - * :attr:`children_user` - user time of all child processes - * :attr:`children_system` - system time of all child processes - * :attr:`elapsed` - elapsed real time since a fixed point in the past - - For backwards compatibility, this object also behaves like a five-tuple - containing :attr:`user`, :attr:`system`, :attr:`children_user`, - :attr:`children_system`, and :attr:`elapsed` in that order. - - See the Unix manual page - :manpage:`times(2)` or the corresponding Windows Platform API documentation. - On Windows, only :attr:`user` and :attr:`system` are known; the other - attributes are zero. - - Availability: Unix, Windows. - - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. - - -.. function:: wait() - - Wait for completion of a child process, and return a tuple containing its pid - and exit status indication: a 16-bit number, whose low byte is the signal number - that killed the process, and whose high byte is the exit status (if the signal - number is zero); the high bit of the low byte is set if a core file was - produced. - - Availability: Unix. - -.. function:: waitid(idtype, id, options) - - Wait for the completion of one or more child processes. - *idtype* can be :data:`P_PID`, :data:`P_PGID` or :data:`P_ALL`. - *id* specifies the pid to wait on. - *options* is constructed from the ORing of one or more of :data:`WEXITED`, - :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with - :data:`WNOHANG` or :data:`WNOWAIT`. The return value is an object - representing the data contained in the :c:type:`siginfo_t` structure, namely: - :attr:`si_pid`, :attr:`si_uid`, :attr:`si_signo`, :attr:`si_status`, - :attr:`si_code` or ``None`` if :data:`WNOHANG` is specified and there are no - children in a waitable state. - - Availability: Unix. - - .. versionadded:: 3.3 - -.. data:: P_PID - P_PGID - P_ALL - - These are the possible values for *idtype* in :func:`waitid`. They affect - how *id* is interpreted. - - Availability: Unix. - - .. versionadded:: 3.3 - -.. data:: WEXITED - WSTOPPED - WNOWAIT - - Flags that can be used in *options* in :func:`waitid` that specify what - child signal to wait for. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLD_EXITED - CLD_DUMPED - CLD_TRAPPED - CLD_CONTINUED - - These are the possible values for :attr:`si_code` in the result returned by - :func:`waitid`. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: waitpid(pid, options) - - The details of this function differ on Unix and Windows. - - On Unix: Wait for completion of a child process given by process id *pid*, and - return a tuple containing its process id and exit status indication (encoded as - for :func:`wait`). The semantics of the call are affected by the value of the - integer *options*, which should be ``0`` for normal operation. - - If *pid* is greater than ``0``, :func:`waitpid` requests status information for - that specific process. If *pid* is ``0``, the request is for the status of any - child in the process group of the current process. If *pid* is ``-1``, the - request pertains to any child of the current process. If *pid* is less than - ``-1``, status is requested for any process in the process group ``-pid`` (the - absolute value of *pid*). - - An :exc:`OSError` is raised with the value of errno when the syscall - returns -1. - - On Windows: Wait for completion of a process given by process handle *pid*, and - return a tuple containing *pid*, and its exit status shifted left by 8 bits - (shifting makes cross-platform use of the function easier). A *pid* less than or - equal to ``0`` has no special meaning on Windows, and raises an exception. The - value of integer *options* has no effect. *pid* can refer to any process whose - id is known, not necessarily a child process. The :func:`spawn\* ` - functions called with :const:`P_NOWAIT` return suitable process handles. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: wait3(options) - - Similar to :func:`waitpid`, except no process id argument is given and a - 3-element tuple containing the child's process id, exit status indication, - and resource usage information is returned. Refer to - :mod:`resource`.\ :func:`~resource.getrusage` for details on resource usage - information. The option argument is the same as that provided to - :func:`waitpid` and :func:`wait4`. - - Availability: Unix. - - -.. function:: wait4(pid, options) - - Similar to :func:`waitpid`, except a 3-element tuple, containing the child's - process id, exit status indication, and resource usage information is returned. - Refer to :mod:`resource`.\ :func:`~resource.getrusage` for details on - resource usage information. The arguments to :func:`wait4` are the same - as those provided to :func:`waitpid`. - - Availability: Unix. - - -.. data:: WNOHANG - - The option for :func:`waitpid` to return immediately if no child process status - is available immediately. The function returns ``(0, 0)`` in this case. - - Availability: Unix. - - -.. data:: WCONTINUED - - This option causes child processes to be reported if they have been continued - from a job control stop since their status was last reported. - - Availability: some Unix systems. - - -.. data:: WUNTRACED - - This option causes child processes to be reported if they have been stopped but - their current state has not been reported since they were stopped. - - Availability: Unix. - - -The following functions take a process status code as returned by -:func:`system`, :func:`wait`, or :func:`waitpid` as a parameter. They may be -used to determine the disposition of a process. - -.. function:: WCOREDUMP(status) - - Return ``True`` if a core dump was generated for the process, otherwise - return ``False``. - - Availability: Unix. - - -.. function:: WIFCONTINUED(status) - - Return ``True`` if the process has been continued from a job control stop, - otherwise return ``False``. - - Availability: Unix. - - -.. function:: WIFSTOPPED(status) - - Return ``True`` if the process has been stopped, otherwise return - ``False``. - - Availability: Unix. - - -.. function:: WIFSIGNALED(status) - - Return ``True`` if the process exited due to a signal, otherwise return - ``False``. - - Availability: Unix. - - -.. function:: WIFEXITED(status) - - Return ``True`` if the process exited using the :manpage:`exit(2)` system call, - otherwise return ``False``. - - Availability: Unix. - - -.. function:: WEXITSTATUS(status) - - If ``WIFEXITED(status)`` is true, return the integer parameter to the - :manpage:`exit(2)` system call. Otherwise, the return value is meaningless. - - Availability: Unix. - - -.. function:: WSTOPSIG(status) - - Return the signal which caused the process to stop. - - Availability: Unix. - - -.. function:: WTERMSIG(status) - - Return the signal which caused the process to exit. - - Availability: Unix. - - -Interface to the scheduler --------------------------- - -These functions control how a process is allocated CPU time by the operating -system. They are only available on some Unix platforms. For more detailed -information, consult your Unix manpages. - -.. versionadded:: 3.3 - -The following scheduling policies are exposed if they are supported by the -operating system. - -.. data:: SCHED_OTHER - - The default scheduling policy. - -.. data:: SCHED_BATCH - - Scheduling policy for CPU-intensive processes that tries to preserve - interactivity on the rest of the computer. - -.. data:: SCHED_IDLE - - Scheduling policy for extremely low priority background tasks. - -.. data:: SCHED_SPORADIC - - Scheduling policy for sporadic server programs. - -.. data:: SCHED_FIFO - - A First In First Out scheduling policy. - -.. data:: SCHED_RR - - A round-robin scheduling policy. - -.. data:: SCHED_RESET_ON_FORK - - This flag can be OR'ed with any other scheduling policy. When a process with - this flag set forks, its child's scheduling policy and priority are reset to - the default. - - -.. class:: sched_param(sched_priority) - - This class represents tunable scheduling parameters used in - :func:`sched_setparam`, :func:`sched_setscheduler`, and - :func:`sched_getparam`. It is immutable. - - At the moment, there is only one possible parameter: - - .. attribute:: sched_priority - - The scheduling priority for a scheduling policy. - - -.. function:: sched_get_priority_min(policy) - - Get the minimum priority value for *policy*. *policy* is one of the - scheduling policy constants above. - - -.. function:: sched_get_priority_max(policy) - - Get the maximum priority value for *policy*. *policy* is one of the - scheduling policy constants above. - - -.. function:: sched_setscheduler(pid, policy, param) - - Set the scheduling policy for the process with PID *pid*. A *pid* of 0 means - the calling process. *policy* is one of the scheduling policy constants - above. *param* is a :class:`sched_param` instance. - - -.. function:: sched_getscheduler(pid) - - Return the scheduling policy for the process with PID *pid*. A *pid* of 0 - means the calling process. The result is one of the scheduling policy - constants above. - - -.. function:: sched_setparam(pid, param) - - Set a scheduling parameters for the process with PID *pid*. A *pid* of 0 means - the calling process. *param* is a :class:`sched_param` instance. - - -.. function:: sched_getparam(pid) - - Return the scheduling parameters as a :class:`sched_param` instance for the - process with PID *pid*. A *pid* of 0 means the calling process. - - -.. function:: sched_rr_get_interval(pid) - - Return the round-robin quantum in seconds for the process with PID *pid*. A - *pid* of 0 means the calling process. - - -.. function:: sched_yield() - - Voluntarily relinquish the CPU. - - -.. function:: sched_setaffinity(pid, mask) - - Restrict the process with PID *pid* (or the current process if zero) to a - set of CPUs. *mask* is an iterable of integers representing the set of - CPUs to which the process should be restricted. - - -.. function:: sched_getaffinity(pid) - - Return the set of CPUs the process with PID *pid* (or the current process - if zero) is restricted to. - - -.. _os-path: - -Miscellaneous System Information --------------------------------- - - -.. function:: confstr(name) - - Return string-valued system configuration values. *name* specifies the - configuration value to retrieve; it may be a string which is the name of a - defined system value; these names are specified in a number of standards (POSIX, - Unix 95, Unix 98, and others). Some platforms define additional names as well. - The names known to the host operating system are given as the keys of the - ``confstr_names`` dictionary. For configuration variables not included in that - mapping, passing an integer for *name* is also accepted. - - If the configuration value specified by *name* isn't defined, ``None`` is - returned. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``confstr_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - Availability: Unix. - - -.. data:: confstr_names - - Dictionary mapping names accepted by :func:`confstr` to the integer values - defined for those names by the host operating system. This can be used to - determine the set of names known to the system. - - Availability: Unix. - - -.. function:: cpu_count() - - Return the number of CPUs in the system. Returns ``None`` if undetermined. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` - - - .. versionadded:: 3.4 - - -.. function:: getloadavg() - - Return the number of processes in the system run queue averaged over the last - 1, 5, and 15 minutes or raises :exc:`OSError` if the load average was - unobtainable. - - Availability: Unix. - - -.. function:: sysconf(name) - - Return integer-valued system configuration values. If the configuration value - specified by *name* isn't defined, ``-1`` is returned. The comments regarding - the *name* parameter for :func:`confstr` apply here as well; the dictionary that - provides information on the known names is given by ``sysconf_names``. - - Availability: Unix. - - -.. data:: sysconf_names - - Dictionary mapping names accepted by :func:`sysconf` to the integer values - defined for those names by the host operating system. This can be used to - determine the set of names known to the system. - - Availability: Unix. - -The following data values are used to support path manipulation operations. These -are defined for all platforms. - -Higher-level operations on pathnames are defined in the :mod:`os.path` module. - - -.. index:: single: . (dot); in pathnames -.. data:: curdir - - The constant string used by the operating system to refer to the current - directory. This is ``'.'`` for Windows and POSIX. Also available via - :mod:`os.path`. - - -.. index:: single: ..; in pathnames -.. data:: pardir - - The constant string used by the operating system to refer to the parent - directory. This is ``'..'`` for Windows and POSIX. Also available via - :mod:`os.path`. - - -.. index:: single: / (slash); in pathnames -.. index:: single: \ (backslash); in pathnames (Windows) -.. data:: sep - - The character used by the operating system to separate pathname components. - This is ``'/'`` for POSIX and ``'\\'`` for Windows. Note that knowing this - is not sufficient to be able to parse or concatenate pathnames --- use - :func:`os.path.split` and :func:`os.path.join` --- but it is occasionally - useful. Also available via :mod:`os.path`. - - -.. index:: single: / (slash); in pathnames -.. data:: altsep - - An alternative character used by the operating system to separate pathname - components, or ``None`` if only one separator character exists. This is set to - ``'/'`` on Windows systems where ``sep`` is a backslash. Also available via - :mod:`os.path`. - - -.. index:: single: . (dot); in pathnames -.. data:: extsep - - The character which separates the base filename from the extension; for example, - the ``'.'`` in :file:`os.py`. Also available via :mod:`os.path`. - - -.. index:: single: : (colon); path separator (POSIX) - single: ; (semicolon) -.. data:: pathsep - - The character conventionally used by the operating system to separate search - path components (as in :envvar:`PATH`), such as ``':'`` for POSIX or ``';'`` for - Windows. Also available via :mod:`os.path`. - - -.. data:: defpath - - The default search path used by :func:`exec\*p\* ` and - :func:`spawn\*p\* ` if the environment doesn't have a ``'PATH'`` - key. Also available via :mod:`os.path`. - - -.. data:: linesep - - The string used to separate (or, rather, terminate) lines on the current - platform. This may be a single character, such as ``'\n'`` for POSIX, or - multiple characters, for example, ``'\r\n'`` for Windows. Do not use - *os.linesep* as a line terminator when writing files opened in text mode (the - default); use a single ``'\n'`` instead, on all platforms. - - -.. data:: devnull - - The file path of the null device. For example: ``'/dev/null'`` for - POSIX, ``'nul'`` for Windows. Also available via :mod:`os.path`. - -.. data:: RTLD_LAZY - RTLD_NOW - RTLD_GLOBAL - RTLD_LOCAL - RTLD_NODELETE - RTLD_NOLOAD - RTLD_DEEPBIND - - Flags for use with the :func:`~sys.setdlopenflags` and - :func:`~sys.getdlopenflags` functions. See the Unix manual page - :manpage:`dlopen(3)` for what the different flags mean. - - .. versionadded:: 3.3 - - -Random numbers --------------- - - -.. function:: getrandom(size, flags=0) - - Get up to *size* random bytes. The function can return less bytes than - requested. - - These bytes can be used to seed user-space random number generators or for - cryptographic purposes. - - ``getrandom()`` relies on entropy gathered from device drivers and other - sources of environmental noise. Unnecessarily reading large quantities of - data will have a negative impact on other users of the ``/dev/random`` and - ``/dev/urandom`` devices. - - The flags argument is a bit mask that can contain zero or more of the - following values ORed together: :py:data:`os.GRND_RANDOM` and - :py:data:`GRND_NONBLOCK`. - - See also the `Linux getrandom() manual page - `_. - - Availability: Linux 3.17 and newer. - - .. versionadded:: 3.6 - -.. function:: urandom(size) - - Return a string of *size* random bytes suitable for cryptographic use. - - This function returns random bytes from an OS-specific randomness source. The - returned data should be unpredictable enough for cryptographic applications, - though its exact quality depends on the OS implementation. - - On Linux, if the ``getrandom()`` syscall is available, it is used in - blocking mode: block until the system urandom entropy pool is initialized - (128 bits of entropy are collected by the kernel). See the :pep:`524` for - the rationale. On Linux, the :func:`getrandom` function can be used to get - random bytes in non-blocking mode (using the :data:`GRND_NONBLOCK` flag) or - to poll until the system urandom entropy pool is initialized. - - On a Unix-like system, random bytes are read from the ``/dev/urandom`` - device. If the ``/dev/urandom`` device is not available or not readable, the - :exc:`NotImplementedError` exception is raised. - - On Windows, it will use ``CryptGenRandom()``. - - .. seealso:: - The :mod:`secrets` module provides higher level functions. For an - easy-to-use interface to the random number generator provided by your - platform, please see :class:`random.SystemRandom`. - - .. versionchanged:: 3.6.0 - On Linux, ``getrandom()`` is now used in blocking mode to increase the - security. - - .. versionchanged:: 3.5.2 - On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool - is not initialized yet), fall back on reading ``/dev/urandom``. - - .. versionchanged:: 3.5 - On Linux 3.17 and newer, the ``getrandom()`` syscall is now used - when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` - function is now used. These functions avoid the usage of an internal file - descriptor. - -.. data:: GRND_NONBLOCK - - By default, when reading from ``/dev/random``, :func:`getrandom` blocks if - no random bytes are available, and when reading from ``/dev/urandom``, it blocks - if the entropy pool has not yet been initialized. - - If the :py:data:`GRND_NONBLOCK` flag is set, then :func:`getrandom` does not - block in these cases, but instead immediately raises :exc:`BlockingIOError`. - - .. versionadded:: 3.6 - -.. data:: GRND_RANDOM - - If this bit is set, then random bytes are drawn from the - ``/dev/random`` pool instead of the ``/dev/urandom`` pool. - - .. versionadded:: 3.6 diff --git a/third_party/python/Doc/library/ossaudiodev.rst b/third_party/python/Doc/library/ossaudiodev.rst deleted file mode 100644 index a7d3dac36..000000000 --- a/third_party/python/Doc/library/ossaudiodev.rst +++ /dev/null @@ -1,448 +0,0 @@ -:mod:`ossaudiodev` --- Access to OSS-compatible audio devices -============================================================= - -.. module:: ossaudiodev - :platform: Linux, FreeBSD - :synopsis: Access to OSS-compatible audio devices. - --------------- - -This module allows you to access the OSS (Open Sound System) audio interface. -OSS is available for a wide range of open-source and commercial Unices, and is -the standard audio interface for Linux and recent versions of FreeBSD. - -.. Things will get more complicated for future Linux versions, since - ALSA is in the standard kernel as of 2.5.x. Presumably if you - use ALSA, you'll have to make sure its OSS compatibility layer - is active to use ossaudiodev, but you're going to need it for the vast - majority of Linux audio apps anyway. - - Sounds like things are also complicated for other BSDs. In response - to my python-dev query, Thomas Wouters said: - - > Likewise, googling shows OpenBSD also uses OSS/Free -- the commercial - > OSS installation manual tells you to remove references to OSS/Free from the - > kernel :) - - but Aleksander Piotrowsk actually has an OpenBSD box, and he quotes - from its : - > * WARNING! WARNING! - > * This is an OSS (Linux) audio emulator. - > * Use the Native NetBSD API for developing new code, and this - > * only for compiling Linux programs. - - There's also an ossaudio manpage on OpenBSD that explains things - further. Presumably NetBSD and OpenBSD have a different standard - audio interface. That's the great thing about standards, there are so - many to choose from ... ;-) - - This probably all warrants a footnote or two, but I don't understand - things well enough right now to write it! --GPW - -.. versionchanged:: 3.3 - Operations in this module now raise :exc:`OSError` where :exc:`IOError` - was raised. - - -.. seealso:: - - `Open Sound System Programmer's Guide `_ - the official documentation for the OSS C API - - The module defines a large number of constants supplied by the OSS device - driver; see ```` on either Linux or FreeBSD for a listing. - -:mod:`ossaudiodev` defines the following variables and functions: - - -.. exception:: OSSAudioError - - This exception is raised on certain errors. The argument is a string describing - what went wrong. - - (If :mod:`ossaudiodev` receives an error from a system call such as - :c:func:`open`, :c:func:`write`, or :c:func:`ioctl`, it raises :exc:`OSError`. - Errors detected directly by :mod:`ossaudiodev` result in :exc:`OSSAudioError`.) - - (For backwards compatibility, the exception class is also available as - ``ossaudiodev.error``.) - - -.. function:: open(mode) - open(device, mode) - - Open an audio device and return an OSS audio device object. This object - supports many file-like methods, such as :meth:`read`, :meth:`write`, and - :meth:`fileno` (although there are subtle differences between conventional Unix - read/write semantics and those of OSS audio devices). It also supports a number - of audio-specific methods; see below for the complete list of methods. - - *device* is the audio device filename to use. If it is not specified, this - module first looks in the environment variable :envvar:`AUDIODEV` for a device - to use. If not found, it falls back to :file:`/dev/dsp`. - - *mode* is one of ``'r'`` for read-only (record) access, ``'w'`` for - write-only (playback) access and ``'rw'`` for both. Since many sound cards - only allow one process to have the recorder or player open at a time, it is a - good idea to open the device only for the activity needed. Further, some - sound cards are half-duplex: they can be opened for reading or writing, but - not both at once. - - Note the unusual calling syntax: the *first* argument is optional, and the - second is required. This is a historical artifact for compatibility with the - older :mod:`linuxaudiodev` module which :mod:`ossaudiodev` supersedes. - - .. XXX it might also be motivated - by my unfounded-but-still-possibly-true belief that the default - audio device varies unpredictably across operating systems. -GW - - -.. function:: openmixer([device]) - - Open a mixer device and return an OSS mixer device object. *device* is the - mixer device filename to use. If it is not specified, this module first looks - in the environment variable :envvar:`MIXERDEV` for a device to use. If not - found, it falls back to :file:`/dev/mixer`. - - -.. _ossaudio-device-objects: - -Audio Device Objects --------------------- - -Before you can write to or read from an audio device, you must call three -methods in the correct order: - -#. :meth:`setfmt` to set the output format - -#. :meth:`channels` to set the number of channels - -#. :meth:`speed` to set the sample rate - -Alternately, you can use the :meth:`setparameters` method to set all three audio -parameters at once. This is more convenient, but may not be as flexible in all -cases. - -The audio device objects returned by :func:`.open` define the following methods -and (read-only) attributes: - - -.. method:: oss_audio_device.close() - - Explicitly close the audio device. When you are done writing to or reading from - an audio device, you should explicitly close it. A closed device cannot be used - again. - - -.. method:: oss_audio_device.fileno() - - Return the file descriptor associated with the device. - - -.. method:: oss_audio_device.read(size) - - Read *size* bytes from the audio input and return them as a Python string. - Unlike most Unix device drivers, OSS audio devices in blocking mode (the - default) will block :func:`read` until the entire requested amount of data is - available. - - -.. method:: oss_audio_device.write(data) - - Write a :term:`bytes-like object` *data* to the audio device and return the - number of bytes written. If the audio device is in blocking mode (the - default), the entire data is always written (again, this is different from - usual Unix device semantics). If the device is in non-blocking mode, some - data may not be written---see :meth:`writeall`. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. method:: oss_audio_device.writeall(data) - - Write a :term:`bytes-like object` *data* to the audio device: waits until - the audio device is able to accept data, writes as much data as it will - accept, and repeats until *data* has been completely written. If the device - is in blocking mode (the default), this has the same effect as - :meth:`write`; :meth:`writeall` is only useful in non-blocking mode. Has - no return value, since the amount of data written is always equal to the - amount of data supplied. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. versionchanged:: 3.2 - Audio device objects also support the context management protocol, i.e. they can - be used in a :keyword:`with` statement. - - -The following methods each map to exactly one :c:func:`ioctl` system call. The -correspondence is obvious: for example, :meth:`setfmt` corresponds to the -``SNDCTL_DSP_SETFMT`` ioctl, and :meth:`sync` to ``SNDCTL_DSP_SYNC`` (this can -be useful when consulting the OSS documentation). If the underlying -:c:func:`ioctl` fails, they all raise :exc:`OSError`. - - -.. method:: oss_audio_device.nonblock() - - Put the device into non-blocking mode. Once in non-blocking mode, there is no - way to return it to blocking mode. - - -.. method:: oss_audio_device.getfmts() - - Return a bitmask of the audio output formats supported by the soundcard. Some - of the formats supported by OSS are: - - +-------------------------+---------------------------------------------+ - | Format | Description | - +=========================+=============================================+ - | :const:`AFMT_MU_LAW` | a logarithmic encoding (used by Sun ``.au`` | - | | files and :file:`/dev/audio`) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_A_LAW` | a logarithmic encoding | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_IMA_ADPCM` | a 4:1 compressed format defined by the | - | | Interactive Multimedia Association | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U8` | Unsigned, 8-bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_LE` | Signed, 16-bit audio, little-endian byte | - | | order (as used by Intel processors) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_BE` | Signed, 16-bit audio, big-endian byte order | - | | (as used by 68k, PowerPC, Sparc) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S8` | Signed, 8 bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_LE` | Unsigned, 16-bit little-endian audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_BE` | Unsigned, 16-bit big-endian audio | - +-------------------------+---------------------------------------------+ - - Consult the OSS documentation for a full list of audio formats, and note that - most devices support only a subset of these formats. Some older devices only - support :const:`AFMT_U8`; the most common format used today is - :const:`AFMT_S16_LE`. - - -.. method:: oss_audio_device.setfmt(format) - - Try to set the current audio format to *format*---see :meth:`getfmts` for a - list. Returns the audio format that the device was set to, which may not be the - requested format. May also be used to return the current audio format---do this - by passing an "audio format" of :const:`AFMT_QUERY`. - - -.. method:: oss_audio_device.channels(nchannels) - - Set the number of output channels to *nchannels*. A value of 1 indicates - monophonic sound, 2 stereophonic. Some devices may have more than 2 channels, - and some high-end devices may not support mono. Returns the number of channels - the device was set to. - - -.. method:: oss_audio_device.speed(samplerate) - - Try to set the audio sampling rate to *samplerate* samples per second. Returns - the rate actually set. Most sound devices don't support arbitrary sampling - rates. Common rates are: - - +-------+-------------------------------------------+ - | Rate | Description | - +=======+===========================================+ - | 8000 | default rate for :file:`/dev/audio` | - +-------+-------------------------------------------+ - | 11025 | speech recording | - +-------+-------------------------------------------+ - | 22050 | | - +-------+-------------------------------------------+ - | 44100 | CD quality audio (at 16 bits/sample and 2 | - | | channels) | - +-------+-------------------------------------------+ - | 96000 | DVD quality audio (at 24 bits/sample) | - +-------+-------------------------------------------+ - - -.. method:: oss_audio_device.sync() - - Wait until the sound device has played every byte in its buffer. (This happens - implicitly when the device is closed.) The OSS documentation recommends closing - and re-opening the device rather than using :meth:`sync`. - - -.. method:: oss_audio_device.reset() - - Immediately stop playing or recording and return the device to a state where it - can accept commands. The OSS documentation recommends closing and re-opening - the device after calling :meth:`reset`. - - -.. method:: oss_audio_device.post() - - Tell the driver that there is likely to be a pause in the output, making it - possible for the device to handle the pause more intelligently. You might use - this after playing a spot sound effect, before waiting for user input, or before - doing disk I/O. - -The following convenience methods combine several ioctls, or one ioctl and some -simple calculations. - - -.. method:: oss_audio_device.setparameters(format, nchannels, samplerate[, strict=False]) - - Set the key audio sampling parameters---sample format, number of channels, and - sampling rate---in one method call. *format*, *nchannels*, and *samplerate* - should be as specified in the :meth:`setfmt`, :meth:`channels`, and - :meth:`speed` methods. If *strict* is true, :meth:`setparameters` checks to - see if each parameter was actually set to the requested value, and raises - :exc:`OSSAudioError` if not. Returns a tuple (*format*, *nchannels*, - *samplerate*) indicating the parameter values that were actually set by the - device driver (i.e., the same as the return values of :meth:`setfmt`, - :meth:`channels`, and :meth:`speed`). - - For example, :: - - (fmt, channels, rate) = dsp.setparameters(fmt, channels, rate) - - is equivalent to :: - - fmt = dsp.setfmt(fmt) - channels = dsp.channels(channels) - rate = dsp.rate(rate) - - -.. method:: oss_audio_device.bufsize() - - Returns the size of the hardware buffer, in samples. - - -.. method:: oss_audio_device.obufcount() - - Returns the number of samples that are in the hardware buffer yet to be played. - - -.. method:: oss_audio_device.obuffree() - - Returns the number of samples that could be queued into the hardware buffer to - be played without blocking. - -Audio device objects also support several read-only attributes: - - -.. attribute:: oss_audio_device.closed - - Boolean indicating whether the device has been closed. - - -.. attribute:: oss_audio_device.name - - String containing the name of the device file. - - -.. attribute:: oss_audio_device.mode - - The I/O mode for the file, either ``"r"``, ``"rw"``, or ``"w"``. - - -.. _mixer-device-objects: - -Mixer Device Objects --------------------- - -The mixer object provides two file-like methods: - - -.. method:: oss_mixer_device.close() - - This method closes the open mixer device file. Any further attempts to use the - mixer after this file is closed will raise an :exc:`OSError`. - - -.. method:: oss_mixer_device.fileno() - - Returns the file handle number of the open mixer device file. - -.. versionchanged:: 3.2 - Mixer objects also support the context management protocol. - - -The remaining methods are specific to audio mixing: - - -.. method:: oss_mixer_device.controls() - - This method returns a bitmask specifying the available mixer controls ("Control" - being a specific mixable "channel", such as :const:`SOUND_MIXER_PCM` or - :const:`SOUND_MIXER_SYNTH`). This bitmask indicates a subset of all available - mixer controls---the :const:`SOUND_MIXER_\*` constants defined at module level. - To determine if, for example, the current mixer object supports a PCM mixer, use - the following Python code:: - - mixer=ossaudiodev.openmixer() - if mixer.controls() & (1 << ossaudiodev.SOUND_MIXER_PCM): - # PCM is supported - ... code ... - - For most purposes, the :const:`SOUND_MIXER_VOLUME` (master volume) and - :const:`SOUND_MIXER_PCM` controls should suffice---but code that uses the mixer - should be flexible when it comes to choosing mixer controls. On the Gravis - Ultrasound, for example, :const:`SOUND_MIXER_VOLUME` does not exist. - - -.. method:: oss_mixer_device.stereocontrols() - - Returns a bitmask indicating stereo mixer controls. If a bit is set, the - corresponding control is stereo; if it is unset, the control is either - monophonic or not supported by the mixer (use in combination with - :meth:`controls` to determine which). - - See the code example for the :meth:`controls` function for an example of getting - data from a bitmask. - - -.. method:: oss_mixer_device.reccontrols() - - Returns a bitmask specifying the mixer controls that may be used to record. See - the code example for :meth:`controls` for an example of reading from a bitmask. - - -.. method:: oss_mixer_device.get(control) - - Returns the volume of a given mixer control. The returned volume is a 2-tuple - ``(left_volume,right_volume)``. Volumes are specified as numbers from 0 - (silent) to 100 (full volume). If the control is monophonic, a 2-tuple is still - returned, but both volumes are the same. - - Raises :exc:`OSSAudioError` if an invalid control is specified, or - :exc:`OSError` if an unsupported control is specified. - - -.. method:: oss_mixer_device.set(control, (left, right)) - - Sets the volume for a given mixer control to ``(left,right)``. ``left`` and - ``right`` must be ints and between 0 (silent) and 100 (full volume). On - success, the new volume is returned as a 2-tuple. Note that this may not be - exactly the same as the volume specified, because of the limited resolution of - some soundcard's mixers. - - Raises :exc:`OSSAudioError` if an invalid mixer control was specified, or if the - specified volumes were out-of-range. - - -.. method:: oss_mixer_device.get_recsrc() - - This method returns a bitmask indicating which control(s) are currently being - used as a recording source. - - -.. method:: oss_mixer_device.set_recsrc(bitmask) - - Call this function to specify a recording source. Returns a bitmask indicating - the new recording source (or sources) if successful; raises :exc:`OSError` if an - invalid source was specified. To set the current recording source to the - microphone input:: - - mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC) diff --git a/third_party/python/Doc/library/othergui.rst b/third_party/python/Doc/library/othergui.rst deleted file mode 100644 index d40abe167..000000000 --- a/third_party/python/Doc/library/othergui.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. _other-gui-packages: - -Other Graphical User Interface Packages -======================================= - -Major cross-platform (Windows, Mac OS X, Unix-like) GUI toolkits are -available for Python: - -.. seealso:: - - `PyGObject `_ - PyGObject provides introspection bindings for C libraries using - `GObject `_. One of - these libraries is the `GTK+ 3 `_ widget set. - GTK+ comes with many more widgets than Tkinter provides. An online - `Python GTK+ 3 Tutorial `_ - is available. - - `PyGTK `_ - PyGTK provides bindings for an older version - of the library, GTK+ 2. It provides an object oriented interface that - is slightly higher level than the C one. There are also bindings to - `GNOME `_. An online `tutorial - `_ is available. - - `PyQt `_ - PyQt is a :program:`sip`\ -wrapped binding to the Qt toolkit. Qt is an - extensive C++ GUI application development framework that is - available for Unix, Windows and Mac OS X. :program:`sip` is a tool - for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. - - `PySide `_ - PySide is a newer binding to the Qt toolkit, provided by Nokia. - Compared to PyQt, its licensing scheme is friendlier to non-open source - applications. - - `wxPython `_ - wxPython is a cross-platform GUI toolkit for Python that is built around - the popular `wxWidgets `_ (formerly wxWindows) - C++ toolkit. It provides a native look and feel for applications on - Windows, Mac OS X, and Unix systems by using each platform's native - widgets where ever possible, (GTK+ on Unix-like systems). In addition to - an extensive set of widgets, wxPython provides classes for online - documentation and context sensitive help, printing, HTML viewing, - low-level device context drawing, drag and drop, system clipboard access, - an XML-based resource format and more, including an ever growing library - of user-contributed modules. - -PyGTK, PyQt, and wxPython, all have a modern look and feel and more -widgets than Tkinter. In addition, there are many other GUI toolkits for -Python, both cross-platform, and platform-specific. See the `GUI Programming -`_ page in the Python Wiki for a -much more complete list, and also for links to documents where the -different GUI toolkits are compared. - diff --git a/third_party/python/Doc/library/parser.rst b/third_party/python/Doc/library/parser.rst deleted file mode 100644 index c3b699a36..000000000 --- a/third_party/python/Doc/library/parser.rst +++ /dev/null @@ -1,356 +0,0 @@ -:mod:`parser` --- Access Python parse trees -=========================================== - -.. module:: parser - :synopsis: Access parse trees for Python source code. - -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - -.. Copyright 1995 Virginia Polytechnic Institute and State University and Fred - L. Drake, Jr. This copyright notice must be distributed on all copies, but - this document otherwise may be distributed as part of the Python - distribution. No fee may be charged for this document in any representation, - either on paper or electronically. This restriction does not affect other - elements in a distributed package in any way. - -.. index:: single: parsing; Python source code - --------------- - -The :mod:`parser` module provides an interface to Python's internal parser and -byte-code compiler. The primary purpose for this interface is to allow Python -code to edit the parse tree of a Python expression and create executable code -from this. This is better than trying to parse and modify an arbitrary Python -code fragment as a string because parsing is performed in a manner identical to -the code forming the application. It is also faster. - -.. note:: - - From Python 2.5 onward, it's much more convenient to cut in at the Abstract - Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` - module. - -There are a few things to note about this module which are important to making -use of the data structures created. This is not a tutorial on editing the parse -trees for Python code, but some examples of using the :mod:`parser` module are -presented. - -Most importantly, a good understanding of the Python grammar processed by the -internal parser is required. For full information on the language syntax, refer -to :ref:`reference-index`. The parser -itself is created from a grammar specification defined in the file -:file:`Grammar/Grammar` in the standard Python distribution. The parse trees -stored in the ST objects created by this module are the actual output from the -internal parser when created by the :func:`expr` or :func:`suite` functions, -described below. The ST objects created by :func:`sequence2st` faithfully -simulate those structures. Be aware that the values of the sequences which are -considered "correct" will vary from one version of Python to another as the -formal grammar for the language is revised. However, transporting code from one -Python version to another as source text will always allow correct parse trees -to be created in the target version, with the only restriction being that -migrating to an older version of the interpreter will not support more recent -language constructs. The parse trees are not typically compatible from one -version to another, whereas source code has always been forward-compatible. - -Each element of the sequences returned by :func:`st2list` or :func:`st2tuple` -has a simple form. Sequences representing non-terminal elements in the grammar -always have a length greater than one. The first element is an integer which -identifies a production in the grammar. These integers are given symbolic names -in the C header file :file:`Include/graminit.h` and the Python module -:mod:`symbol`. Each additional element of the sequence represents a component -of the production as recognized in the input string: these are always sequences -which have the same form as the parent. An important aspect of this structure -which should be noted is that keywords used to identify the parent node type, -such as the keyword :keyword:`if` in an :const:`if_stmt`, are included in the -node tree without any special treatment. For example, the :keyword:`if` keyword -is represented by the tuple ``(1, 'if')``, where ``1`` is the numeric value -associated with all :const:`NAME` tokens, including variable and function names -defined by the user. In an alternate form returned when line number information -is requested, the same token might be represented as ``(1, 'if', 12)``, where -the ``12`` represents the line number at which the terminal symbol was found. - -Terminal elements are represented in much the same way, but without any child -elements and the addition of the source text which was identified. The example -of the :keyword:`if` keyword above is representative. The various types of -terminal symbols are defined in the C header file :file:`Include/token.h` and -the Python module :mod:`token`. - -The ST objects are not required to support the functionality of this module, -but are provided for three purposes: to allow an application to amortize the -cost of processing complex parse trees, to provide a parse tree representation -which conserves memory space when compared to the Python list or tuple -representation, and to ease the creation of additional modules in C which -manipulate parse trees. A simple "wrapper" class may be created in Python to -hide the use of ST objects. - -The :mod:`parser` module defines functions for a few distinct purposes. The -most important purposes are to create ST objects and to convert ST objects to -other representations such as parse trees and compiled code objects, but there -are also functions which serve to query the type of parse tree represented by an -ST object. - - -.. seealso:: - - Module :mod:`symbol` - Useful constants representing internal nodes of the parse tree. - - Module :mod:`token` - Useful constants representing leaf nodes of the parse tree and functions for - testing node values. - - -.. _creating-sts: - -Creating ST Objects -------------------- - -ST objects may be created from source code or from a parse tree. When creating -an ST object from source, different functions are used to create the ``'eval'`` -and ``'exec'`` forms. - - -.. function:: expr(source) - - The :func:`expr` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'eval')``. If the parse succeeds, an ST object - is created to hold the internal parse tree representation, otherwise an - appropriate exception is raised. - - -.. function:: suite(source) - - The :func:`suite` function parses the parameter *source* as if it were an input - to ``compile(source, 'file.py', 'exec')``. If the parse succeeds, an ST object - is created to hold the internal parse tree representation, otherwise an - appropriate exception is raised. - - -.. function:: sequence2st(sequence) - - This function accepts a parse tree represented as a sequence and builds an - internal representation if possible. If it can validate that the tree conforms - to the Python grammar and all nodes are valid node types in the host version of - Python, an ST object is created from the internal representation and returned - to the called. If there is a problem creating the internal representation, or - if the tree cannot be validated, a :exc:`ParserError` exception is raised. An - ST object created this way should not be assumed to compile correctly; normal - exceptions raised by compilation may still be initiated when the ST object is - passed to :func:`compilest`. This may indicate problems not related to syntax - (such as a :exc:`MemoryError` exception), but may also be due to constructs such - as the result of parsing ``del f(0)``, which escapes the Python parser but is - checked by the bytecode compiler. - - Sequences representing terminal tokens may be represented as either two-element - lists of the form ``(1, 'name')`` or as three-element lists of the form ``(1, - 'name', 56)``. If the third element is present, it is assumed to be a valid - line number. The line number may be specified for any subset of the terminal - symbols in the input tree. - - -.. function:: tuple2st(sequence) - - This is the same function as :func:`sequence2st`. This entry point is - maintained for backward compatibility. - - -.. _converting-sts: - -Converting ST Objects ---------------------- - -ST objects, regardless of the input used to create them, may be converted to -parse trees represented as list- or tuple- trees, or may be compiled into -executable code objects. Parse trees may be extracted with or without line -numbering information. - - -.. function:: st2list(st, line_info=False, col_info=False) - - This function accepts an ST object from the caller in *st* and returns a - Python list representing the equivalent parse tree. The resulting list - representation can be used for inspection or the creation of a new parse tree in - list form. This function does not fail so long as memory is available to build - the list representation. If the parse tree will only be used for inspection, - :func:`st2tuple` should be used instead to reduce memory consumption and - fragmentation. When the list representation is required, this function is - significantly faster than retrieving a tuple representation and converting that - to nested lists. - - If *line_info* is true, line number information will be included for all - terminal tokens as a third element of the list representing the token. Note - that the line number provided specifies the line on which the token *ends*. - This information is omitted if the flag is false or omitted. - - -.. function:: st2tuple(st, line_info=False, col_info=False) - - This function accepts an ST object from the caller in *st* and returns a - Python tuple representing the equivalent parse tree. Other than returning a - tuple instead of a list, this function is identical to :func:`st2list`. - - If *line_info* is true, line number information will be included for all - terminal tokens as a third element of the list representing the token. This - information is omitted if the flag is false or omitted. - - -.. function:: compilest(st, filename='') - - .. index:: - builtin: exec - builtin: eval - - The Python byte compiler can be invoked on an ST object to produce code objects - which can be used as part of a call to the built-in :func:`exec` or :func:`eval` - functions. This function provides the interface to the compiler, passing the - internal parse tree from *st* to the parser, using the source file name - specified by the *filename* parameter. The default value supplied for *filename* - indicates that the source was an ST object. - - Compiling an ST object may result in exceptions related to compilation; an - example would be a :exc:`SyntaxError` caused by the parse tree for ``del f(0)``: - this statement is considered legal within the formal grammar for Python but is - not a legal language construct. The :exc:`SyntaxError` raised for this - condition is actually generated by the Python byte-compiler normally, which is - why it can be raised at this point by the :mod:`parser` module. Most causes of - compilation failure can be diagnosed programmatically by inspection of the parse - tree. - - -.. _querying-sts: - -Queries on ST Objects ---------------------- - -Two functions are provided which allow an application to determine if an ST was -created as an expression or a suite. Neither of these functions can be used to -determine if an ST was created from source code via :func:`expr` or -:func:`suite` or from a parse tree via :func:`sequence2st`. - - -.. function:: isexpr(st) - - .. index:: builtin: compile - - When *st* represents an ``'eval'`` form, this function returns true, otherwise - it returns false. This is useful, since code objects normally cannot be queried - for this information using existing built-in functions. Note that the code - objects created by :func:`compilest` cannot be queried like this either, and - are identical to those created by the built-in :func:`compile` function. - - -.. function:: issuite(st) - - This function mirrors :func:`isexpr` in that it reports whether an ST object - represents an ``'exec'`` form, commonly known as a "suite." It is not safe to - assume that this function is equivalent to ``not isexpr(st)``, as additional - syntactic fragments may be supported in the future. - - -.. _st-errors: - -Exceptions and Error Handling ------------------------------ - -The parser module defines a single exception, but may also pass other built-in -exceptions from other portions of the Python runtime environment. See each -function for information about the exceptions it can raise. - - -.. exception:: ParserError - - Exception raised when a failure occurs within the parser module. This is - generally produced for validation failures rather than the built-in - :exc:`SyntaxError` raised during normal parsing. The exception argument is - either a string describing the reason of the failure or a tuple containing a - sequence causing the failure from a parse tree passed to :func:`sequence2st` - and an explanatory string. Calls to :func:`sequence2st` need to be able to - handle either type of exception, while calls to other functions in the module - will only need to be aware of the simple string values. - -Note that the functions :func:`compilest`, :func:`expr`, and :func:`suite` may -raise exceptions which are normally raised by the parsing and compilation -process. These include the built in exceptions :exc:`MemoryError`, -:exc:`OverflowError`, :exc:`SyntaxError`, and :exc:`SystemError`. In these -cases, these exceptions carry all the meaning normally associated with them. -Refer to the descriptions of each function for detailed information. - - -.. _st-objects: - -ST Objects ----------- - -Ordered and equality comparisons are supported between ST objects. Pickling of -ST objects (using the :mod:`pickle` module) is also supported. - - -.. data:: STType - - The type of the objects returned by :func:`expr`, :func:`suite` and - :func:`sequence2st`. - -ST objects have the following methods: - - -.. method:: ST.compile(filename='') - - Same as ``compilest(st, filename)``. - - -.. method:: ST.isexpr() - - Same as ``isexpr(st)``. - - -.. method:: ST.issuite() - - Same as ``issuite(st)``. - - -.. method:: ST.tolist(line_info=False, col_info=False) - - Same as ``st2list(st, line_info, col_info)``. - - -.. method:: ST.totuple(line_info=False, col_info=False) - - Same as ``st2tuple(st, line_info, col_info)``. - - -Example: Emulation of :func:`compile` -------------------------------------- - -While many useful operations may take place between parsing and bytecode -generation, the simplest operation is to do nothing. For this purpose, using -the :mod:`parser` module to produce an intermediate data structure is equivalent -to the code :: - - >>> code = compile('a + 5', 'file.py', 'eval') - >>> a = 5 - >>> eval(code) - 10 - -The equivalent operation using the :mod:`parser` module is somewhat longer, and -allows the intermediate internal parse tree to be retained as an ST object:: - - >>> import parser - >>> st = parser.expr('a + 5') - >>> code = st.compile('file.py') - >>> a = 5 - >>> eval(code) - 10 - -An application which needs both ST and code objects can package this code into -readily available functions:: - - import parser - - def load_suite(source_string): - st = parser.suite(source_string) - return st, st.compile() - - def load_expression(source_string): - st = parser.expr(source_string) - return st, st.compile() diff --git a/third_party/python/Doc/library/pathlib-inheritance.png b/third_party/python/Doc/library/pathlib-inheritance.png deleted file mode 100644 index b81c3deb6..000000000 Binary files a/third_party/python/Doc/library/pathlib-inheritance.png and /dev/null differ diff --git a/third_party/python/Doc/library/pathlib-inheritance.svg b/third_party/python/Doc/library/pathlib-inheritance.svg deleted file mode 100644 index 9f42005e0..000000000 --- a/third_party/python/Doc/library/pathlib-inheritance.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/third_party/python/Doc/library/pathlib.rst b/third_party/python/Doc/library/pathlib.rst deleted file mode 100644 index 0377027da..000000000 --- a/third_party/python/Doc/library/pathlib.rst +++ /dev/null @@ -1,1052 +0,0 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== - -.. module:: pathlib - :synopsis: Object-oriented filesystem paths - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/pathlib.py` - -.. index:: single: path; operations - --------------- - -This module offers classes representing filesystem paths with semantics -appropriate for different operating systems. Path classes are divided -between :ref:`pure paths `, which provide purely computational -operations without I/O, and :ref:`concrete paths `, which -inherit from pure paths but also provide I/O operations. - -.. image:: pathlib-inheritance.png - :align: center - -If you've never used this module before or just aren't sure which class is -right for your task, :class:`Path` is most likely what you need. It instantiates -a :ref:`concrete path ` for the platform the code is running on. - -Pure paths are useful in some special cases; for example: - -#. If you want to manipulate Windows paths on a Unix machine (or vice versa). - You cannot instantiate a :class:`WindowsPath` when running on Unix, but you - can instantiate :class:`PureWindowsPath`. -#. You want to make sure that your code only manipulates paths without actually - accessing the OS. In this case, instantiating one of the pure classes may be - useful since those simply don't have any OS-accessing operations. - -.. seealso:: - :pep:`428`: The pathlib module -- object-oriented filesystem paths. - -.. seealso:: - For low-level path manipulation on strings, you can also use the - :mod:`os.path` module. - - -Basic use ---------- - -Importing the main class:: - - >>> from pathlib import Path - -Listing subdirectories:: - - >>> p = Path('.') - >>> [x for x in p.iterdir() if x.is_dir()] - [PosixPath('.hg'), PosixPath('docs'), PosixPath('dist'), - PosixPath('__pycache__'), PosixPath('build')] - -Listing Python source files in this directory tree:: - - >>> list(p.glob('**/*.py')) - [PosixPath('test_pathlib.py'), PosixPath('setup.py'), - PosixPath('pathlib.py'), PosixPath('docs/conf.py'), - PosixPath('build/lib/pathlib.py')] - -Navigating inside a directory tree:: - - >>> p = Path('/etc') - >>> q = p / 'init.d' / 'reboot' - >>> q - PosixPath('/etc/init.d/reboot') - >>> q.resolve() - PosixPath('/etc/rc.d/init.d/halt') - -Querying path properties:: - - >>> q.exists() - True - >>> q.is_dir() - False - -Opening a file:: - - >>> with q.open() as f: f.readline() - ... - '#!/bin/bash\n' - - -.. _pure-paths: - -Pure paths ----------- - -Pure path objects provide path-handling operations which don't actually -access a filesystem. There are three ways to access these classes, which -we also call *flavours*: - -.. class:: PurePath(*pathsegments) - - A generic class that represents the system's path flavour (instantiating - it creates either a :class:`PurePosixPath` or a :class:`PureWindowsPath`):: - - >>> PurePath('setup.py') # Running on a Unix machine - PurePosixPath('setup.py') - - Each element of *pathsegments* can be either a string representing a - path segment, an object implementing the :class:`os.PathLike` interface - which returns a string, or another path object:: - - >>> PurePath('foo', 'some/path', 'bar') - PurePosixPath('foo/some/path/bar') - >>> PurePath(Path('foo'), Path('bar')) - PurePosixPath('foo/bar') - - When *pathsegments* is empty, the current directory is assumed:: - - >>> PurePath() - PurePosixPath('.') - - When several absolute paths are given, the last is taken as an anchor - (mimicking :func:`os.path.join`'s behaviour):: - - >>> PurePath('/etc', '/usr', 'lib64') - PurePosixPath('/usr/lib64') - >>> PureWindowsPath('c:/Windows', 'd:bar') - PureWindowsPath('d:bar') - - However, in a Windows path, changing the local root doesn't discard the - previous drive setting:: - - >>> PureWindowsPath('c:/Windows', '/Program Files') - PureWindowsPath('c:/Program Files') - - Spurious slashes and single dots are collapsed, but double dots (``'..'``) - are not, since this would change the meaning of a path in the face of - symbolic links:: - - >>> PurePath('foo//bar') - PurePosixPath('foo/bar') - >>> PurePath('foo/./bar') - PurePosixPath('foo/bar') - >>> PurePath('foo/../bar') - PurePosixPath('foo/../bar') - - (a naïve approach would make ``PurePosixPath('foo/../bar')`` equivalent - to ``PurePosixPath('bar')``, which is wrong if ``foo`` is a symbolic link - to another directory) - - Pure path objects implement the :class:`os.PathLike` interface, allowing them - to be used anywhere the interface is accepted. - - .. versionchanged:: 3.6 - Added support for the :class:`os.PathLike` interface. - -.. class:: PurePosixPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents non-Windows - filesystem paths:: - - >>> PurePosixPath('/etc') - PurePosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PureWindowsPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents Windows - filesystem paths:: - - >>> PureWindowsPath('c:/Program Files/') - PureWindowsPath('c:/Program Files') - - *pathsegments* is specified similarly to :class:`PurePath`. - -Regardless of the system you're running on, you can instantiate all of -these classes, since they don't provide any operation that does system calls. - - -General properties -^^^^^^^^^^^^^^^^^^ - -Paths are immutable and hashable. Paths of a same flavour are comparable -and orderable. These properties respect the flavour's case-folding -semantics:: - - >>> PurePosixPath('foo') == PurePosixPath('FOO') - False - >>> PureWindowsPath('foo') == PureWindowsPath('FOO') - True - >>> PureWindowsPath('FOO') in { PureWindowsPath('foo') } - True - >>> PureWindowsPath('C:') < PureWindowsPath('d:') - True - -Paths of a different flavour compare unequal and cannot be ordered:: - - >>> PureWindowsPath('foo') == PurePosixPath('foo') - False - >>> PureWindowsPath('foo') < PurePosixPath('foo') - Traceback (most recent call last): - File "", line 1, in - TypeError: '<' not supported between instances of 'PureWindowsPath' and 'PurePosixPath' - - -Operators -^^^^^^^^^ - -The slash operator helps create child paths, similarly to :func:`os.path.join`:: - - >>> p = PurePath('/etc') - >>> p - PurePosixPath('/etc') - >>> p / 'init.d' / 'apache2' - PurePosixPath('/etc/init.d/apache2') - >>> q = PurePath('bin') - >>> '/usr' / q - PurePosixPath('/usr/bin') - -A path object can be used anywhere an object implementing :class:`os.PathLike` -is accepted:: - - >>> import os - >>> p = PurePath('/etc') - >>> os.fspath(p) - '/etc' - -The string representation of a path is the raw filesystem path itself -(in native form, e.g. with backslashes under Windows), which you can -pass to any function taking a file path as a string:: - - >>> p = PurePath('/etc') - >>> str(p) - '/etc' - >>> p = PureWindowsPath('c:/Program Files') - >>> str(p) - 'c:\\Program Files' - -Similarly, calling :class:`bytes` on a path gives the raw filesystem path as a -bytes object, as encoded by :func:`os.fsencode`:: - - >>> bytes(p) - b'/etc' - -.. note:: - Calling :class:`bytes` is only recommended under Unix. Under Windows, - the unicode form is the canonical representation of filesystem paths. - - -Accessing individual parts -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To access the individual "parts" (components) of a path, use the following -property: - -.. data:: PurePath.parts - - A tuple giving access to the path's various components:: - - >>> p = PurePath('/usr/bin/python3') - >>> p.parts - ('/', 'usr', 'bin', 'python3') - - >>> p = PureWindowsPath('c:/Program Files/PSF') - >>> p.parts - ('c:\\', 'Program Files', 'PSF') - - (note how the drive and local root are regrouped in a single part) - - -Methods and properties -^^^^^^^^^^^^^^^^^^^^^^ - -Pure paths provide the following methods and properties: - -.. data:: PurePath.drive - - A string representing the drive letter or name, if any:: - - >>> PureWindowsPath('c:/Program Files/').drive - 'c:' - >>> PureWindowsPath('/Program Files/').drive - '' - >>> PurePosixPath('/etc').drive - '' - - UNC shares are also considered drives:: - - >>> PureWindowsPath('//host/share/foo.txt').drive - '\\\\host\\share' - -.. data:: PurePath.root - - A string representing the (local or global) root, if any:: - - >>> PureWindowsPath('c:/Program Files/').root - '\\' - >>> PureWindowsPath('c:Program Files/').root - '' - >>> PurePosixPath('/etc').root - '/' - - UNC shares always have a root:: - - >>> PureWindowsPath('//host/share').root - '\\' - -.. data:: PurePath.anchor - - The concatenation of the drive and root:: - - >>> PureWindowsPath('c:/Program Files/').anchor - 'c:\\' - >>> PureWindowsPath('c:Program Files/').anchor - 'c:' - >>> PurePosixPath('/etc').anchor - '/' - >>> PureWindowsPath('//host/share').anchor - '\\\\host\\share\\' - - -.. data:: PurePath.parents - - An immutable sequence providing access to the logical ancestors of - the path:: - - >>> p = PureWindowsPath('c:/foo/bar/setup.py') - >>> p.parents[0] - PureWindowsPath('c:/foo/bar') - >>> p.parents[1] - PureWindowsPath('c:/foo') - >>> p.parents[2] - PureWindowsPath('c:/') - - -.. data:: PurePath.parent - - The logical parent of the path:: - - >>> p = PurePosixPath('/a/b/c/d') - >>> p.parent - PurePosixPath('/a/b/c') - - You cannot go past an anchor, or empty path:: - - >>> p = PurePosixPath('/') - >>> p.parent - PurePosixPath('/') - >>> p = PurePosixPath('.') - >>> p.parent - PurePosixPath('.') - - .. note:: - This is a purely lexical operation, hence the following behaviour:: - - >>> p = PurePosixPath('foo/..') - >>> p.parent - PurePosixPath('foo') - - If you want to walk an arbitrary filesystem path upwards, it is - recommended to first call :meth:`Path.resolve` so as to resolve - symlinks and eliminate `".."` components. - - -.. data:: PurePath.name - - A string representing the final path component, excluding the drive and - root, if any:: - - >>> PurePosixPath('my/library/setup.py').name - 'setup.py' - - UNC drive names are not considered:: - - >>> PureWindowsPath('//some/share/setup.py').name - 'setup.py' - >>> PureWindowsPath('//some/share').name - '' - - -.. data:: PurePath.suffix - - The file extension of the final component, if any:: - - >>> PurePosixPath('my/library/setup.py').suffix - '.py' - >>> PurePosixPath('my/library.tar.gz').suffix - '.gz' - >>> PurePosixPath('my/library').suffix - '' - - -.. data:: PurePath.suffixes - - A list of the path's file extensions:: - - >>> PurePosixPath('my/library.tar.gar').suffixes - ['.tar', '.gar'] - >>> PurePosixPath('my/library.tar.gz').suffixes - ['.tar', '.gz'] - >>> PurePosixPath('my/library').suffixes - [] - - -.. data:: PurePath.stem - - The final path component, without its suffix:: - - >>> PurePosixPath('my/library.tar.gz').stem - 'library.tar' - >>> PurePosixPath('my/library.tar').stem - 'library' - >>> PurePosixPath('my/library').stem - 'library' - - -.. method:: PurePath.as_posix() - - Return a string representation of the path with forward slashes (``/``):: - - >>> p = PureWindowsPath('c:\\windows') - >>> str(p) - 'c:\\windows' - >>> p.as_posix() - 'c:/windows' - - -.. method:: PurePath.as_uri() - - Represent the path as a ``file`` URI. :exc:`ValueError` is raised if - the path isn't absolute. - - >>> p = PurePosixPath('/etc/passwd') - >>> p.as_uri() - 'file:///etc/passwd' - >>> p = PureWindowsPath('c:/Windows') - >>> p.as_uri() - 'file:///c:/Windows' - - -.. method:: PurePath.is_absolute() - - Return whether the path is absolute or not. A path is considered absolute - if it has both a root and (if the flavour allows) a drive:: - - >>> PurePosixPath('/a/b').is_absolute() - True - >>> PurePosixPath('a/b').is_absolute() - False - - >>> PureWindowsPath('c:/a/b').is_absolute() - True - >>> PureWindowsPath('/a/b').is_absolute() - False - >>> PureWindowsPath('c:').is_absolute() - False - >>> PureWindowsPath('//some/share').is_absolute() - True - - -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. - - -.. method:: PurePath.joinpath(*other) - - Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: - - >>> PurePosixPath('/etc').joinpath('passwd') - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath(PurePosixPath('passwd')) - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath('init.d', 'apache2') - PurePosixPath('/etc/init.d/apache2') - >>> PureWindowsPath('c:').joinpath('/Program Files') - PureWindowsPath('c:/Program Files') - - -.. method:: PurePath.match(pattern) - - Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. - - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: - - >>> PurePath('a/b.py').match('*.py') - True - >>> PurePath('/a/b/c.py').match('b/*.py') - True - >>> PurePath('/a/b/c.py').match('a/*.py') - False - - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: - - >>> PurePath('/a.py').match('/*.py') - True - >>> PurePath('a/b.py').match('/*.py') - False - - As with other methods, case-sensitivity is observed:: - - >>> PureWindowsPath('b.py').match('*.PY') - True - - -.. method:: PurePath.relative_to(*other) - - Compute a version of this path relative to the path represented by - *other*. If it's impossible, ValueError is raised:: - - >>> p = PurePosixPath('/etc/passwd') - >>> p.relative_to('/') - PurePosixPath('etc/passwd') - >>> p.relative_to('/etc') - PurePosixPath('passwd') - >>> p.relative_to('/usr') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 694, in relative_to - .format(str(self), str(formatted))) - ValueError: '/etc/passwd' does not start with '/usr' - - -.. method:: PurePath.with_name(name) - - Return a new path with the :attr:`name` changed. If the original path - doesn't have a name, ValueError is raised:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_name('setup.py') - PureWindowsPath('c:/Downloads/setup.py') - >>> p = PureWindowsPath('c:/') - >>> p.with_name('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "/home/antoine/cpython/default/Lib/pathlib.py", line 751, in with_name - raise ValueError("%r has an empty name" % (self,)) - ValueError: PureWindowsPath('c:/') has an empty name - - -.. method:: PurePath.with_suffix(suffix) - - Return a new path with the :attr:`suffix` changed. If the original path - doesn't have a suffix, the new *suffix* is appended instead. If the - *suffix* is an empty string, the original suffix is removed:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_suffix('.bz2') - PureWindowsPath('c:/Downloads/pathlib.tar.bz2') - >>> p = PureWindowsPath('README') - >>> p.with_suffix('.txt') - PureWindowsPath('README.txt') - >>> p = PureWindowsPath('README.txt') - >>> p.with_suffix('') - PureWindowsPath('README') - - -.. _concrete-paths: - - -Concrete paths --------------- - -Concrete paths are subclasses of the pure path classes. In addition to -operations provided by the latter, they also provide methods to do system -calls on path objects. There are three ways to instantiate concrete paths: - -.. class:: Path(*pathsegments) - - A subclass of :class:`PurePath`, this class represents concrete paths of - the system's path flavour (instantiating it creates either a - :class:`PosixPath` or a :class:`WindowsPath`):: - - >>> Path('setup.py') - PosixPath('setup.py') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PosixPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PurePosixPath`, this class - represents concrete non-Windows filesystem paths:: - - >>> PosixPath('/etc') - PosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: WindowsPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PureWindowsPath`, this class - represents concrete Windows filesystem paths:: - - >>> WindowsPath('c:/Program Files/') - WindowsPath('c:/Program Files') - - *pathsegments* is specified similarly to :class:`PurePath`. - -You can only instantiate the class flavour that corresponds to your system -(allowing system calls on non-compatible path flavours could lead to -bugs or failures in your application):: - - >>> import os - >>> os.name - 'posix' - >>> Path('setup.py') - PosixPath('setup.py') - >>> PosixPath('setup.py') - PosixPath('setup.py') - >>> WindowsPath('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 798, in __new__ - % (cls.__name__,)) - NotImplementedError: cannot instantiate 'WindowsPath' on your system - - -Methods -^^^^^^^ - -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist): - -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct):: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - -.. method:: Path.stat() - - Return information about this path (similarly to :func:`os.stat`). - The result is looked up at each call to this method. - - >>> p = Path('setup.py') - >>> p.stat().st_size - 956 - >>> p.stat().st_mtime - 1327883547.852554 - - -.. method:: Path.chmod(mode) - - Change the file mode and permissions, like :func:`os.chmod`:: - - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 - - -.. method:: Path.exists() - - Whether the path points to an existing file or directory:: - - >>> Path('.').exists() - True - >>> Path('setup.py').exists() - True - >>> Path('/etc').exists() - True - >>> Path('nonexistentfile').exists() - False - - .. note:: - If the path points to a symlink, :meth:`exists` returns whether the - symlink *points to* an existing file or directory. - - -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`:: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - - .. versionadded:: 3.5 - - -.. method:: Path.glob(pattern) - - Glob the given *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: - - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - - The "``**``" pattern means "this directory and all subdirectories, - recursively". In other words, it enables recursive globbing:: - - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - -.. method:: Path.group() - - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. - - -.. method:: Path.is_dir() - - Return ``True`` if the path points to a directory (or a symbolic link - pointing to a directory), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_file() - - Return ``True`` if the path points to a regular file (or a symbolic link - pointing to a regular file), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_symlink() - - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. - - -.. method:: Path.is_socket() - - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_fifo() - - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_block_device() - - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_char_device() - - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.iterdir() - - When the path points to a directory, yield path objects of the directory - contents:: - - >>> p = Path('docs') - >>> for child in p.iterdir(): child - ... - PosixPath('docs/conf.py') - PosixPath('docs/_templates') - PosixPath('docs/make.bat') - PosixPath('docs/index.rst') - PosixPath('docs/_build') - PosixPath('docs/_static') - PosixPath('docs/Makefile') - -.. method:: Path.lchmod(mode) - - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. - - -.. method:: Path.lstat() - - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. - - -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) - - Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode - and access flags. If the path already exists, :exc:`FileExistsError` - is raised. - - If *parents* is true, any missing parents of this path are created - as needed; they are created with the default permissions without taking - *mode* into account (mimicking the POSIX ``mkdir -p`` command). - - If *parents* is false (the default), a missing parent raises - :exc:`FileNotFoundError`. - - If *exist_ok* is false (the default), :exc:`FileExistsError` is - raised if the target directory already exists. - - If *exist_ok* is true, :exc:`FileExistsError` exceptions will be - ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the - last path component is not an existing non-directory file. - - .. versionchanged:: 3.5 - The *exist_ok* parameter was added. - - -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: - - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' - - -.. method:: Path.owner() - - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. - - -.. method:: Path.read_bytes() - - Return the binary contents of the pointed-to file as a bytes object:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - .. versionadded:: 3.5 - - -.. method:: Path.read_text(encoding=None, errors=None) - - Return the decoded contents of the pointed-to file as a string:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. - - .. versionadded:: 3.5 - - -.. method:: Path.rename(target) - - Rename this file or directory to the given *target*. On Unix, if - *target* exists and is a file, it will be replaced silently if the user - has permission. *target* can be either a string or another path object:: - - >>> p = Path('foo') - >>> p.open('w').write('some text') - 9 - >>> target = Path('bar') - >>> p.rename(target) - >>> target.open().read() - 'some text' - - -.. method:: Path.replace(target) - - Rename this file or directory to the given *target*. If *target* points - to an existing file or directory, it will be unconditionally replaced. - - -.. method:: Path.resolve(strict=False) - - Make the path absolute, resolving any symlinks. A new path object is - returned:: - - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') - - "``..``" components are also eliminated (this is the only method to do so):: - - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - - If the path doesn't exist and *strict* is ``True``, :exc:`FileNotFoundError` - is raised. If *strict* is ``False``, the path is resolved as far as possible - and any remainder is appended without checking whether it exists. If an - infinite loop is encountered along the resolution path, :exc:`RuntimeError` - is raised. - - .. versionadded:: 3.6 - The *strict* argument. - -.. method:: Path.rglob(pattern) - - This is like calling :meth:`Path.glob` with "``**``" added in front of the - given *pattern*: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link to *target*. Under Windows, - *target_is_directory* must be true (default ``False``) if the link's target - is a directory. Under POSIX, *target_is_directory*'s value is ignored. - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. - - -.. method:: Path.unlink() - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. - - -.. method:: Path.write_bytes(data) - - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - An existing file of the same name is overwritten. - - .. versionadded:: 3.5 - - -.. method:: Path.write_text(data, encoding=None, errors=None) - - Open the file pointed to in text mode, write *data* to it, and close the - file:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - .. versionadded:: 3.5 diff --git a/third_party/python/Doc/library/pdb.rst b/third_party/python/Doc/library/pdb.rst deleted file mode 100644 index 6225a3a1f..000000000 --- a/third_party/python/Doc/library/pdb.rst +++ /dev/null @@ -1,519 +0,0 @@ -.. _debugger: - -:mod:`pdb` --- The Python Debugger -================================== - -.. module:: pdb - :synopsis: The Python debugger for interactive interpreters. - -**Source code:** :source:`Lib/pdb.py` - -.. index:: single: debugging - --------------- - -The module :mod:`pdb` defines an interactive source code debugger for Python -programs. It supports setting (conditional) breakpoints and single stepping at -the source line level, inspection of stack frames, source code listing, and -evaluation of arbitrary Python code in the context of any stack frame. It also -supports post-mortem debugging and can be called under program control. - -.. index:: - single: Pdb (class in pdb) - module: bdb - module: cmd - -The debugger is extensible -- it is actually defined as the class :class:`Pdb`. -This is currently undocumented but easily understood by reading the source. The -extension interface uses the modules :mod:`bdb` and :mod:`cmd`. - -The debugger's prompt is ``(Pdb)``. Typical usage to run a program under control -of the debugger is:: - - >>> import pdb - >>> import mymodule - >>> pdb.run('mymodule.test()') - > (0)?() - (Pdb) continue - > (1)?() - (Pdb) continue - NameError: 'spam' - > (1)?() - (Pdb) - -.. versionchanged:: 3.3 - Tab-completion via the :mod:`readline` module is available for commands and - command arguments, e.g. the current global and local names are offered as - arguments of the ``p`` command. - -:file:`pdb.py` can also be invoked as a script to debug other scripts. For -example:: - - python3 -m pdb myscript.py - -When invoked as a script, pdb will automatically enter post-mortem debugging if -the program being debugged exits abnormally. After post-mortem debugging (or -after normal exit of the program), pdb will restart the program. Automatic -restarting preserves pdb's state (such as breakpoints) and in most cases is more -useful than quitting the debugger upon program's exit. - -.. versionadded:: 3.2 - :file:`pdb.py` now accepts a ``-c`` option that executes commands as if given - in a :file:`.pdbrc` file, see :ref:`debugger-commands`. - -The typical usage to break into the debugger from a running program is to -insert :: - - import pdb; pdb.set_trace() - -at the location you want to break into the debugger. You can then step through -the code following this statement, and continue running without the debugger -using the :pdbcmd:`continue` command. - -The typical usage to inspect a crashed program is:: - - >>> import pdb - >>> import mymodule - >>> mymodule.test() - Traceback (most recent call last): - File "", line 1, in - File "./mymodule.py", line 4, in test - test2() - File "./mymodule.py", line 3, in test2 - print(spam) - NameError: spam - >>> pdb.pm() - > ./mymodule.py(3)test2() - -> print(spam) - (Pdb) - - -The module defines the following functions; each enters the debugger in a -slightly different way: - -.. function:: run(statement, globals=None, locals=None) - - Execute the *statement* (given as a string or a code object) under debugger - control. The debugger prompt appears before any code is executed; you can - set breakpoints and type :pdbcmd:`continue`, or you can step through the - statement using :pdbcmd:`step` or :pdbcmd:`next` (all these commands are - explained below). The optional *globals* and *locals* arguments specify the - environment in which the code is executed; by default the dictionary of the - module :mod:`__main__` is used. (See the explanation of the built-in - :func:`exec` or :func:`eval` functions.) - - -.. function:: runeval(expression, globals=None, locals=None) - - Evaluate the *expression* (given as a string or a code object) under debugger - control. When :func:`runeval` returns, it returns the value of the - expression. Otherwise this function is similar to :func:`run`. - - -.. function:: runcall(function, *args, **kwds) - - Call the *function* (a function or method object, not a string) with the - given arguments. When :func:`runcall` returns, it returns whatever the - function call returned. The debugger prompt appears as soon as the function - is entered. - - -.. function:: set_trace() - - Enter the debugger at the calling stack frame. This is useful to hard-code a - breakpoint at a given point in a program, even if the code is not otherwise - being debugged (e.g. when an assertion fails). - - -.. function:: post_mortem(traceback=None) - - Enter post-mortem debugging of the given *traceback* object. If no - *traceback* is given, it uses the one of the exception that is currently - being handled (an exception must be being handled if the default is to be - used). - - -.. function:: pm() - - Enter post-mortem debugging of the traceback found in - :data:`sys.last_traceback`. - - -The ``run*`` functions and :func:`set_trace` are aliases for instantiating the -:class:`Pdb` class and calling the method of the same name. If you want to -access further features, you have to do this yourself: - -.. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) - - :class:`Pdb` is the debugger class. - - The *completekey*, *stdin* and *stdout* arguments are passed to the - underlying :class:`cmd.Cmd` class; see the description there. - - The *skip* argument, if given, must be an iterable of glob-style module name - patterns. The debugger will not step into frames that originate in a module - that matches one of these patterns. [1]_ - - By default, Pdb sets a handler for the SIGINT signal (which is sent when the - user presses :kbd:`Ctrl-C` on the console) when you give a ``continue`` command. - This allows you to break into the debugger again by pressing :kbd:`Ctrl-C`. If you - want Pdb not to touch the SIGINT handler, set *nosigint* to true. - - The *readrc* argument defaults to true and controls whether Pdb will load - .pdbrc files from the filesystem. - - Example call to enable tracing with *skip*:: - - import pdb; pdb.Pdb(skip=['django.*']).set_trace() - - .. versionadded:: 3.1 - The *skip* argument. - - .. versionadded:: 3.2 - The *nosigint* argument. Previously, a SIGINT handler was never set by - Pdb. - - .. versionchanged:: 3.6 - The *readrc* argument. - - .. method:: run(statement, globals=None, locals=None) - runeval(expression, globals=None, locals=None) - runcall(function, *args, **kwds) - set_trace() - - See the documentation for the functions explained above. - - -.. _debugger-commands: - -Debugger Commands ------------------ - -The commands recognized by the debugger are listed below. Most commands can be -abbreviated to one or two letters as indicated; e.g. ``h(elp)`` means that -either ``h`` or ``help`` can be used to enter the help command (but not ``he`` -or ``hel``, nor ``H`` or ``Help`` or ``HELP``). Arguments to commands must be -separated by whitespace (spaces or tabs). Optional arguments are enclosed in -square brackets (``[]``) in the command syntax; the square brackets must not be -typed. Alternatives in the command syntax are separated by a vertical bar -(``|``). - -Entering a blank line repeats the last command entered. Exception: if the last -command was a :pdbcmd:`list` command, the next 11 lines are listed. - -Commands that the debugger doesn't recognize are assumed to be Python statements -and are executed in the context of the program being debugged. Python -statements can also be prefixed with an exclamation point (``!``). This is a -powerful way to inspect the program being debugged; it is even possible to -change a variable or call a function. When an exception occurs in such a -statement, the exception name is printed but the debugger's state is not -changed. - -The debugger supports :ref:`aliases `. Aliases can have -parameters which allows one a certain level of adaptability to the context under -examination. - -Multiple commands may be entered on a single line, separated by ``;;``. (A -single ``;`` is not used as it is the separator for multiple commands in a line -that is passed to the Python parser.) No intelligence is applied to separating -the commands; the input is split at the first ``;;`` pair, even if it is in the -middle of a quoted string. - -.. index:: - pair: .pdbrc; file - triple: debugger; configuration; file - -If a file :file:`.pdbrc` exists in the user's home directory or in the current -directory, it is read in and executed as if it had been typed at the debugger -prompt. This is particularly useful for aliases. If both files exist, the one -in the home directory is read first and aliases defined there can be overridden -by the local file. - -.. versionchanged:: 3.2 - :file:`.pdbrc` can now contain commands that continue debugging, such as - :pdbcmd:`continue` or :pdbcmd:`next`. Previously, these commands had no - effect. - - -.. pdbcommand:: h(elp) [command] - - Without argument, print the list of available commands. With a *command* as - argument, print help about that command. ``help pdb`` displays the full - documentation (the docstring of the :mod:`pdb` module). Since the *command* - argument must be an identifier, ``help exec`` must be entered to get help on - the ``!`` command. - -.. pdbcommand:: w(here) - - Print a stack trace, with the most recent frame at the bottom. An arrow - indicates the current frame, which determines the context of most commands. - -.. pdbcommand:: d(own) [count] - - Move the current frame *count* (default one) levels down in the stack trace - (to a newer frame). - -.. pdbcommand:: u(p) [count] - - Move the current frame *count* (default one) levels up in the stack trace (to - an older frame). - -.. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. - - If a second argument is present, it is an expression which must evaluate to - true before the breakpoint is honored. - - Without argument, list all breaks, including for each breakpoint, the number - of times that breakpoint has been hit, the current ignore count, and the - associated condition if any. - -.. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] - - Temporary breakpoint, which is removed automatically when it is first hit. - The arguments are the same as for :pdbcmd:`break`. - -.. pdbcommand:: cl(ear) [filename:lineno | bpnumber [bpnumber ...]] - - With a *filename:lineno* argument, clear all the breakpoints at this line. - With a space separated list of breakpoint numbers, clear those breakpoints. - Without argument, clear all breaks (but first ask confirmation). - -.. pdbcommand:: disable [bpnumber [bpnumber ...]] - - Disable the breakpoints given as a space separated list of breakpoint - numbers. Disabling a breakpoint means it cannot cause the program to stop - execution, but unlike clearing a breakpoint, it remains in the list of - breakpoints and can be (re-)enabled. - -.. pdbcommand:: enable [bpnumber [bpnumber ...]] - - Enable the breakpoints specified. - -.. pdbcommand:: ignore bpnumber [count] - - Set the ignore count for the given breakpoint number. If count is omitted, - the ignore count is set to 0. A breakpoint becomes active when the ignore - count is zero. When non-zero, the count is decremented each time the - breakpoint is reached and the breakpoint is not disabled and any associated - condition evaluates to true. - -.. pdbcommand:: condition bpnumber [condition] - - Set a new *condition* for the breakpoint, an expression which must evaluate - to true before the breakpoint is honored. If *condition* is absent, any - existing condition is removed; i.e., the breakpoint is made unconditional. - -.. pdbcommand:: commands [bpnumber] - - Specify a list of commands for breakpoint number *bpnumber*. The commands - themselves appear on the following lines. Type a line containing just - ``end`` to terminate the commands. An example:: - - (Pdb) commands 1 - (com) p some_variable - (com) end - (Pdb) - - To remove all commands from a breakpoint, type commands and follow it - immediately with ``end``; that is, give no commands. - - With no *bpnumber* argument, commands refers to the last breakpoint set. - - You can use breakpoint commands to start your program up again. Simply use - the continue command, or step, or any other command that resumes execution. - - Specifying any command resuming execution (currently continue, step, next, - return, jump, quit and their abbreviations) terminates the command list (as if - that command was immediately followed by end). This is because any time you - resume execution (even with a simple next or step), you may encounter another - breakpoint—which could have its own command list, leading to ambiguities about - which list to execute. - - If you use the 'silent' command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. - -.. pdbcommand:: s(tep) - - Execute the current line, stop at the first possible occasion (either in a - function that is called or on the next line in the current function). - -.. pdbcommand:: n(ext) - - Continue execution until the next line in the current function is reached or - it returns. (The difference between :pdbcmd:`next` and :pdbcmd:`step` is - that :pdbcmd:`step` stops inside a called function, while :pdbcmd:`next` - executes called functions at (nearly) full speed, only stopping at the next - line in the current function.) - -.. pdbcommand:: unt(il) [lineno] - - Without argument, continue execution until the line with a number greater - than the current one is reached. - - With a line number, continue execution until a line with a number greater or - equal to that is reached. In both cases, also stop when the current frame - returns. - - .. versionchanged:: 3.2 - Allow giving an explicit line number. - -.. pdbcommand:: r(eturn) - - Continue execution until the current function returns. - -.. pdbcommand:: c(ont(inue)) - - Continue execution, only stop when a breakpoint is encountered. - -.. pdbcommand:: j(ump) lineno - - Set the next line that will be executed. Only available in the bottom-most - frame. This lets you jump back and execute code again, or jump forward to - skip code that you don't want to run. - - It should be noted that not all jumps are allowed -- for instance it is not - possible to jump into the middle of a :keyword:`for` loop or out of a - :keyword:`finally` clause. - -.. pdbcommand:: l(ist) [first[, last]] - - List source code for the current file. Without arguments, list 11 lines - around the current line or continue the previous listing. With ``.`` as - argument, list 11 lines around the current line. With one argument, - list 11 lines around at that line. With two arguments, list the given range; - if the second argument is less than the first, it is interpreted as a count. - - The current line in the current frame is indicated by ``->``. If an - exception is being debugged, the line where the exception was originally - raised or propagated is indicated by ``>>``, if it differs from the current - line. - - .. versionadded:: 3.2 - The ``>>`` marker. - -.. pdbcommand:: ll | longlist - - List all source code for the current function or frame. Interesting lines - are marked as for :pdbcmd:`list`. - - .. versionadded:: 3.2 - -.. pdbcommand:: a(rgs) - - Print the argument list of the current function. - -.. pdbcommand:: p expression - - Evaluate the *expression* in the current context and print its value. - - .. note:: - - ``print()`` can also be used, but is not a debugger command --- this executes the - Python :func:`print` function. - - -.. pdbcommand:: pp expression - - Like the :pdbcmd:`p` command, except the value of the expression is - pretty-printed using the :mod:`pprint` module. - -.. pdbcommand:: whatis expression - - Print the type of the *expression*. - -.. pdbcommand:: source expression - - Try to get source code for the given object and display it. - - .. versionadded:: 3.2 - -.. pdbcommand:: display [expression] - - Display the value of the expression if it changed, each time execution stops - in the current frame. - - Without expression, list all display expressions for the current frame. - - .. versionadded:: 3.2 - -.. pdbcommand:: undisplay [expression] - - Do not display the expression any more in the current frame. Without - expression, clear all display expressions for the current frame. - - .. versionadded:: 3.2 - -.. pdbcommand:: interact - - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. - - .. versionadded:: 3.2 - -.. _debugger-aliases: - -.. pdbcommand:: alias [name [command]] - - Create an alias called *name* that executes *command*. The command must - *not* be enclosed in quotes. Replaceable parameters can be indicated by - ``%1``, ``%2``, and so on, while ``%*`` is replaced by all the parameters. - If no command is given, the current alias for *name* is shown. If no - arguments are given, all aliases are listed. - - Aliases may be nested and can contain anything that can be legally typed at - the pdb prompt. Note that internal pdb commands *can* be overridden by - aliases. Such a command is then hidden until the alias is removed. Aliasing - is recursively applied to the first word of the command line; all other words - in the line are left alone. - - As an example, here are two useful aliases (especially when placed in the - :file:`.pdbrc` file):: - - # Print instance variables (usage "pi classInst") - alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k]) - # Print instance variables in self - alias ps pi self - -.. pdbcommand:: unalias name - - Delete the specified alias. - -.. pdbcommand:: ! statement - - Execute the (one-line) *statement* in the context of the current stack frame. - The exclamation point can be omitted unless the first word of the statement - resembles a debugger command. To set a global variable, you can prefix the - assignment command with a :keyword:`global` statement on the same line, - e.g.:: - - (Pdb) global list_options; list_options = ['-l'] - (Pdb) - -.. pdbcommand:: run [args ...] - restart [args ...] - - Restart the debugged Python program. If an argument is supplied, it is split - with :mod:`shlex` and the result is used as the new :data:`sys.argv`. - History, breakpoints, actions and debugger options are preserved. - :pdbcmd:`restart` is an alias for :pdbcmd:`run`. - -.. pdbcommand:: q(uit) - - Quit from the debugger. The program being executed is aborted. - - -.. rubric:: Footnotes - -.. [1] Whether a frame is considered to originate in a certain module - is determined by the ``__name__`` in the frame globals. diff --git a/third_party/python/Doc/library/persistence.rst b/third_party/python/Doc/library/persistence.rst deleted file mode 100644 index d5bb193fa..000000000 --- a/third_party/python/Doc/library/persistence.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _persistence: - -**************** -Data Persistence -**************** - -The modules described in this chapter support storing Python data in a -persistent form on disk. The :mod:`pickle` and :mod:`marshal` modules can turn -many Python data types into a stream of bytes and then recreate the objects from -the bytes. The various DBM-related modules support a family of hash-based file -formats that store a mapping of strings to other strings. - -The list of modules described in this chapter is: - - -.. toctree:: - - pickle.rst - copyreg.rst - shelve.rst - marshal.rst - dbm.rst - sqlite3.rst diff --git a/third_party/python/Doc/library/pickle.rst b/third_party/python/Doc/library/pickle.rst deleted file mode 100644 index 52cbb6241..000000000 --- a/third_party/python/Doc/library/pickle.rst +++ /dev/null @@ -1,935 +0,0 @@ -:mod:`pickle` --- Python object serialization -============================================= - -.. module:: pickle - :synopsis: Convert Python objects to streams of bytes and back. - -.. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw - -**Source code:** :source:`Lib/pickle.py` - -.. index:: - single: persistence - pair: persistent; objects - pair: serializing; objects - pair: marshalling; objects - pair: flattening; objects - pair: pickling; objects - --------------- - -The :mod:`pickle` module implements binary protocols for serializing and -de-serializing a Python object structure. *"Pickling"* is the process -whereby a Python object hierarchy is converted into a byte stream, and -*"unpickling"* is the inverse operation, whereby a byte stream -(from a :term:`binary file` or :term:`bytes-like object`) is converted -back into an object hierarchy. Pickling (and unpickling) is alternatively -known as "serialization", "marshalling," [#]_ or "flattening"; however, to -avoid confusion, the terms used here are "pickling" and "unpickling". - -.. warning:: - - The :mod:`pickle` module is not secure against erroneous or maliciously - constructed data. Never unpickle data received from an untrusted or - unauthenticated source. - - -Relationship to other Python modules ------------------------------------- - -Comparison with ``marshal`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python -objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` -files. - -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. - -* :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, - however the class definition must be importable and live in the same module as - when the object was stored. - -* The :mod:`marshal` serialization format is not guaranteed to be portable - across Python versions. Because its primary job in life is to support - :file:`.pyc` files, the Python implementers reserve the right to change the - serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible - across Python releases. - -Comparison with ``json`` -^^^^^^^^^^^^^^^^^^^^^^^^ - -There are fundamental differences between the pickle protocols and -`JSON (JavaScript Object Notation) `_: - -* JSON is a text serialization format (it outputs unicode text, although - most of the time it is then encoded to ``utf-8``), while pickle is - a binary serialization format; - -* JSON is human-readable, while pickle is not; - -* JSON is interoperable and widely used outside of the Python ecosystem, - while pickle is Python-specific; - -* JSON, by default, can only represent a subset of the Python built-in - types, and no custom classes; pickle can represent an extremely large - number of Python types (many of them automatically, by clever usage - of Python's introspection facilities; complex cases can be tackled by - implementing :ref:`specific object APIs `). - -.. seealso:: - The :mod:`json` module: a standard library module allowing JSON - serialization and deserialization. - - -.. _pickle-protocols: - -Data stream format ------------------- - -.. index:: - single: External Data Representation - -The data format used by :mod:`pickle` is Python-specific. This has the -advantage that there are no restrictions imposed by external standards such as -JSON or XDR (which can't represent pointer sharing); however it means that -non-Python programs may not be able to reconstruct pickled Python objects. - -By default, the :mod:`pickle` data format uses a relatively compact binary -representation. If you need optimal size characteristics, you can efficiently -:doc:`compress ` pickled data. - -The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive -comments about opcodes used by pickle protocols. - -There are currently 5 different protocols which can be used for pickling. -The higher the protocol used, the more recent the version of Python needed -to read the pickle produced. - -* Protocol version 0 is the original "human-readable" protocol and is - backwards compatible with earlier versions of Python. - -* Protocol version 1 is an old binary format which is also compatible with - earlier versions of Python. - -* Protocol version 2 was introduced in Python 2.3. It provides much more - efficient pickling of :term:`new-style class`\es. Refer to :pep:`307` for - information about improvements brought by protocol 2. - -* Protocol version 3 was added in Python 3.0. It has explicit support for - :class:`bytes` objects and cannot be unpickled by Python 2.x. This is - the default protocol, and the recommended protocol when compatibility with - other Python 3 versions is required. - -* Protocol version 4 was added in Python 3.4. It adds support for very large - objects, pickling more kinds of objects, and some data format - optimizations. Refer to :pep:`3154` for information about improvements - brought by protocol 4. - -.. note:: - Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of - naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex - object into a byte stream and it can transform the byte stream into an object - with the same internal structure. Perhaps the most obvious thing to do with - these byte streams is to write them onto a file, but it is also conceivable to - send them across a network or store them in a database. The :mod:`shelve` - module provides a simple interface to pickle and unpickle objects on - DBM-style database files. - - -Module Interface ----------------- - -To serialize an object hierarchy, you simply call the :func:`dumps` function. -Similarly, to de-serialize a data stream, you call the :func:`loads` function. -However, if you want more control over serialization and de-serialization, -you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. - -The :mod:`pickle` module provides the following constants: - - -.. data:: HIGHEST_PROTOCOL - - An integer, the highest :ref:`protocol version ` - available. This value can be passed as a *protocol* value to functions - :func:`dump` and :func:`dumps` as well as the :class:`Pickler` - constructor. - -.. data:: DEFAULT_PROTOCOL - - An integer, the default :ref:`protocol version ` used - for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 3, a new protocol designed for Python 3. - - -The :mod:`pickle` module provides the following functions to make the pickling -process more convenient: - -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True) - - Write a pickled representation of *obj* to the open :term:`file object` *file*. - This is equivalent to ``Pickler(file, protocol).dump(obj)``. - - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. - - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. - -.. function:: dumps(obj, protocol=None, \*, fix_imports=True) - - Return the pickled representation of the object as a :class:`bytes` object, - instead of writing it to a file. - - Arguments *protocol* and *fix_imports* have the same meaning as in - :func:`dump`. - -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") - - Read a pickled object representation from the open :term:`file object` - *file* and return the reconstituted object hierarchy specified therein. - This is equivalent to ``Unpickler(file).load()``. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. - - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened for - binary reading, an :class:`io.BytesIO` object, or any other custom object - that meets this interface. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. - -.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") - - Read a pickled object hierarchy from a :class:`bytes` object and return the - reconstituted object hierarchy specified therein. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. - - -The :mod:`pickle` module defines three exceptions: - -.. exception:: PickleError - - Common base class for the other pickling exceptions. It inherits - :exc:`Exception`. - -.. exception:: PicklingError - - Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. - - Refer to :ref:`pickle-picklable` to learn what kinds of objects can be - pickled. - -.. exception:: UnpicklingError - - Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. - - Note that other exceptions may also be raised during unpickling, including - (but not necessarily limited to) AttributeError, EOFError, ImportError, and - IndexError. - - -The :mod:`pickle` module exports two classes, :class:`Pickler` and -:class:`Unpickler`: - -.. class:: Pickler(file, protocol=None, \*, fix_imports=True) - - This takes a binary file for writing a pickle data stream. - - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. - - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. - - .. method:: dump(obj) - - Write a pickled representation of *obj* to the open file object given in - the constructor. - - .. method:: persistent_id(obj) - - Do nothing by default. This exists so a subclass can override it. - - If :meth:`persistent_id` returns ``None``, *obj* is pickled as usual. Any - other value causes :class:`Pickler` to emit the returned value as a - persistent ID for *obj*. The meaning of this persistent ID should be - defined by :meth:`Unpickler.persistent_load`. Note that the value - returned by :meth:`persistent_id` cannot itself have a persistent ID. - - See :ref:`pickle-persistent` for details and examples of uses. - - .. attribute:: dispatch_table - - A pickler object's dispatch table is a registry of *reduction - functions* of the kind which can be declared using - :func:`copyreg.pickle`. It is a mapping whose keys are classes - and whose values are reduction functions. A reduction function - takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` - method. - - By default, a pickler object will not have a - :attr:`dispatch_table` attribute, and it will instead use the - global dispatch table managed by the :mod:`copyreg` module. - However, to customize the pickling for a specific pickler object - one can set the :attr:`dispatch_table` attribute to a dict-like - object. Alternatively, if a subclass of :class:`Pickler` has a - :attr:`dispatch_table` attribute then this will be used as the - default dispatch table for instances of that class. - - See :ref:`pickle-dispatch` for usage examples. - - .. versionadded:: 3.3 - - .. attribute:: fast - - Deprecated. Enable fast mode if set to a true value. The fast mode - disables the usage of memo, therefore speeding the pickling process by not - generating superfluous PUT opcodes. It should not be used with - self-referential objects, doing otherwise will cause :class:`Pickler` to - recurse infinitely. - - Use :func:`pickletools.optimize` if you need more compact pickles. - - -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") - - This takes a binary file for reading a pickle data stream. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. - - The argument *file* must have two methods, a read() method that takes an - integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object - opened for binary reading, an :class:`io.BytesIO` object, or any other - custom object that meets this interface. - - Optional keyword arguments are *fix_imports*, *encoding* and *errors*, - which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and - *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - - .. method:: load() - - Read a pickled object representation from the open file object given in - the constructor, and return the reconstituted object hierarchy specified - therein. Bytes past the pickled object's representation are ignored. - - .. method:: persistent_load(pid) - - Raise an :exc:`UnpicklingError` by default. - - If defined, :meth:`persistent_load` should return the object specified by - the persistent ID *pid*. If an invalid persistent ID is encountered, an - :exc:`UnpicklingError` should be raised. - - See :ref:`pickle-persistent` for details and examples of uses. - - .. method:: find_class(module, name) - - Import *module* if necessary and return the object called *name* from it, - where the *module* and *name* arguments are :class:`str` objects. Note, - unlike its name suggests, :meth:`find_class` is also used for finding - functions. - - Subclasses may override this to gain control over what type of objects and - how they can be loaded, potentially reducing security risks. Refer to - :ref:`pickle-restrict` for details. - - -.. _pickle-picklable: - -What can be pickled and unpickled? ----------------------------------- - -The following types can be pickled: - -* ``None``, ``True``, and ``False`` - -* integers, floating point numbers, complex numbers - -* strings, bytes, bytearrays - -* tuples, lists, sets, and dictionaries containing only picklable objects - -* functions defined at the top level of a module (using :keyword:`def`, not - :keyword:`lambda`) - -* built-in functions defined at the top level of a module - -* classes that are defined at the top level of a module - -* instances of such classes whose :attr:`~object.__dict__` or the result of - calling :meth:`__getstate__` is picklable (see section :ref:`pickle-inst` for - details). - -Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` -exception; when this happens, an unspecified number of bytes may have already -been written to the underlying file. Trying to pickle a highly recursive data -structure may exceed the maximum recursion depth, a :exc:`RecursionError` will be -raised in this case. You can carefully raise this limit with -:func:`sys.setrecursionlimit`. - -Note that functions (built-in and user-defined) are pickled by "fully qualified" -name reference, not by value. [#]_ This means that only the function name is -pickled, along with the name of the module the function is defined in. Neither -the function's code, nor any of its function attributes are pickled. Thus the -defining module must be importable in the unpickling environment, and the module -must contain the named object, otherwise an exception will be raised. [#]_ - -Similarly, classes are pickled by named reference, so the same restrictions in -the unpickling environment apply. Note that none of the class's code or data is -pickled, so in the following example the class attribute ``attr`` is not -restored in the unpickling environment:: - - class Foo: - attr = 'A class attribute' - - picklestring = pickle.dumps(Foo) - -These restrictions are why picklable functions and classes must be defined in -the top level of a module. - -Similarly, when class instances are pickled, their class's code and data are not -pickled along with them. Only the instance data are pickled. This is done on -purpose, so you can fix bugs in a class or add methods to the class and still -load objects that were created with an earlier version of the class. If you -plan to have long-lived objects that will see many versions of a class, it may -be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. - - -.. _pickle-inst: - -Pickling Class Instances ------------------------- - -.. currentmodule:: None - -In this section, we describe the general mechanisms available to you to define, -customize, and control how class instances are pickled and unpickled. - -In most cases, no additional code is needed to make instances picklable. By -default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method -is usually *not* invoked. The default behaviour first creates an uninitialized -instance and then restores the saved attributes. The following code shows an -implementation of this behaviour:: - - def save(obj): - return (obj.__class__, obj.__dict__) - - def load(cls, attributes): - obj = cls.__new__(cls) - obj.__dict__.update(attributes) - return obj - -Classes can alter the default behaviour by providing one or several special -methods: - -.. method:: object.__getnewargs_ex__() - - In protocols 2 and newer, classes that implements the - :meth:`__getnewargs_ex__` method can dictate the values passed to the - :meth:`__new__` method upon unpickling. The method must return a pair - ``(args, kwargs)`` where *args* is a tuple of positional arguments - and *kwargs* a dictionary of named arguments for constructing the - object. Those will be passed to the :meth:`__new__` method upon - unpickling. - - You should implement this method if the :meth:`__new__` method of your - class requires keyword-only arguments. Otherwise, it is recommended for - compatibility to implement :meth:`__getnewargs__`. - - .. versionchanged:: 3.6 - :meth:`__getnewargs_ex__` is now used in protocols 2 and 3. - - -.. method:: object.__getnewargs__() - - This method serves a similar purpose as :meth:`__getnewargs_ex__`, but - supports only positional arguments. It must return a tuple of arguments - ``args`` which will be passed to the :meth:`__new__` method upon unpickling. - - :meth:`__getnewargs__` will not be called if :meth:`__getnewargs_ex__` is - defined. - - .. versionchanged:: 3.6 - Before Python 3.6, :meth:`__getnewargs__` was called instead of - :meth:`__getnewargs_ex__` in protocols 2 and 3. - - -.. method:: object.__getstate__() - - Classes can further influence how their instances are pickled; if the class - defines the method :meth:`__getstate__`, it is called and the returned object - is pickled as the contents for the instance, instead of the contents of the - instance's dictionary. If the :meth:`__getstate__` method is absent, the - instance's :attr:`~object.__dict__` is pickled as usual. - - -.. method:: object.__setstate__(state) - - Upon unpickling, if the class defines :meth:`__setstate__`, it is called with - the unpickled state. In that case, there is no requirement for the state - object to be a dictionary. Otherwise, the pickled state must be a dictionary - and its items are assigned to the new instance's dictionary. - - .. note:: - - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. - - -Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. - -.. note:: - - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the - instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__getnewargs__` or - :meth:`__getnewargs_ex__` to establish such an invariant; otherwise, - neither :meth:`__new__` nor :meth:`__init__` will be called. - -.. index:: pair: copy; protocol - -As we shall see, pickle does not use directly the methods described above. In -fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified -interface for retrieving the data necessary for pickling and copying -objects. [#]_ - -Although powerful, implementing :meth:`__reduce__` directly in your classes is -error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling -or both. - -.. method:: object.__reduce__() - - The interface is currently defined as follows. The :meth:`__reduce__` method - takes no argument and shall return either a string or preferably a tuple (the - returned object is often referred to as the "reduce value"). - - If a string is returned, the string should be interpreted as the name of a - global variable. It should be the object's local name relative to its - module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. - - When a tuple is returned, it must be between two and five items long. - Optional items can either be omitted, or ``None`` can be provided as their - value. The semantics of each item are in order: - - .. XXX Mention __newobj__ special-case? - - * A callable object that will be called to create the initial version of the - object. - - * A tuple of arguments for the callable object. An empty tuple must be given - if the callable does not accept any argument. - - * Optionally, the object's state, which will be passed to the object's - :meth:`__setstate__` method as previously described. If the object has no - such method then, the value must be a dictionary and it will be added to - the object's :attr:`~object.__dict__` attribute. - - * Optionally, an iterator (and not a sequence) yielding successive items. - These items will be appended to the object either using - ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. - This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is - used depends on which pickle protocol version is used as well as the number - of items to append, so both must be supported.) - - * Optionally, an iterator (not a sequence) yielding successive key-value - pairs. These items will be stored to the object using ``obj[key] = - value``. This is primarily used for dictionary subclasses, but may be used - by other classes as long as they implement :meth:`__setitem__`. - - -.. method:: object.__reduce_ex__(protocol) - - Alternatively, a :meth:`__reduce_ex__` method may be defined. The only - difference is this method should take a single integer argument, the protocol - version. When defined, pickle will prefer it over the :meth:`__reduce__` - method. In addition, :meth:`__reduce__` automatically becomes a synonym for - the extended version. The main use for this method is to provide - backwards-compatible reduce values for older Python releases. - -.. currentmodule:: pickle - -.. _pickle-persistent: - -Persistence of External Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: persistent_id (pickle protocol) - single: persistent_load (pickle protocol) - -For the benefit of object persistence, the :mod:`pickle` module supports the -notion of a reference to an object outside the pickled data stream. Such -objects are referenced by a persistent ID, which should be either a string of -alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for -any newer protocol). - -The resolution of such persistent IDs is not defined by the :mod:`pickle` -module; it will delegate this resolution to the user defined methods on the -pickler and unpickler, :meth:`~Pickler.persistent_id` and -:meth:`~Unpickler.persistent_load` respectively. - -To pickle objects that have an external persistent id, the pickler must have a -custom :meth:`~Pickler.persistent_id` method that takes an object as an -argument and returns either ``None`` or the persistent id for that object. -When ``None`` is returned, the pickler simply pickles the object as normal. -When a persistent ID string is returned, the pickler will pickle that object, -along with a marker so that the unpickler will recognize it as a persistent ID. - -To unpickle external objects, the unpickler must have a custom -:meth:`~Unpickler.persistent_load` method that takes a persistent ID object and -returns the referenced object. - -Here is a comprehensive example presenting how persistent ID can be used to -pickle external objects by reference. - -.. literalinclude:: ../includes/dbpickle.py - -.. _pickle-dispatch: - -Dispatch Tables -^^^^^^^^^^^^^^^ - -If one wants to customize pickling of some classes without disturbing -any other code which depends on pickling, then one can create a -pickler with a private dispatch table. - -The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a -private dispatch table. - -For example :: - - f = io.BytesIO() - p = pickle.Pickler(f) - p.dispatch_table = copyreg.dispatch_table.copy() - p.dispatch_table[SomeClass] = reduce_SomeClass - -creates an instance of :class:`pickle.Pickler` with a private dispatch -table which handles the ``SomeClass`` class specially. Alternatively, -the code :: - - class MyPickler(pickle.Pickler): - dispatch_table = copyreg.dispatch_table.copy() - dispatch_table[SomeClass] = reduce_SomeClass - f = io.BytesIO() - p = MyPickler(f) - -does the same, but all instances of ``MyPickler`` will by default -share the same dispatch table. The equivalent code using the -:mod:`copyreg` module is :: - - copyreg.pickle(SomeClass, reduce_SomeClass) - f = io.BytesIO() - p = pickle.Pickler(f) - -.. _pickle-state: - -Handling Stateful Objects -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __getstate__() (copy protocol) - single: __setstate__() (copy protocol) - -Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and -line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object -member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: - - class TextReader: - """Print and number lines in a text file.""" - - def __init__(self, filename): - self.filename = filename - self.file = open(filename) - self.lineno = 0 - - def readline(self): - self.lineno += 1 - line = self.file.readline() - if not line: - return None - if line.endswith('\n'): - line = line[:-1] - return "%i: %s" % (self.lineno, line) - - def __getstate__(self): - # Copy the object's state from self.__dict__ which contains - # all our instance attributes. Always use the dict.copy() - # method to avoid modifying the original state. - state = self.__dict__.copy() - # Remove the unpicklable entries. - del state['file'] - return state - - def __setstate__(self, state): - # Restore instance attributes (i.e., filename and lineno). - self.__dict__.update(state) - # Restore the previously opened file's state. To do so, we need to - # reopen it and read from it until the line count is restored. - file = open(self.filename) - for _ in range(self.lineno): - file.readline() - # Finally, save the file. - self.file = file - - -A sample usage might be something like this:: - - >>> reader = TextReader("hello.txt") - >>> reader.readline() - '1: Hello world!' - >>> reader.readline() - '2: I am line number two.' - >>> new_reader = pickle.loads(pickle.dumps(reader)) - >>> new_reader.readline() - '3: Goodbye!' - - -.. _pickle-restrict: - -Restricting Globals -------------------- - -.. index:: - single: find_class() (pickle protocol) - -By default, unpickling will import any class or function that it finds in the -pickle data. For many applications, this behaviour is unacceptable as it -permits the unpickler to import and invoke arbitrary code. Just consider what -this hand-crafted pickle data stream does when loaded:: - - >>> import pickle - >>> pickle.loads(b"cos\nsystem\n(S'echo hello world'\ntR.") - hello world - 0 - -In this example, the unpickler imports the :func:`os.system` function and then -apply the string argument "echo hello world". Although this example is -inoffensive, it is not difficult to imagine one that could damage your system. - -For this reason, you may want to control what gets unpickled by customizing -:meth:`Unpickler.find_class`. Unlike its name suggests, -:meth:`Unpickler.find_class` is called whenever a global (i.e., a class or -a function) is requested. Thus it is possible to either completely forbid -globals or restrict them to a safe subset. - -Here is an example of an unpickler allowing only few safe classes from the -:mod:`builtins` module to be loaded:: - - import builtins - import io - import pickle - - safe_builtins = { - 'range', - 'complex', - 'set', - 'frozenset', - 'slice', - } - - class RestrictedUnpickler(pickle.Unpickler): - - def find_class(self, module, name): - # Only allow safe classes from builtins. - if module == "builtins" and name in safe_builtins: - return getattr(builtins, name) - # Forbid everything else. - raise pickle.UnpicklingError("global '%s.%s' is forbidden" % - (module, name)) - - def restricted_loads(s): - """Helper function analogous to pickle.loads().""" - return RestrictedUnpickler(io.BytesIO(s)).load() - -A sample usage of our unpickler working has intended:: - - >>> restricted_loads(pickle.dumps([1, 2, range(15)])) - [1, 2, range(0, 15)] - >>> restricted_loads(b"cos\nsystem\n(S'echo hello world'\ntR.") - Traceback (most recent call last): - ... - pickle.UnpicklingError: global 'os.system' is forbidden - >>> restricted_loads(b'cbuiltins\neval\n' - ... b'(S\'getattr(__import__("os"), "system")' - ... b'("echo hello world")\'\ntR.') - Traceback (most recent call last): - ... - pickle.UnpicklingError: global 'builtins.eval' is forbidden - - -.. XXX Add note about how extension codes could evade our protection - mechanism (e.g. cached classes do not invokes find_class()). - -As our examples shows, you have to be careful with what you allow to be -unpickled. Therefore if security is a concern, you may want to consider -alternatives such as the marshalling API in :mod:`xmlrpc.client` or -third-party solutions. - - -Performance ------------ - -Recent versions of the pickle protocol (from protocol 2 and upwards) feature -efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. - - -.. _pickle-example: - -Examples --------- - -For the simplest code, use the :func:`dump` and :func:`load` functions. :: - - import pickle - - # An arbitrary collection of objects supported by pickle. - data = { - 'a': [1, 2.0, 3, 4+6j], - 'b': ("character string", b"byte string"), - 'c': {None, True, False} - } - - with open('data.pickle', 'wb') as f: - # Pickle the 'data' dictionary using the highest protocol available. - pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) - - -The following example reads the resulting pickled data. :: - - import pickle - - with open('data.pickle', 'rb') as f: - # The protocol version used is detected automatically, so we do not - # have to specify it. - data = pickle.load(f) - - -.. XXX: Add examples showing how to optimize pickles for size (like using -.. pickletools.optimize() or the gzip module). - - -.. seealso:: - - Module :mod:`copyreg` - Pickle interface constructor registration for extension types. - - Module :mod:`pickletools` - Tools for working with and analyzing pickled data. - - Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. - - Module :mod:`copy` - Shallow and deep object copying. - - Module :mod:`marshal` - High-performance serialization of built-in types. - - -.. rubric:: Footnotes - -.. [#] Don't confuse this with the :mod:`marshal` module - -.. [#] This is why :keyword:`lambda` functions cannot be pickled: all - :keyword:`lambda` functions share the same name: ````. - -.. [#] The exception raised will likely be an :exc:`ImportError` or an - :exc:`AttributeError` but it could be something else. - -.. [#] The :mod:`copy` module uses this protocol for shallow and deep copying - operations. - -.. [#] The limitation on alphanumeric characters is due to the fact - the persistent IDs, in protocol 0, are delimited by the newline - character. Therefore if any kind of newline characters occurs in - persistent IDs, the resulting pickle will become unreadable. diff --git a/third_party/python/Doc/library/pickletools.rst b/third_party/python/Doc/library/pickletools.rst deleted file mode 100644 index 480f4a6d3..000000000 --- a/third_party/python/Doc/library/pickletools.rst +++ /dev/null @@ -1,110 +0,0 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== - -.. module:: pickletools - :synopsis: Contains extensive comments about the pickle protocols and - pickle-machine opcodes, as well as some useful functions. - -**Source code:** :source:`Lib/pickletools.py` - --------------- - - -This module contains various constants relating to the intimate details of the -:mod:`pickle` module, some lengthy comments about the implementation, and a -few useful functions for analyzing pickled data. The contents of this module -are useful for Python core developers who are working on the :mod:`pickle`; -ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. - -Command line usage ------------------- - -.. versionadded:: 3.2 - -When invoked from the command line, ``python -m pickletools`` will -disassemble the contents of one or more pickle files. Note that if -you want to see the Python object stored in the pickle rather than the -details of pickle format, you may want to use ``-m pickle`` instead. -However, when the pickle file that you want to examine comes from an -untrusted source, ``-m pickletools`` is a safer option because it does -not execute pickle bytecode. - -For example, with a tuple ``(1, 2)`` pickled in file ``x.pickle``: - -.. code-block:: shell-session - - $ python -m pickle x.pickle - (1, 2) - - $ python -m pickletools x.pickle - 0: \x80 PROTO 3 - 2: K BININT1 1 - 4: K BININT1 2 - 6: \x86 TUPLE2 - 7: q BINPUT 0 - 9: . STOP - highest protocol among opcodes = 2 - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. program:: pickletools - -.. cmdoption:: -a, --annotate - - Annotate each line with a short opcode description. - -.. cmdoption:: -o, --output= - - Name of a file where the output should be written. - -.. cmdoption:: -l, --indentlevel= - - The number of blanks by which to indent a new MARK level. - -.. cmdoption:: -m, --memo - - When multiple objects are disassembled, preserve memo between - disassemblies. - -.. cmdoption:: -p, --preamble= - - When more than one pickle file are specified, print given preamble - before each disassembly. - - - -Programmatic Interface ----------------------- - - -.. function:: dis(pickle, out=None, memo=None, indentlevel=4, annotate=0) - - Outputs a symbolic disassembly of the pickle to the file-like - object *out*, defaulting to ``sys.stdout``. *pickle* can be a - string or a file-like object. *memo* can be a Python dictionary - that will be used as the pickle's memo; it can be used to perform - disassemblies across multiple pickles created by the same - pickler. Successive levels, indicated by ``MARK`` opcodes in the - stream, are indented by *indentlevel* spaces. If a nonzero value - is given to *annotate*, each opcode in the output is annotated with - a short description. The value of *annotate* is used as a hint for - the column where annotation should start. - - .. versionadded:: 3.2 - The *annotate* argument. - -.. function:: genops(pickle) - - Provides an :term:`iterator` over all of the opcodes in a pickle, returning a - sequence of ``(opcode, arg, pos)`` triples. *opcode* is an instance of an - :class:`OpcodeInfo` class; *arg* is the decoded value, as a Python object, of - the opcode's argument; *pos* is the position at which this opcode is located. - *pickle* can be a string or a file-like object. - -.. function:: optimize(picklestring) - - Returns a new equivalent pickle string after eliminating unused ``PUT`` - opcodes. The optimized pickle is shorter, takes less transmission time, - requires less storage space, and unpickles more efficiently. diff --git a/third_party/python/Doc/library/pipes.rst b/third_party/python/Doc/library/pipes.rst deleted file mode 100644 index 0a22da1f5..000000000 --- a/third_party/python/Doc/library/pipes.rst +++ /dev/null @@ -1,95 +0,0 @@ -:mod:`pipes` --- Interface to shell pipelines -============================================= - -.. module:: pipes - :platform: Unix - :synopsis: A Python interface to Unix shell pipelines. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/pipes.py` - --------------- - -The :mod:`pipes` module defines a class to abstract the concept of a *pipeline* ---- a sequence of converters from one file to another. - -Because the module uses :program:`/bin/sh` command lines, a POSIX or compatible -shell for :func:`os.system` and :func:`os.popen` is required. - -The :mod:`pipes` module defines the following class: - - -.. class:: Template() - - An abstraction of a pipeline. - -Example:: - - >>> import pipes - >>> t = pipes.Template() - >>> t.append('tr a-z A-Z', '--') - >>> f = t.open('pipefile', 'w') - >>> f.write('hello world') - >>> f.close() - >>> open('pipefile').read() - 'HELLO WORLD' - - -.. _template-objects: - -Template Objects ----------------- - -Template objects following methods: - - -.. method:: Template.reset() - - Restore a pipeline template to its initial state. - - -.. method:: Template.clone() - - Return a new, equivalent, pipeline template. - - -.. method:: Template.debug(flag) - - If *flag* is true, turn debugging on. Otherwise, turn debugging off. When - debugging is on, commands to be executed are printed, and the shell is given - ``set -x`` command to be more verbose. - - -.. method:: Template.append(cmd, kind) - - Append a new action at the end. The *cmd* variable must be a valid bourne shell - command. The *kind* variable consists of two letters. - - The first letter can be either of ``'-'`` (which means the command reads its - standard input), ``'f'`` (which means the commands reads a given file on the - command line) or ``'.'`` (which means the commands reads no input, and hence - must be first.) - - Similarly, the second letter can be either of ``'-'`` (which means the command - writes to standard output), ``'f'`` (which means the command writes a file on - the command line) or ``'.'`` (which means the command does not write anything, - and hence must be last.) - - -.. method:: Template.prepend(cmd, kind) - - Add a new action at the beginning. See :meth:`append` for explanations of the - arguments. - - -.. method:: Template.open(file, mode) - - Return a file-like object, open to *file*, but read from or written to by the - pipeline. Note that only one of ``'r'``, ``'w'`` may be given. - - -.. method:: Template.copy(infile, outfile) - - Copy *infile* to *outfile* through the pipe. - diff --git a/third_party/python/Doc/library/pkgutil.rst b/third_party/python/Doc/library/pkgutil.rst deleted file mode 100644 index fba0ea641..000000000 --- a/third_party/python/Doc/library/pkgutil.rst +++ /dev/null @@ -1,229 +0,0 @@ -:mod:`pkgutil` --- Package extension utility -============================================ - -.. module:: pkgutil - :synopsis: Utilities for the import system. - -**Source code:** :source:`Lib/pkgutil.py` - --------------- - -This module provides utilities for the import system, in particular package -support. - -.. class:: ModuleInfo(module_finder, name, ispkg) - - A namedtuple that holds a brief summary of a module's info. - - .. versionadded:: 3.6 - -.. function:: extend_path(path, name) - - Extend the search path for the modules which comprise a package. Intended - use is to place the following code in a package's :file:`__init__.py`:: - - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - - This will add to the package's ``__path__`` all subdirectories of directories - on ``sys.path`` named after the package. This is useful if one wants to - distribute different parts of a single logical package as multiple - directories. - - It also looks for :file:`\*.pkg` files beginning where ``*`` matches the - *name* argument. This feature is similar to :file:`\*.pth` files (see the - :mod:`site` module for more information), except that it doesn't special-case - lines starting with ``import``. A :file:`\*.pkg` file is trusted at face - value: apart from checking for duplicates, all entries found in a - :file:`\*.pkg` file are added to the path, regardless of whether they exist - on the filesystem. (This is a feature.) - - If the input path is not a list (as is the case for frozen packages) it is - returned unchanged. The input path is not modified; an extended copy is - returned. Items are only appended to the copy at the end. - - It is assumed that :data:`sys.path` is a sequence. Items of :data:`sys.path` - that are not strings referring to existing directories are ignored. Unicode - items on :data:`sys.path` that cause errors when used as filenames may cause - this function to raise an exception (in line with :func:`os.path.isdir` - behavior). - - -.. class:: ImpImporter(dirname=None) - - :pep:`302` Finder that wraps Python's "classic" import algorithm. - - If *dirname* is a string, a :pep:`302` finder is created that searches that - directory. If *dirname* is ``None``, a :pep:`302` finder is created that - searches the current :data:`sys.path`, plus any modules that are frozen or - built-in. - - Note that :class:`ImpImporter` does not currently support being used by - placement on :data:`sys.meta_path`. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully PEP 302 compliant and available in :mod:`importlib`. - - -.. class:: ImpLoader(fullname, file, filename, etc) - - :term:`Loader` that wraps Python's "classic" import algorithm. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully PEP 302 compliant and available in :mod:`importlib`. - - -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - -.. function:: get_importer(path_item) - - Retrieve a :term:`finder` for the given *path_item*. - - The returned finder is cached in :data:`sys.path_importer_cache` if it was - newly created by a path hook. - - The cache (or part of it) can be cleared manually if a rescan of - :data:`sys.path_hooks` is necessary. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - -.. function:: iter_importers(fullname='') - - Yield :term:`finder` objects for the given module name. - - If fullname contains a '.', the finders will be for the package - containing fullname, otherwise they will be all registered top level - finders (i.e. those on both sys.meta_path and sys.path_hooks). - - If the named module is in a package, that package is imported as a side - effect of invoking this function. - - If no module name is specified, all top level finders are produced. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - -.. function:: iter_modules(path=None, prefix='') - - Yields :class:`ModuleInfo` for all submodules on *path*, or, if - *path* is ``None``, all top-level modules on ``sys.path``. - - *path* should be either ``None`` or a list of paths to look for modules in. - - *prefix* is a string to output on the front of every module name on output. - - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - -.. function:: walk_packages(path=None, prefix='', onerror=None) - - Yields :class:`ModuleInfo` for all modules recursively on - *path*, or, if *path* is ``None``, all accessible modules. - - *path* should be either ``None`` or a list of paths to look for modules in. - - *prefix* is a string to output on the front of every module name on output. - - Note that this function must import all *packages* (*not* all modules!) on - the given *path*, in order to access the ``__path__`` attribute to find - submodules. - - *onerror* is a function which gets called with one argument (the name of the - package which was being imported) if any exception occurs while trying to - import a package. If no *onerror* function is supplied, :exc:`ImportError`\s - are caught and ignored, while all other exceptions are propagated, - terminating the search. - - Examples:: - - # list all modules python can access - walk_packages() - - # list all submodules of ctypes - walk_packages(ctypes.__path__, ctypes.__name__ + '.') - - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - -.. function:: get_data(package, resource) - - Get a resource from a package. - - This is a wrapper for the :term:`loader` - :meth:`get_data ` API. The - *package* argument should be the name of a package, in standard module format - (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). - - The function returns a binary string that is the contents of the specified - resource. - - For packages located in the filesystem, which have already been imported, - this is the rough equivalent of:: - - d = os.path.dirname(sys.modules[package].__file__) - data = open(os.path.join(d, resource), 'rb').read() - - If the package cannot be located or loaded, or it uses a :term:`loader` - which does not support :meth:`get_data `, - then ``None`` is returned. In particular, the :term:`loader` for - :term:`namespace packages ` does not support - :meth:`get_data `. diff --git a/third_party/python/Doc/library/platform.rst b/third_party/python/Doc/library/platform.rst deleted file mode 100644 index 7d29dc186..000000000 --- a/third_party/python/Doc/library/platform.rst +++ /dev/null @@ -1,284 +0,0 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== - -.. module:: platform - :synopsis: Retrieves as much platform identifying data as possible. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Bjorn Pettersen - -**Source code:** :source:`Lib/platform.py` - --------------- - -.. note:: - - Specific platforms listed alphabetically, with Linux included in the Unix - section. - - -Cross Platform --------------- - - -.. function:: architecture(executable=sys.executable, bits='', linkage='') - - Queries the given executable (defaults to the Python interpreter binary) for - various architecture information. - - Returns a tuple ``(bits, linkage)`` which contain information about the bit - architecture and the linkage format used for the executable. Both values are - returned as strings. - - Values that cannot be determined are returned as given by the parameter presets. - If bits is given as ``''``, the ``sizeof(pointer)`` (or - ``sizeof(long)`` on Python version < 1.5.2) is used as indicator for the - supported pointer size. - - The function relies on the system's :file:`file` command to do the actual work. - This is available on most if not all Unix platforms and some non-Unix platforms - and then only if the executable points to the Python interpreter. Reasonable - defaults are used when the above needs are not met. - - .. note:: - - On Mac OS X (and perhaps other platforms), executable files may be - universal files containing multiple architectures. - - To get at the "64-bitness" of the current interpreter, it is more - reliable to query the :attr:`sys.maxsize` attribute:: - - is_64bits = sys.maxsize > 2**32 - - -.. function:: machine() - - Returns the machine type, e.g. ``'i386'``. An empty string is returned if the - value cannot be determined. - - -.. function:: node() - - Returns the computer's network name (may not be fully qualified!). An empty - string is returned if the value cannot be determined. - - -.. function:: platform(aliased=0, terse=0) - - Returns a single string identifying the underlying platform with as much useful - information as possible. - - The output is intended to be *human readable* rather than machine parseable. It - may look different on different platforms and this is intended. - - If *aliased* is true, the function will use aliases for various platforms that - report system names which differ from their common names, for example SunOS will - be reported as Solaris. The :func:`system_alias` function is used to implement - this. - - Setting *terse* to true causes the function to return only the absolute minimum - information needed to identify the platform. - - -.. function:: processor() - - Returns the (real) processor name, e.g. ``'amdk6'``. - - An empty string is returned if the value cannot be determined. Note that many - platforms do not provide this information or simply return the same value as for - :func:`machine`. NetBSD does this. - - -.. function:: python_build() - - Returns a tuple ``(buildno, builddate)`` stating the Python build number and - date as strings. - - -.. function:: python_compiler() - - Returns a string identifying the compiler used for compiling Python. - - -.. function:: python_branch() - - Returns a string identifying the Python implementation SCM branch. - - -.. function:: python_implementation() - - Returns a string identifying the Python implementation. Possible return values - are: 'CPython', 'IronPython', 'Jython', 'PyPy'. - - -.. function:: python_revision() - - Returns a string identifying the Python implementation SCM revision. - - -.. function:: python_version() - - Returns the Python version as string ``'major.minor.patchlevel'``. - - Note that unlike the Python ``sys.version``, the returned value will always - include the patchlevel (it defaults to 0). - - -.. function:: python_version_tuple() - - Returns the Python version as tuple ``(major, minor, patchlevel)`` of strings. - - Note that unlike the Python ``sys.version``, the returned value will always - include the patchlevel (it defaults to ``'0'``). - - -.. function:: release() - - Returns the system's release, e.g. ``'2.2.0'`` or ``'NT'`` An empty string is - returned if the value cannot be determined. - - -.. function:: system() - - Returns the system/OS name, e.g. ``'Linux'``, ``'Windows'``, or ``'Java'``. An - empty string is returned if the value cannot be determined. - - -.. function:: system_alias(system, release, version) - - Returns ``(system, release, version)`` aliased to common marketing names used - for some systems. It also does some reordering of the information in some cases - where it would otherwise cause confusion. - - -.. function:: version() - - Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is - returned if the value cannot be determined. - - -.. function:: uname() - - Fairly portable uname interface. Returns a :func:`~collections.namedtuple` - containing six attributes: :attr:`system`, :attr:`node`, :attr:`release`, - :attr:`version`, :attr:`machine`, and :attr:`processor`. - - Note that this adds a sixth attribute (:attr:`processor`) not present - in the :func:`os.uname` result. Also, the attribute names are different - for the first two attributes; :func:`os.uname` names them - :attr:`sysname` and :attr:`nodename`. - - Entries which cannot be determined are set to ``''``. - - .. versionchanged:: 3.3 - Result changed from a tuple to a namedtuple. - - -Java Platform -------------- - - -.. function:: java_ver(release='', vendor='', vminfo=('','',''), osinfo=('','','')) - - Version interface for Jython. - - Returns a tuple ``(release, vendor, vminfo, osinfo)`` with *vminfo* being a - tuple ``(vm_name, vm_release, vm_vendor)`` and *osinfo* being a tuple - ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to - the defaults given as parameters (which all default to ``''``). - - -Windows Platform ----------------- - - -.. function:: win32_ver(release='', version='', csd='', ptype='') - - Get additional version information from the Windows Registry and return a tuple - ``(release, version, csd, ptype)`` referring to OS release, version number, - CSD level (service pack) and OS type (multi/single processor). - - As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines - and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers - to the OS version being free of debugging code. It could also state *'Checked'* - which means the OS version uses debugging code, i.e. code that checks arguments, - ranges, etc. - - .. note:: - - This function works best with Mark Hammond's - :mod:`win32all` package installed, but also on Python 2.3 and - later (support for this was added in Python 2.6). It obviously - only runs on Win32 compatible platforms. - - -Win95/98 specific -^^^^^^^^^^^^^^^^^ - -.. function:: popen(cmd, mode='r', bufsize=-1) - - Portable :func:`popen` interface. Find a working popen implementation - preferring :func:`win32pipe.popen`. On Windows NT, :func:`win32pipe.popen` - should work; on Windows 9x it hangs due to bugs in the MS C library. - - .. deprecated:: 3.3 - This function is obsolete. Use the :mod:`subprocess` module. Check - especially the :ref:`subprocess-replacements` section. - - -Mac OS Platform ---------------- - - -.. function:: mac_ver(release='', versioninfo=('','',''), machine='') - - Get Mac OS version information and return it as tuple ``(release, versioninfo, - machine)`` with *versioninfo* being a tuple ``(version, dev_stage, - non_release_version)``. - - Entries which cannot be determined are set to ``''``. All tuple entries are - strings. - - -Unix Platforms --------------- - - -.. function:: dist(distname='', version='', id='', supported_dists=('SuSE','debian','redhat','mandrake',...)) - - This is another name for :func:`linux_distribution`. - - .. deprecated-removed:: 3.5 3.8 - See alternative like the `distro `_ package. - -.. function:: linux_distribution(distname='', version='', id='', supported_dists=('SuSE','debian','redhat','mandrake',...), full_distribution_name=1) - - Tries to determine the name of the Linux OS distribution name. - - ``supported_dists`` may be given to define the set of Linux distributions to - look for. It defaults to a list of currently supported Linux distributions - identified by their release file name. - - If ``full_distribution_name`` is true (default), the full distribution read - from the OS is returned. Otherwise the short name taken from - ``supported_dists`` is used. - - Returns a tuple ``(distname,version,id)`` which defaults to the args given as - parameters. ``id`` is the item in parentheses after the version number. It - is usually the version codename. - - .. deprecated-removed:: 3.5 3.8 - See alternative like the `distro `_ package. - -.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=16384) - - Tries to determine the libc version against which the file executable (defaults - to the Python interpreter) is linked. Returns a tuple of strings ``(lib, - version)`` which default to the given parameters in case the lookup fails. - - Note that this function has intimate knowledge of how different libc versions - add symbols to the executable is probably only usable for executables compiled - using :program:`gcc`. - - The file is read and scanned in chunks of *chunksize* bytes. - diff --git a/third_party/python/Doc/library/plistlib.rst b/third_party/python/Doc/library/plistlib.rst deleted file mode 100644 index 9ba226607..000000000 --- a/third_party/python/Doc/library/plistlib.rst +++ /dev/null @@ -1,244 +0,0 @@ -:mod:`plistlib` --- Generate and parse Mac OS X ``.plist`` files -================================================================ - -.. module:: plistlib - :synopsis: Generate and parse Mac OS X plist files. - -.. moduleauthor:: Jack Jansen -.. sectionauthor:: Georg Brandl -.. (harvested from docstrings in the original file) - -**Source code:** :source:`Lib/plistlib.py` - -.. index:: - pair: plist; file - single: property list - --------------- - -This module provides an interface for reading and writing the "property list" -files used mainly by Mac OS X and supports both binary and XML plist files. - -The property list (``.plist``) file format is a simple serialization supporting -basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. - -To write out and to parse a plist file, use the :func:`dump` and -:func:`load` functions. - -To work with plist data in bytes objects, use :func:`dumps` -and :func:`loads`. - -Values can be strings, integers, floats, booleans, tuples, lists, dictionaries -(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray` -or :class:`datetime.datetime` objects. - -.. versionchanged:: 3.4 - New API, old API deprecated. Support for binary format plists added. - -.. seealso:: - - `PList manual page `_ - Apple's documentation of the file format. - - -This module defines the following functions: - -.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict) - - Read a plist file. *fp* should be a readable and binary file object. - Return the unpacked root object (which usually is a - dictionary). - - The *fmt* is the format of the file and the following values are valid: - - * :data:`None`: Autodetect the file format - - * :data:`FMT_XML`: XML file format - - * :data:`FMT_BINARY`: Binary plist format - - If *use_builtin_types* is true (the default) binary data will be returned - as instances of :class:`bytes`, otherwise it is returned as instances of - :class:`Data`. - - The *dict_type* is the type used for dictionaries that are read from the - plist file. The exact structure of the plist can be recovered by using - :class:`collections.OrderedDict` (although the order of keys shouldn't be - important in plist files). - - XML data for the :data:`FMT_XML` format is parsed using the Expat parser - from :mod:`xml.parsers.expat` -- see its documentation for possible - exceptions on ill-formed XML. Unknown elements will simply be ignored - by the plist parser. - - The parser for the binary format raises :exc:`InvalidFileException` - when the file cannot be parsed. - - .. versionadded:: 3.4 - - -.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict) - - Load a plist from a bytes object. See :func:`load` for an explanation of - the keyword arguments. - - .. versionadded:: 3.4 - - -.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Write *value* to a plist file. *Fp* should be a writable, binary - file object. - - The *fmt* argument specifies the format of the plist file and can be - one of the following values: - - * :data:`FMT_XML`: XML formatted plist file - - * :data:`FMT_BINARY`: Binary formatted plist file - - When *sort_keys* is true (the default) the keys for dictionaries will be - written to the plist in sorted order, otherwise they will be written in - the iteration order of the dictionary. - - When *skipkeys* is false (the default) the function raises :exc:`TypeError` - when a key of a dictionary is not a string, otherwise such keys are skipped. - - A :exc:`TypeError` will be raised if the object is of an unsupported type or - a container that contains objects of unsupported types. - - An :exc:`OverflowError` will be raised for integer values that cannot - be represented in (binary) plist files. - - .. versionadded:: 3.4 - - -.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Return *value* as a plist-formatted bytes object. See - the documentation for :func:`dump` for an explanation of the keyword - arguments of this function. - - .. versionadded:: 3.4 - -The following functions are deprecated: - -.. function:: readPlist(pathOrFile) - - Read a plist file. *pathOrFile* may be either a file name or a (readable - and binary) file object. Returns the unpacked root object (which usually - is a dictionary). - - This function calls :func:`load` to do the actual work, see the documentation - of :func:`that function ` for an explanation of the keyword arguments. - - .. note:: - - Dict values in the result have a ``__getattr__`` method that defers - to ``__getitem_``. This means that you can use attribute access to - access items of these dictionaries. - - .. deprecated:: 3.4 Use :func:`load` instead. - - -.. function:: writePlist(rootObject, pathOrFile) - - Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name - or a (writable and binary) file object - - .. deprecated:: 3.4 Use :func:`dump` instead. - - -.. function:: readPlistFromBytes(data) - - Read a plist data from a bytes object. Return the root object. - - See :func:`load` for a description of the keyword arguments. - - .. note:: - - Dict values in the result have a ``__getattr__`` method that defers - to ``__getitem_``. This means that you can use attribute access to - access items of these dictionaries. - - .. deprecated:: 3.4 Use :func:`loads` instead. - - -.. function:: writePlistToBytes(rootObject) - - Return *rootObject* as an XML plist-formatted bytes object. - - .. deprecated:: 3.4 Use :func:`dumps` instead. - - -The following classes are available: - -.. class:: Dict([dict]): - - Return an extended mapping object with the same value as dictionary - *dict*. - - This class is a subclass of :class:`dict` where attribute access can - be used to access items. That is, ``aDict.key`` is the same as - ``aDict['key']`` for getting, setting and deleting items in the mapping. - - .. deprecated:: 3.0 - - -.. class:: Data(data) - - Return a "data" wrapper object around the bytes object *data*. This is used - in functions converting from/to plists to represent the ```` type - available in plists. - - It has one attribute, :attr:`data`, that can be used to retrieve the Python - bytes object stored in it. - - .. deprecated:: 3.4 Use a :class:`bytes` object instead. - - -The following constants are available: - -.. data:: FMT_XML - - The XML format for plist files. - - .. versionadded:: 3.4 - - -.. data:: FMT_BINARY - - The binary format for plist files - - .. versionadded:: 3.4 - - -Examples --------- - -Generating a plist:: - - pl = dict( - aString = "Doodah", - aList = ["A", "B", 12, 32.1, [1, 2, 3]], - aFloat = 0.1, - anInt = 728, - aDict = dict( - anotherString = "", - aThirdString = "M\xe4ssig, Ma\xdf", - aTrueValue = True, - aFalseValue = False, - ), - someData = b"", - someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), - ) - with open(fileName, 'wb') as fp: - dump(pl, fp) - -Parsing a plist:: - - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) diff --git a/third_party/python/Doc/library/poplib.rst b/third_party/python/Doc/library/poplib.rst deleted file mode 100644 index d72b660d6..000000000 --- a/third_party/python/Doc/library/poplib.rst +++ /dev/null @@ -1,255 +0,0 @@ -:mod:`poplib` --- POP3 protocol client -====================================== - -.. module:: poplib - :synopsis: POP3 protocol client (requires sockets). - -.. sectionauthor:: Andrew T. Csillag -.. revised by ESR, January 2000 - -**Source code:** :source:`Lib/poplib.py` - -.. index:: pair: POP3; protocol - --------------- - -This module defines a class, :class:`POP3`, which encapsulates a connection to a -POP3 server and implements the protocol as defined in :rfc:`1939`. The -:class:`POP3` class supports both the minimal and optional command sets from -:rfc:`1939`. The :class:`POP3` class also supports the ``STLS`` command introduced -in :rfc:`2595` to enable encrypted communication on an already established connection. - -Additionally, this module provides a class :class:`POP3_SSL`, which provides -support for connecting to POP3 servers that use SSL as an underlying protocol -layer. - -Note that POP3, though widely supported, is obsolescent. The implementation -quality of POP3 servers varies widely, and too many are quite poor. If your -mailserver supports IMAP, you would be better off using the -:class:`imaplib.IMAP4` class, as IMAP servers tend to be better implemented. - -The :mod:`poplib` module provides two classes: - - -.. class:: POP3(host, port=POP3_PORT[, timeout]) - - This class implements the actual POP3 protocol. The connection is created when - the instance is initialized. If *port* is omitted, the standard POP3 port (110) - is used. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt (if not specified, the global default timeout setting will - be used). - - -.. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) - - This is a subclass of :class:`POP3` that connects to the server over an SSL - encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL - port is used. *timeout* works as in the :class:`POP3` constructor. - *context* is an optional :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into a - single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* - they can - point to PEM-formatted private key and certificate chain files, - respectively, for the SSL connection. - - .. versionchanged:: 3.2 - *context* parameter added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - -One exception is defined as an attribute of the :mod:`poplib` module: - - -.. exception:: error_proto - - Exception raised on any errors from this module (errors from :mod:`socket` - module are not caught). The reason for the exception is passed to the - constructor as a string. - - -.. seealso:: - - Module :mod:`imaplib` - The standard Python IMAP module. - - `Frequently Asked Questions About Fetchmail `_ - The FAQ for the :program:`fetchmail` POP/IMAP client collects information on - POP3 server variations and RFC noncompliance that may be useful if you need to - write an application based on the POP protocol. - - -.. _pop3-objects: - -POP3 Objects ------------- - -All POP3 commands are represented by methods of the same name, in lower-case; -most return the response text sent by the server. - -An :class:`POP3` instance has the following methods: - - -.. method:: POP3.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. - - -.. method:: POP3.getwelcome() - - Returns the greeting string sent by the POP3 server. - - -.. method:: POP3.capa() - - Query the server's capabilities as specified in :rfc:`2449`. - Returns a dictionary in the form ``{'name': ['param'...]}``. - - .. versionadded:: 3.4 - - -.. method:: POP3.user(username) - - Send user command, response should indicate that a password is required. - - -.. method:: POP3.pass_(password) - - Send password, response includes message count and mailbox size. Note: the - mailbox on the server is locked until :meth:`~poplib.quit` is called. - - -.. method:: POP3.apop(user, secret) - - Use the more secure APOP authentication to log into the POP3 server. - - -.. method:: POP3.rpop(user) - - Use RPOP authentication (similar to UNIX r-commands) to log into POP3 server. - - -.. method:: POP3.stat() - - Get mailbox status. The result is a tuple of 2 integers: ``(message count, - mailbox size)``. - - -.. method:: POP3.list([which]) - - Request message list, result is in the form ``(response, ['mesg_num octets', - ...], octets)``. If *which* is set, it is the message to list. - - -.. method:: POP3.retr(which) - - Retrieve whole message number *which*, and set its seen flag. Result is in form - ``(response, ['line', ...], octets)``. - - -.. method:: POP3.dele(which) - - Flag message number *which* for deletion. On most servers deletions are not - actually performed until QUIT (the major exception is Eudora QPOP, which - deliberately violates the RFCs by doing pending deletes on any disconnect). - - -.. method:: POP3.rset() - - Remove any deletion marks for the mailbox. - - -.. method:: POP3.noop() - - Do nothing. Might be used as a keep-alive. - - -.. method:: POP3.quit() - - Signoff: commit changes, unlock mailbox, drop connection. - - -.. method:: POP3.top(which, howmuch) - - Retrieves the message header plus *howmuch* lines of the message after the - header of message number *which*. Result is in form ``(response, ['line', ...], - octets)``. - - The POP3 TOP command this method uses, unlike the RETR command, doesn't set the - message's seen flag; unfortunately, TOP is poorly specified in the RFCs and is - frequently broken in off-brand servers. Test this method by hand against the - POP3 servers you will use before trusting it. - - -.. method:: POP3.uidl(which=None) - - Return message digest (unique id) list. If *which* is specified, result contains - the unique id for that message in the form ``'response mesgnum uid``, otherwise - result is list ``(response, ['mesgnum uid', ...], octets)``. - - -.. method:: POP3.utf8() - - Try to switch to UTF-8 mode. Returns the server response if successful, - raises :class:`error_proto` if not. Specified in :RFC:`6856`. - - .. versionadded:: 3.5 - - -.. method:: POP3.stls(context=None) - - Start a TLS session on the active connection as specified in :rfc:`2595`. - This is only allowed before user authentication - - *context* parameter is a :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into - a single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - This method supports hostname checking via - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. versionadded:: 3.4 - - -Instances of :class:`POP3_SSL` have no additional methods. The interface of this -subclass is identical to its parent. - - -.. _pop3-example: - -POP3 Example ------------- - -Here is a minimal example (without error checking) that opens a mailbox and -retrieves and prints all messages:: - - import getpass, poplib - - M = poplib.POP3('localhost') - M.user(getpass.getuser()) - M.pass_(getpass.getpass()) - numMessages = len(M.list()[1]) - for i in range(numMessages): - for j in M.retr(i+1)[1]: - print(j) - -At the end of the module, there is a test section that contains a more extensive -example of usage. - diff --git a/third_party/python/Doc/library/posix.rst b/third_party/python/Doc/library/posix.rst deleted file mode 100644 index 9cbc5505a..000000000 --- a/third_party/python/Doc/library/posix.rst +++ /dev/null @@ -1,92 +0,0 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== - -.. module:: posix - :platform: Unix - :synopsis: The most common POSIX system calls (normally used via module os). - --------------- - -This module provides access to operating system functionality that is -standardized by the C Standard and the POSIX standard (a thinly disguised Unix -interface). - -.. index:: module: os - -**Do not import this module directly.** Instead, import the module :mod:`os`, -which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always -available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, -:mod:`os` provides some additional functionality, such as automatically calling -:func:`~os.putenv` when an entry in ``os.environ`` is changed. - -Errors are reported as exceptions; the usual exceptions are given for type -errors, while errors reported by the system calls raise :exc:`OSError`. - - -.. _posix-large-files: - -Large File Support ------------------- - -.. index:: - single: large files - single: file; large files - -.. sectionauthor:: Steve Clift - -Several operating systems (including AIX, HP-UX, Irix and Solaris) provide -support for files that are larger than 2 GiB from a C programming model where -:c:type:`int` and :c:type:`long` are 32-bit values. This is typically accomplished -by defining the relevant size and offset types as 64-bit values. Such files are -sometimes referred to as :dfn:`large files`. - -Large file support is enabled in Python when the size of an :c:type:`off_t` is -larger than a :c:type:`long` and the :c:type:`long long` type is available and is -at least as large as an :c:type:`off_t`. -It may be necessary to configure and compile Python with certain compiler flags -to enable this mode. For example, it is enabled by default with recent versions -of Irix, but with Solaris 2.6 and 2.7 you need to do something like:: - - CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \ - ./configure - -On large-file-capable Linux systems, this might work:: - - CFLAGS='-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64' OPT="-g -O2 $CFLAGS" \ - ./configure - - -.. _posix-contents: - -Notable Module Contents ------------------------ - -In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: - -.. data:: environ - - A dictionary representing the string environment at the time the interpreter - was started. Keys and values are bytes on Unix and str on Windows. For - example, ``environ[b'HOME']`` (``environ['HOME']`` on Windows) is the - pathname of your home directory, equivalent to ``getenv("HOME")`` in C. - - Modifying this dictionary does not affect the string environment passed on by - :func:`~os.execv`, :func:`~os.popen` or :func:`~os.system`; if you need to - change the environment, pass ``environ`` to :func:`~os.execve` or add - variable assignments and export statements to the command string for - :func:`~os.system` or :func:`~os.popen`. - - .. versionchanged:: 3.2 - On Unix, keys and values are bytes. - - .. note:: - - The :mod:`os` module provides an alternate implementation of ``environ`` - which updates the environment on modification. Note also that updating - :data:`os.environ` will render this dictionary obsolete. Use of the - :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. diff --git a/third_party/python/Doc/library/pprint.rst b/third_party/python/Doc/library/pprint.rst deleted file mode 100644 index deadf1820..000000000 --- a/third_party/python/Doc/library/pprint.rst +++ /dev/null @@ -1,374 +0,0 @@ -:mod:`pprint` --- Data pretty printer -===================================== - -.. module:: pprint - :synopsis: Data pretty printer. - -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/pprint.py` - --------------- - -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary -Python data structures in a form which can be used as input to the interpreter. -If the formatted structures include objects which are not fundamental Python -types, the representation may not be loadable. This may be the case if objects -such as files, sockets or classes are included, as well as many other -objects which are not representable as Python literals. - -The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. - -Dictionaries are sorted by key before the display is computed. - -The :mod:`pprint` module defines one class: - -.. First the implementation class: - - -.. index:: single: ...; placeholder - -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False) - - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. An output stream may be set using the *stream* - keyword; the only method used on the stream object is the file protocol's - :meth:`write` method. If not specified, the :class:`PrettyPrinter` adopts - ``sys.stdout``. The - amount of indentation added for each recursive level is specified by *indent*; - the default is one. Other values can cause output to look a little odd, but can - make nesting easier to spot. The number of levels which may be printed is - controlled by *depth*; if the data structure being printed is too deep, the next - contained level is replaced by ``...``. By default, there is no constraint on - the depth of the objects being formatted. The desired output width is - constrained using the *width* parameter; the default is 80 characters. If a - structure cannot be formatted within the constrained width, a best effort will - be made. If *compact* is false (the default) each item of a long sequence - will be formatted on a separate line. If *compact* is true, as many items - as will fit within the *width* will be formatted on each output line. - - .. versionchanged:: 3.4 - Added the *compact* parameter. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - - -The :mod:`pprint` module also provides several shortcut functions: - -.. function:: pformat(object, indent=1, width=80, depth=None, *, compact=False) - - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth* and *compact* will be passed to the :class:`PrettyPrinter` - constructor as formatting parameters. - - .. versionchanged:: 3.4 - Added the *compact* parameter. - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). *indent*, *width*, *depth* and *compact* will be passed - to the :class:`PrettyPrinter` constructor as formatting parameters. - - .. versionchanged:: 3.4 - Added the *compact* parameter. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - - -.. function:: isreadable(object) - - .. index:: builtin: eval - - Determine if the formatted representation of *object* is "readable," or can be - used to reconstruct the value using :func:`eval`. This always returns ``False`` - for recursive objects. - - >>> pprint.isreadable(stuff) - False - - -.. function:: isrecursive(object) - - Determine if *object* requires a recursive representation. - - -One more support function is also defined: - -.. function:: saferepr(object) - - Return a string representation of *object*, protected against recursive data - structures. If the representation of *object* exposes a recursive entry, the - recursive reference will be represented as ````. The representation is not otherwise formatted. - - >>> pprint.saferepr(stuff) - "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - - -.. _prettyprinter-objects: - -PrettyPrinter Objects ---------------------- - -:class:`PrettyPrinter` instances have the following methods: - - -.. method:: PrettyPrinter.pformat(object) - - Return the formatted representation of *object*. This takes into account the - options passed to the :class:`PrettyPrinter` constructor. - - -.. method:: PrettyPrinter.pprint(object) - - Print the formatted representation of *object* on the configured stream, - followed by a newline. - -The following methods provide the implementations for the corresponding -functions of the same names. Using these methods on an instance is slightly -more efficient since new :class:`PrettyPrinter` objects don't need to be -created. - - -.. method:: PrettyPrinter.isreadable(object) - - .. index:: builtin: eval - - Determine if the formatted representation of the object is "readable," or can be - used to reconstruct the value using :func:`eval`. Note that this returns - ``False`` for recursive objects. If the *depth* parameter of the - :class:`PrettyPrinter` is set and the object is deeper than allowed, this - returns ``False``. - - -.. method:: PrettyPrinter.isrecursive(object) - - Determine if the object requires a recursive representation. - -This method is provided as a hook to allow subclasses to modify the way objects -are converted to strings. The default implementation uses the internals of the -:func:`saferepr` implementation. - - -.. method:: PrettyPrinter.format(object, context, maxlevels, level) - - Returns three values: the formatted version of *object* as a string, a flag - indicating whether the result is readable, and a flag indicating whether - recursion was detected. The first argument is the object to be presented. The - second is a dictionary which contains the :func:`id` of objects that are part of - the current presentation context (direct and indirect containers for *object* - that are affecting the presentation) as the keys; if an object needs to be - presented which is already represented in *context*, the third return value - should be ``True``. Recursive calls to the :meth:`.format` method should add - additional entries for containers to this dictionary. The third argument, - *maxlevels*, gives the requested limit to recursion; this will be ``0`` if there - is no requested limit. This argument should be passed unmodified to recursive - calls. The fourth argument, *level*, gives the current level; recursive calls - should be passed a value less than that of the current call. - - -.. _pprint-example: - -Example -------- - -To demonstrate several uses of the :func:`pprint` function and its parameters, -let's fetch information about a project from `PyPI `_:: - - >>> import json - >>> import pprint - >>> from urllib.request import urlopen - >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: - ... project_info = json.load(resp)['info'] - -In its basic form, :func:`pprint` shows the whole object:: - - >>> pprint.pprint(project_info) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': ['Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Software Development :: Build Tools'], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ' - 'ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and ' - 'should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of ' - 'the project, basic\n' - 'usage examples, etc. Generally, including the project ' - 'changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the ' - 'most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {'Download': 'UNKNOWN', - 'Homepage': 'https://github.com/pypa/sampleproject'}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} - -The result can be limited to a certain *depth* (ellipsis is used for deeper -contents):: - - >>> pprint.pprint(project_info, depth=1) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ' - 'ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and ' - 'should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of ' - 'the project, basic\n' - 'usage examples, etc. Generally, including the project ' - 'changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the ' - 'most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} - -Additionally, maximum character *width* can be suggested. If a long object -cannot be split, the specified width will be exceeded:: - - >>> pprint.pprint(project_info, depth=1, width=60) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the ' - 'project.\n' - '\n' - 'The file should use UTF-8 encoding and be ' - 'written using ReStructured Text. It\n' - 'will be used to generate the project ' - 'webpage on PyPI, and should be written ' - 'for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would ' - 'include an overview of the project, ' - 'basic\n' - 'usage examples, etc. Generally, including ' - 'the project changelog in here is not\n' - 'a good idea, although a simple "What\'s ' - 'New" section for the most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} diff --git a/third_party/python/Doc/library/profile.rst b/third_party/python/Doc/library/profile.rst deleted file mode 100644 index 845065e38..000000000 --- a/third_party/python/Doc/library/profile.rst +++ /dev/null @@ -1,655 +0,0 @@ -.. _profile: - -******************** -The Python Profilers -******************** - -**Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` - --------------- - -.. _profiler-introduction: - -Introduction to the profilers -============================= - -.. index:: - single: deterministic profiling - single: profiling, deterministic - -:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. - -The Python standard library provides two different implementations of the same -profiling interface: - -1. :mod:`cProfile` is recommended for most users; it's a C extension with - reasonable overhead that makes it suitable for profiling long-running - programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted - Czotter. - -2. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. - -.. note:: - - The profiler modules are designed to provide an execution profile for a given - program, not for benchmarking purposes (for that, there is :mod:`timeit` for - reasonably accurate results). This particularly applies to benchmarking - Python code against C code: the profilers introduce overhead for Python code, - but not for C-level functions, and so the C code would seem faster than any - Python one. - - -.. _profile-instant: - -Instant User's Manual -===================== - -This section is provided for users that "don't want to read the manual." It -provides a very brief overview, and allows a user to rapidly perform profiling -on an existing application. - -To profile a function that takes a single argument, you can do:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")') - -(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on -your system.) - -The above action would run :func:`re.compile` and print profile results like -the following:: - - 197 function calls (192 primitive calls) in 0.002 seconds - - Ordered by: standard name - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 re.py:212(compile) - 1 0.000 0.000 0.001 0.001 re.py:268(_compile) - 1 0.000 0.000 0.000 0.000 sre_compile.py:172(_compile_charset) - 1 0.000 0.000 0.000 0.000 sre_compile.py:201(_optimize_charset) - 4 0.000 0.000 0.000 0.000 sre_compile.py:25(_identityfunction) - 3/1 0.000 0.000 0.000 0.000 sre_compile.py:33(_compile) - -The first line indicates that 197 calls were monitored. Of those calls, 192 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: standard name``, indicates that the text string in the -far right column was used to sort the output. The column headings include: - -ncalls - for the number of calls. - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') - -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -The file :mod:`cProfile` can also be invoked as a script to profile another -script. For example:: - - python -m cProfile [-o output_file] [-s sort_order] myscript.py - -``-o`` writes the profile results to a file instead of to stdout - -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. - -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: - - import pstats - p = pstats.Stats('restats') - p.strip_dirs().sort_stats(-1).print_stats() - -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: - - p.sort_stats('name') - p.print_stats() - -The first call will actually sort the list by function name, and the second call -will print out the statistics. The following are some interesting calls to -experiment with:: - - p.sort_stats('cumulative').print_stats(10) - -This sorts the profile by cumulative time in a function, and then only prints -the ten most significant lines. If you want to understand what algorithms are -taking time, the above line is what you would use. - -If you were looking to see what functions were looping a lot, and taking a lot -of time, you would do:: - - p.sort_stats('time').print_stats(10) - -to sort according to time spent within each function, and then print the -statistics for the top ten functions. - -You might also try:: - - p.sort_stats('file').print_stats('__init__') - -This will sort all the statistics by file name, and then print out statistics -for only the class init methods (since they are spelled with ``__init__`` in -them). As one final example, you could try:: - - p.sort_stats('time', 'cumulative').print_stats(.5, 'init') - -This line sorts statistics with a primary key of time, and a secondary key of -cumulative time, and then prints out some of the statistics. To be specific, the -list is first culled down to 50% (re: ``.5``) of its original size, then only -lines containing ``init`` are maintained, and that sub-sub-list is printed. - -If you wondered what functions called the above functions, you could now (``p`` -is still sorted according to the last criteria) do:: - - p.print_callers(.5, 'init') - -and you would get a list of callers for each of the listed functions. - -If you want more functionality, you're going to have to read the manual, or -guess what the following functions do:: - - p.print_callees() - p.add('restats') - -Invoked as a script, the :mod:`pstats` module is a statistics browser for -reading and examining profile dumps. It has a simple line-oriented interface -(implemented using :mod:`cmd`) and interactive help. - -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: - -.. function:: run(command, filename=None, sort=-1) - - This function takes a single argument that can be passed to the :func:`exec` - function, and an optional file name. In all cases this routine executes:: - - exec(command, __main__.__dict__, __main__.__dict__) - - and gathers profiling statistics from the execution. If no file name is - present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified, - it is passed to this :class:`~pstats.Stats` instance to control how the - results are sorted. - -.. function:: runctx(command, globals, locals, filename=None, sort=-1) - - This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine - executes:: - - exec(command, globals, locals) - - and gathers profiling statistics as in the :func:`run` function above. - -.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) - - This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. - - A custom timer can be supplied for measuring how long code takes to run via - the *timer* argument. This must be a function that returns a single number - representing the current time. If the number is an integer, the *timeunit* - specifies a multiplier that specifies the duration of each unit of time. For - example, if the timer returns times measured in thousands of seconds, the - time unit would be ``.001``. - - Directly using the :class:`Profile` class allows formatting profile results - without writing the profile data to a file:: - - import cProfile, pstats, io - pr = cProfile.Profile() - pr.enable() - # ... do something ... - pr.disable() - s = io.StringIO() - sortby = 'cumulative' - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - ps.print_stats() - print(s.getvalue()) - - .. method:: enable() - - Start collecting profiling data. - - .. method:: disable() - - Stop collecting profiling data. - - .. method:: create_stats() - - Stop collecting profiling data and record the results internally - as the current profile. - - .. method:: print_stats(sort=-1) - - Create a :class:`~pstats.Stats` object based on the current - profile and print the results to stdout. - - .. method:: dump_stats(filename) - - Write the results of the current profile to *filename*. - - .. method:: run(cmd) - - Profile the cmd via :func:`exec`. - - .. method:: runctx(cmd, globals, locals) - - Profile the cmd via :func:`exec` with the specified global and - local environment. - - .. method:: runcall(func, *args, **kwargs) - - Profile ``func(*args, **kwargs)`` - -Note that profiling will only work if the called command/function actually -returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call -during the called command/function execution) no profiling results will be -printed. - -.. _profile-stats: - -The :class:`Stats` Class -======================== - -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. - -.. module:: pstats - :synopsis: Statistics object for use with the profiler. - -.. class:: Stats(*filenames or profile, stream=sys.stdout) - - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. - - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers, or the same profiler run on a different operating system. If - several files are provided, all the statistics for identical functions will - be coalesced, so that an overall view of several processes can be considered - in a single report. If additional files need to be combined with data in an - existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method - can be used. - - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument is typically a string identifying the - basis of a sort (example: ``'time'`` or ``'name'``). - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats('name', 'file')`` will sort all the - entries according to their function name, and resolve all ties (identical - function names) by sorting by file name. - - Abbreviations can be used for any key names, as long as the abbreviation - is unambiguous. The following are the keys currently defined: - - +------------------+----------------------+ - | Valid Arg | Meaning | - +==================+======================+ - | ``'calls'`` | call count | - +------------------+----------------------+ - | ``'cumulative'`` | cumulative time | - +------------------+----------------------+ - | ``'cumtime'`` | cumulative time | - +------------------+----------------------+ - | ``'file'`` | file name | - +------------------+----------------------+ - | ``'filename'`` | file name | - +------------------+----------------------+ - | ``'module'`` | file name | - +------------------+----------------------+ - | ``'ncalls'`` | call count | - +------------------+----------------------+ - | ``'pcalls'`` | primitive call count | - +------------------+----------------------+ - | ``'line'`` | line number | - +------------------+----------------------+ - | ``'name'`` | function name | - +------------------+----------------------+ - | ``'nfl'`` | name/file/line | - +------------------+----------------------+ - | ``'stdname'`` | standard name | - +------------------+----------------------+ - | ``'time'`` | internal time | - +------------------+----------------------+ - | ``'tottime'`` | internal time | - +------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``'nfl'`` and ``'stdname'`` is that the standard name is a sort of the - name as printed, which means that the embedded line numbers get compared - in an odd way. For example, lines 3, 20, and 40 would (if the file names - were the same) appear in the string order 20, 3 and 40. In contrast, - ``'nfl'`` does a numeric compare of the line numbers. In fact, - ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', - 'line')``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will interpreted as a - regular expression (to pattern match the standard name that is printed). - If several restrictions are provided, then they are applied sequentially. - For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - - -.. _deterministic-profiling: - -What Is Deterministic Profiling? -================================ - -:dfn:`Deterministic profiling` is meant to reflect the fact that all *function -call*, *function return*, and *exception* events are monitored, and precise -timings are made for the intervals between these events (during which time the -user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. - -In Python, since there is an interpreter active during execution, the presence -of instrumented code is not required to do deterministic profiling. Python -automatically provides a :dfn:`hook` (optional callback) for each event. In -addition, the interpreted nature of Python tends to add so much overhead to -execution, that deterministic profiling tends to only add small processing -overhead in typical applications. The result is that deterministic profiling is -not that expensive, yet provides extensive run time statistics about the -execution of a Python program. - -Call count statistics can be used to identify bugs in code (surprising counts), -and to identify possible inline-expansion points (high call counts). Internal -time statistics can be used to identify "hot loops" that should be carefully -optimized. Cumulative time statistics should be used to identify high level -errors in the selection of algorithms. Note that the unusual handling of -cumulative times in this profiler allows statistics for recursive -implementations of algorithms to be directly compared to iterative -implementations. - - -.. _profile-limitations: - -Limitations -=========== - -One limitation has to do with accuracy of timing information. There is a -fundamental problem with deterministic profilers involving accuracy. The most -obvious restriction is that the underlying "clock" is only ticking at a rate -(typically) of about .001 seconds. Hence no measurements will be more accurate -than the underlying clock. If enough measurements are taken, then the "error" -will tend to average out. Unfortunately, removing this first error induces a -second source of error. - -The second problem is that it "takes a while" from when an event is dispatched -until the profiler's call to get the time actually *gets* the state of the -clock. Similarly, there is a certain lag when exiting the profiler event -handler from the time that the clock's value was obtained (and then squirreled -away), until the user's code is once again executing. As a result, functions -that are called many times, or call many functions, will typically accumulate -this error. The error that accumulates in this fashion is typically less than -the accuracy of the clock (less than one clock tick), but it *can* accumulate -and become very significant. - -The problem is more important with :mod:`profile` than with the lower-overhead -:mod:`cProfile`. For this reason, :mod:`profile` provides a means of -calibrating itself for a given platform so that this error can be -probabilistically (on the average) removed. After the profiler is calibrated, it -will be more accurate (in a least square sense), but it will sometimes produce -negative numbers (when call counts are exceptionally low, and the gods of -probability work against you :-). ) Do *not* be alarmed by negative numbers in -the profile. They should *only* appear if you have calibrated your profiler, -and the results are actually better than without calibration. - - -.. _profile-calibration: - -Calibration -=========== - -The profiler of the :mod:`profile` module subtracts a constant from each event -handling time to compensate for the overhead of calling the time function, and -socking away the results. By default, the constant is 0. The following -procedure can be used to obtain a better constant for a given platform (see -:ref:`profile-limitations`). :: - - import profile - pr = profile.Profile() - for i in range(5): - print(pr.calibrate(10000)) - -The method executes the number of Python calls given by the argument, directly -and again under the profiler, measuring the time for both. It then computes the -hidden overhead per profiler event, and returns that as a float. For example, -on a 1.8Ghz Intel Core i5 running Mac OS X, and using Python's time.clock() as -the timer, the magical number is about 4.04e-6. - -The object of this exercise is to get a fairly consistent result. If your -computer is *very* fast, or your timer function has poor resolution, you might -have to pass 100000, or even 1000000, to get consistent results. - -When you have a consistent answer, there are three ways you can use it:: - - import profile - - # 1. Apply computed bias to all Profile instances created hereafter. - profile.Profile.bias = your_computed_bias - - # 2. Apply computed bias to a specific Profile instance. - pr = profile.Profile() - pr.bias = your_computed_bias - - # 3. Specify computed bias in instance constructor. - pr = profile.Profile(bias=your_computed_bias) - -If you have a choice, you are better off choosing a smaller constant, and then -your results will "less often" show up as negative in profile statistics. - -.. _profile-timers: - -Using a custom timer -==================== - -If you want to change how current time is determined (for example, to force use -of wall-clock time or elapsed process time), pass the timing function you want -to the :class:`Profile` class constructor:: - - pr = profile.Profile(your_time_func) - -The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, -``your_time_func``'s return value will be interpreted differently: - -:class:`profile.Profile` - ``your_time_func`` should return a single number, or a list of numbers whose - sum is the current time (like what :func:`os.times` returns). If the - function returns a single time number, or the list of returned numbers has - length 2, then you will get an especially fast version of the dispatch - routine. - - Be warned that you should calibrate the profiler class for the timer function - that you choose (see :ref:`profile-calibration`). For most machines, a timer - that returns a lone integer value will provide the best results in terms of - low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a - better timer in the cleanest fashion, derive a class and hardwire a - replacement dispatch method that best handles your timer call, along with the - appropriate calibration constant. - -:class:`cProfile.Profile` - ``your_time_func`` should return a single number. If it returns integers, - you can also invoke the class constructor with a second argument specifying - the real duration of one unit of time. For example, if - ``your_integer_time_func`` returns times measured in thousands of seconds, - you would construct the :class:`Profile` instance as follows:: - - pr = cProfile.Profile(your_integer_time_func, 0.001) - - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. - -Python 3.3 adds several new functions in :mod:`time` that can be used to make -precise measurements of process or wall-clock time. For example, see -:func:`time.perf_counter`. diff --git a/third_party/python/Doc/library/pty.rst b/third_party/python/Doc/library/pty.rst deleted file mode 100644 index 0ab766065..000000000 --- a/third_party/python/Doc/library/pty.rst +++ /dev/null @@ -1,93 +0,0 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== - -.. module:: pty - :platform: Linux - :synopsis: Pseudo-Terminal Handling for Linux. - -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/pty.py` - --------------- - -The :mod:`pty` module defines operations for handling the pseudo-terminal -concept: starting another process and being able to write to and read from its -controlling terminal programmatically. - -Because pseudo-terminal handling is highly platform dependent, there is code to -do it only for Linux. (The Linux code is supposed to work on other platforms, -but hasn't been tested yet.) - -The :mod:`pty` module defines the following functions: - - -.. function:: fork() - - Fork. Connect the child's controlling terminal to a pseudo-terminal. Return - value is ``(pid, fd)``. Note that the child gets *pid* 0, and the *fd* is - *invalid*. The parent's return value is the *pid* of the child, and *fd* is a - file descriptor connected to the child's controlling terminal (and also to the - child's standard input and output). - - -.. function:: openpty() - - Open a new pseudo-terminal pair, using :func:`os.openpty` if possible, or - emulation code for generic Unix systems. Return a pair of file descriptors - ``(master, slave)``, for the master and the slave end, respectively. - - -.. function:: spawn(argv[, master_read[, stdin_read]]) - - Spawn a process, and connect its controlling terminal with the current - process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. - - The functions *master_read* and *stdin_read* should be functions which read from - a file descriptor. The defaults try to read 1024 bytes each time they are - called. - - .. versionchanged:: 3.4 - :func:`spawn` now returns the status value from :func:`os.waitpid` - on the child process. - -Example -------- - -.. sectionauthor:: Steen Lumholt - -The following program acts like the Unix command :manpage:`script(1)`, using a -pseudo-terminal to record all input and output of a terminal session in a -"typescript". :: - - import argparse - import os - import pty - import sys - import time - - parser = argparse.ArgumentParser() - parser.add_argument('-a', dest='append', action='store_true') - parser.add_argument('-p', dest='use_python', action='store_true') - parser.add_argument('filename', nargs='?', default='typescript') - options = parser.parse_args() - - shell = sys.executable if options.use_python else os.environ.get('SHELL', 'sh') - filename = options.filename - mode = 'ab' if options.append else 'wb' - - with open(filename, mode) as script: - def read(fd): - data = os.read(fd, 1024) - script.write(data) - return data - - print('Script started, file is', filename) - script.write(('Script started on %s\n' % time.asctime()).encode()) - - pty.spawn(shell, read) - - script.write(('Script done on %s\n' % time.asctime()).encode()) - print('Script done, file is', filename) diff --git a/third_party/python/Doc/library/pwd.rst b/third_party/python/Doc/library/pwd.rst deleted file mode 100644 index 03ebb02e4..000000000 --- a/third_party/python/Doc/library/pwd.rst +++ /dev/null @@ -1,76 +0,0 @@ -:mod:`pwd` --- The password database -==================================== - -.. module:: pwd - :platform: Unix - :synopsis: The password database (getpwnam() and friends). - --------------- - -This module provides access to the Unix user account and password database. It -is available on all Unix versions. - -Password database entries are reported as a tuple-like object, whose attributes -correspond to the members of the ``passwd`` structure (Attribute field below, -see ````): - -+-------+---------------+-----------------------------+ -| Index | Attribute | Meaning | -+=======+===============+=============================+ -| 0 | ``pw_name`` | Login name | -+-------+---------------+-----------------------------+ -| 1 | ``pw_passwd`` | Optional encrypted password | -+-------+---------------+-----------------------------+ -| 2 | ``pw_uid`` | Numerical user ID | -+-------+---------------+-----------------------------+ -| 3 | ``pw_gid`` | Numerical group ID | -+-------+---------------+-----------------------------+ -| 4 | ``pw_gecos`` | User name or comment field | -+-------+---------------+-----------------------------+ -| 5 | ``pw_dir`` | User home directory | -+-------+---------------+-----------------------------+ -| 6 | ``pw_shell`` | User command interpreter | -+-------+---------------+-----------------------------+ - -The uid and gid items are integers, all others are strings. :exc:`KeyError` is -raised if the entry asked for cannot be found. - -.. note:: - - .. index:: module: crypt - - In traditional Unix the field ``pw_passwd`` usually contains a password - encrypted with a DES derived algorithm (see module :mod:`crypt`). However most - modern unices use a so-called *shadow password* system. On those unices the - *pw_passwd* field only contains an asterisk (``'*'``) or the letter ``'x'`` - where the encrypted password is stored in a file :file:`/etc/shadow` which is - not world readable. Whether the *pw_passwd* field contains anything useful is - system-dependent. If available, the :mod:`spwd` module should be used where - access to the encrypted password is required. - -It defines the following items: - - -.. function:: getpwuid(uid) - - Return the password database entry for the given numeric user ID. - - -.. function:: getpwnam(name) - - Return the password database entry for the given user name. - - -.. function:: getpwall() - - Return a list of all available password database entries, in arbitrary order. - - -.. seealso:: - - Module :mod:`grp` - An interface to the group database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/third_party/python/Doc/library/py_compile.rst b/third_party/python/Doc/library/py_compile.rst deleted file mode 100644 index 0af8fb15b..000000000 --- a/third_party/python/Doc/library/py_compile.rst +++ /dev/null @@ -1,89 +0,0 @@ -:mod:`py_compile` --- Compile Python source files -================================================= - -.. module:: py_compile - :synopsis: Generate byte-code files from Python source files. - -.. sectionauthor:: Fred L. Drake, Jr. -.. documentation based on module docstrings - -**Source code:** :source:`Lib/py_compile.py` - -.. index:: pair: file; byte-code - --------------- - -The :mod:`py_compile` module provides a function to generate a byte-code file -from a source file, and another function used when the module source file is -invoked as a script. - -Though not often needed, this function can be useful when installing modules for -shared use, especially if some of the users may not have permission to write the -byte-code cache files in the directory containing the source code. - - -.. exception:: PyCompileError - - Exception raised when an error occurs while attempting to compile the file. - - -.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) - - Compile a source file to byte-code and write out the byte-code cache file. - The source code is loaded from the file named *file*. The byte-code is - written to *cfile*, which defaults to the :pep:`3147`/:pep:`488` path, ending - in ``.pyc``. - For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to - ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is - specified, it is used as the name of the source file in error messages when - instead of *file*. If *doraise* is true, a :exc:`PyCompileError` is raised - when an error is encountered while compiling *file*. If *doraise* is false - (the default), an error string is written to ``sys.stderr``, but no exception - is raised. This function returns the path to byte-compiled file, i.e. - whatever *cfile* value was used. - - If the path that *cfile* becomes (either explicitly specified or computed) - is a symlink or non-regular file, :exc:`FileExistsError` will be raised. - This is to act as a warning that import will turn those paths into regular - files if it is allowed to write byte-compiled files to those paths. This is - a side-effect of import using file renaming to place the final byte-compiled - file into place to prevent concurrent file writing issues. - - *optimize* controls the optimization level and is passed to the built-in - :func:`compile` function. The default of ``-1`` selects the optimization - level of the current interpreter. - - .. versionchanged:: 3.2 - Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous - default was *file* + ``'c'`` (``'o'`` if optimization was enabled). - Also added the *optimize* parameter. - - .. versionchanged:: 3.4 - Changed code to use :mod:`importlib` for the byte-code cache file writing. - This means file creation/writing semantics now match what :mod:`importlib` - does, e.g. permissions, write-and-move semantics, etc. Also added the - caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or - non-regular file. - - -.. function:: main(args=None) - - Compile several source files. The files named in *args* (or on the command - line, if *args* is ``None``) are compiled and the resulting byte-code is - cached in the normal manner. This function does not search a directory - structure to locate source files; it only compiles files named explicitly. - If ``'-'`` is the only parameter in args, the list of files is taken from - standard input. - - .. versionchanged:: 3.2 - Added support for ``'-'``. - -When this module is run as a script, the :func:`main` is used to compile all the -files named on the command line. The exit status is nonzero if one of the files -could not be compiled. - - -.. seealso:: - - Module :mod:`compileall` - Utilities to compile all Python source files in a directory tree. diff --git a/third_party/python/Doc/library/pyclbr.rst b/third_party/python/Doc/library/pyclbr.rst deleted file mode 100644 index 32842717b..000000000 --- a/third_party/python/Doc/library/pyclbr.rst +++ /dev/null @@ -1,116 +0,0 @@ -:mod:`pyclbr` --- Python class browser support -============================================== - -.. module:: pyclbr - :synopsis: Supports information extraction for a Python class browser. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/pyclbr.py` - --------------- - -The :mod:`pyclbr` module can be used to determine some limited information -about the classes, methods and top-level functions defined in a module. The -information provided is sufficient to implement a traditional three-pane -class browser. The information is extracted from the source code rather -than by importing the module, so this module is safe to use with untrusted -code. This restriction makes it impossible to use this module with modules -not implemented in Python, including all standard and optional extension -modules. - - -.. function:: readmodule(module, path=None) - - Read a module and return a dictionary mapping class names to class - descriptor objects. The parameter *module* should be the name of a - module as a string; it may be the name of a module within a package. The - *path* parameter should be a sequence, and is used to augment the value - of ``sys.path``, which is used to locate module source code. - - -.. function:: readmodule_ex(module, path=None) - - Like :func:`readmodule`, but the returned dictionary, in addition to - mapping class names to class descriptor objects, also maps top-level - function names to function descriptor objects. Moreover, if the module - being read is a package, the key ``'__path__'`` in the returned - dictionary has as its value a list which contains the package search - path. - - -.. _pyclbr-class-objects: - -Class Objects -------------- - -The :class:`Class` objects used as values in the dictionary returned by -:func:`readmodule` and :func:`readmodule_ex` provide the following data -attributes: - - -.. attribute:: Class.module - - The name of the module defining the class described by the class descriptor. - - -.. attribute:: Class.name - - The name of the class. - - -.. attribute:: Class.super - - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule` are - listed as a string with the class name instead of as :class:`Class` - objects. - - -.. attribute:: Class.methods - - A dictionary mapping method names to line numbers. - - -.. attribute:: Class.file - - Name of the file containing the ``class`` statement defining the class. - - -.. attribute:: Class.lineno - - The line number of the ``class`` statement within the file named by - :attr:`~Class.file`. - - -.. _pyclbr-function-objects: - -Function Objects ----------------- - -The :class:`Function` objects used as values in the dictionary returned by -:func:`readmodule_ex` provide the following attributes: - - -.. attribute:: Function.module - - The name of the module defining the function described by the function - descriptor. - - -.. attribute:: Function.name - - The name of the function. - - -.. attribute:: Function.file - - Name of the file containing the ``def`` statement defining the function. - - -.. attribute:: Function.lineno - - The line number of the ``def`` statement within the file named by - :attr:`~Function.file`. - diff --git a/third_party/python/Doc/library/pydoc.rst b/third_party/python/Doc/library/pydoc.rst deleted file mode 100644 index f1bfab9a3..000000000 --- a/third_party/python/Doc/library/pydoc.rst +++ /dev/null @@ -1,100 +0,0 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== - -.. module:: pydoc - :synopsis: Documentation generator and online help system. - -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - -**Source code:** :source:`Lib/pydoc.py` - -.. index:: - single: documentation; generation - single: documentation; online - single: help; online - --------------- - -The :mod:`pydoc` module automatically generates documentation from Python -modules. The documentation can be presented as pages of text on the console, -served to a Web browser, or saved to HTML files. - -For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, -and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just -above the definition of the class, function or method in the source file, or at -the top of the module (see :func:`inspect.getcomments`). - -The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation -as text on the console. The same text documentation can also be viewed from -outside the Python interpreter by running :program:`pydoc` as a script at the -operating system's command prompt. For example, running :: - - pydoc sys - -at a shell prompt will display documentation on the :mod:`sys` module, in a -style similar to the manual pages shown by the Unix :program:`man` command. The -argument to :program:`pydoc` can be the name of a function, module, or package, -or a dotted reference to a class, method, or function within a module or module -in a package. If the argument to :program:`pydoc` looks like a path (that is, -it contains the path separator for your operating system, such as a slash in -Unix), and refers to an existing Python source file, then documentation is -produced for that file. - -.. note:: - - In order to find objects and their documentation, :mod:`pydoc` imports the - module(s) to be documented. Therefore, any code on module level will be - executed on that occasion. Use an ``if __name__ == '__main__':`` guard to - only execute code when a file is invoked as a script and not just imported. - -When printing output to the console, :program:`pydoc` attempts to paginate the -output for easier reading. If the :envvar:`PAGER` environment variable is set, -:program:`pydoc` will use its value as a pagination program. - -Specifying a ``-w`` flag before the argument will cause HTML documentation -to be written out to a file in the current directory, instead of displaying text -on the console. - -Specifying a ``-k`` flag before the argument will search the synopsis -lines of all available modules for the keyword given as the argument, again in a -manner similar to the Unix :program:`man` command. The synopsis line of a -module is the first line of its documentation string. - -You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting Web browsers. :program:`pydoc -p 1234` -will start a HTTP server on port 1234, allowing you to browse the -documentation at ``http://localhost:1234/`` in your preferred Web browser. -Specifying ``0`` as the port number will select an arbitrary unused port. - -:program:`pydoc -b` will start the server and additionally open a web -browser to a module index page. Each served page has a navigation bar at the -top where you can *Get* help on an individual item, *Search* all modules with a -keyword in their synopsis line, and go to the *Module index*, *Topics* and -*Keywords* pages. - -When :program:`pydoc` generates documentation, it uses the current environment -and path to locate modules. Thus, invoking :program:`pydoc spam` -documents precisely the version of the module you would get if you started the -Python interpreter and typed ``import spam``. - -Module docs for core modules are assumed to reside in -``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the -major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable -to a different URL or to a local directory containing the Library -Reference Manual pages. - -.. versionchanged:: 3.2 - Added the ``-b`` option. - -.. versionchanged:: 3.3 - The ``-g`` command line option was removed. - -.. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than - :func:`inspect.getfullargspec` to extract signature information from - callables. diff --git a/third_party/python/Doc/library/pyexpat.rst b/third_party/python/Doc/library/pyexpat.rst deleted file mode 100644 index e43b9aecd..000000000 --- a/third_party/python/Doc/library/pyexpat.rst +++ /dev/null @@ -1,876 +0,0 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= - -.. module:: xml.parsers.expat - :synopsis: An interface to the Expat non-validating XML parser. - -.. moduleauthor:: Paul Prescod - --------------- - -.. Markup notes: - - Many of the attributes of the XMLParser objects are callbacks. Since - signature information must be presented, these are described using the method - directive. Since they are attributes which are set by client code, in-text - references to these attributes should be marked using the :member: role. - - -.. warning:: - - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - -.. index:: single: Expat - -The :mod:`xml.parsers.expat` module is a Python interface to the Expat -non-validating XML parser. The module provides a single extension type, -:class:`xmlparser`, that represents the current state of an XML parser. After -an :class:`xmlparser` object has been created, various attributes of the object -can be set to handler functions. When an XML document is then fed to the -parser, the handler functions are called for the character data and markup in -the XML document. - -.. index:: module: pyexpat - -This module uses the :mod:`pyexpat` module to provide access to the Expat -parser. Direct use of the :mod:`pyexpat` module is deprecated. - -This module provides one exception and one type object: - - -.. exception:: ExpatError - - The exception raised when Expat reports an error. See section - :ref:`expaterror-objects` for more information on interpreting Expat errors. - - -.. exception:: error - - Alias for :exc:`ExpatError`. - - -.. data:: XMLParserType - - The type of the return values from the :func:`ParserCreate` function. - -The :mod:`xml.parsers.expat` module contains two functions: - - -.. function:: ErrorString(errno) - - Returns an explanatory string for a given error number *errno*. - - -.. function:: ParserCreate(encoding=None, namespace_separator=None) - - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. - - Expat can optionally do XML namespace processing for you, enabled by providing a - value for *namespace_separator*. The value must be a one-character string; a - :exc:`ValueError` will be raised if the string has an illegal length (``None`` - is considered the same as omission). When namespace processing is enabled, - element type names and attribute names that belong to a namespace will be - expanded. The element name passed to the element handlers - :attr:`StartElementHandler` and :attr:`EndElementHandler` will be the - concatenation of the namespace URI, the namespace separator character, and the - local part of the name. If the namespace separator is a zero byte (``chr(0)``) - then the namespace URI and the local part will be concatenated without any - separator. - - For example, if *namespace_separator* is set to a space character (``' '``) and - the following document is parsed: - - .. code-block:: xml - - - - - - - - :attr:`StartElementHandler` will receive the following strings for each - element:: - - http://default-namespace.org/ root - http://www.python.org/ns/ elem1 - elem2 - - Due to limitations in the ``Expat`` library used by :mod:`pyexpat`, - the :class:`xmlparser` instance returned can only be used to parse a single - XML document. Call ``ParserCreate`` for each document to provide unique - parser instances. - - -.. seealso:: - - `The Expat XML Parser `_ - Home page of the Expat project. - - -.. _xmlparser-objects: - -XMLParser Objects ------------------ - -:class:`xmlparser` objects have the following methods: - - -.. method:: xmlparser.Parse(data[, isfinal]) - - Parses the contents of the string *data*, calling the appropriate handler - functions to process the parsed data. *isfinal* must be true on the final call - to this method; it allows the parsing of a single file in fragments, - not the submission of multiple files. - *data* can be the empty string at any time. - - -.. method:: xmlparser.ParseFile(file) - - Parse XML data reading from the object *file*. *file* only needs to provide - the ``read(nbytes)`` method, returning the empty string when there's no more - data. - - -.. method:: xmlparser.SetBase(base) - - Sets the base to be used for resolving relative URIs in system identifiers in - declarations. Resolving relative identifiers is left to the application: this - value will be passed through as the *base* argument to the - :func:`ExternalEntityRefHandler`, :func:`NotationDeclHandler`, and - :func:`UnparsedEntityDeclHandler` functions. - - -.. method:: xmlparser.GetBase() - - Returns a string containing the base set by a previous call to :meth:`SetBase`, - or ``None`` if :meth:`SetBase` hasn't been called. - - -.. method:: xmlparser.GetInputContext() - - Returns the input data that generated the current event as a string. The data is - in the encoding of the entity which contains the text. When called while an - event handler is not active, the return value is ``None``. - - -.. method:: xmlparser.ExternalEntityParserCreate(context[, encoding]) - - Create a "child" parser which can be used to parse an external parsed entity - referred to by content parsed by the parent parser. The *context* parameter - should be the string passed to the :meth:`ExternalEntityRefHandler` handler - function, described below. The child parser is created with the - :attr:`ordered_attributes` and :attr:`specified_attributes` set to the values of - this parser. - -.. method:: xmlparser.SetParamEntityParsing(flag) - - Control parsing of parameter entities (including the external DTD subset). - Possible *flag* values are :const:`XML_PARAM_ENTITY_PARSING_NEVER`, - :const:`XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE` and - :const:`XML_PARAM_ENTITY_PARSING_ALWAYS`. Return true if setting the flag - was successful. - -.. method:: xmlparser.UseForeignDTD([flag]) - - Calling this with a true value for *flag* (the default) will cause Expat to call - the :attr:`ExternalEntityRefHandler` with :const:`None` for all arguments to - allow an alternate DTD to be loaded. If the document does not contain a - document type declaration, the :attr:`ExternalEntityRefHandler` will still be - called, but the :attr:`StartDoctypeDeclHandler` and - :attr:`EndDoctypeDeclHandler` will not be called. - - Passing a false value for *flag* will cancel a previous call that passed a true - value, but otherwise has no effect. - - This method can only be called before the :meth:`Parse` or :meth:`ParseFile` - methods are called; calling it after either of those have been called causes - :exc:`ExpatError` to be raised with the :attr:`code` attribute set to - ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. - -:class:`xmlparser` objects have the following attributes: - - -.. attribute:: xmlparser.buffer_size - - The size of the buffer used when :attr:`buffer_text` is true. - A new buffer size can be set by assigning a new integer value - to this attribute. - When the size is changed, the buffer will be flushed. - - -.. attribute:: xmlparser.buffer_text - - Setting this to true causes the :class:`xmlparser` object to buffer textual - content returned by Expat to avoid multiple calls to the - :meth:`CharacterDataHandler` callback whenever possible. This can improve - performance substantially since Expat normally breaks character data into chunks - at every line ending. This attribute is false by default, and may be changed at - any time. - - -.. attribute:: xmlparser.buffer_used - - If :attr:`buffer_text` is enabled, the number of bytes stored in the buffer. - These bytes represent UTF-8 encoded text. This attribute has no meaningful - interpretation when :attr:`buffer_text` is false. - - -.. attribute:: xmlparser.ordered_attributes - - Setting this attribute to a non-zero integer causes the attributes to be - reported as a list rather than a dictionary. The attributes are presented in - the order found in the document text. For each attribute, two list entries are - presented: the attribute name and the attribute value. (Older versions of this - module also used this format.) By default, this attribute is false; it may be - changed at any time. - - -.. attribute:: xmlparser.specified_attributes - - If set to a non-zero integer, the parser will report only those attributes which - were specified in the document instance and not those which were derived from - attribute declarations. Applications which set this need to be especially - careful to use what additional information is available from the declarations as - needed to comply with the standards for the behavior of XML processors. By - default, this attribute is false; it may be changed at any time. - - -The following attributes contain values relating to the most recent error -encountered by an :class:`xmlparser` object, and will only have correct values -once a call to :meth:`Parse` or :meth:`ParseFile` has raised an -:exc:`xml.parsers.expat.ExpatError` exception. - - -.. attribute:: xmlparser.ErrorByteIndex - - Byte index at which an error occurred. - - -.. attribute:: xmlparser.ErrorCode - - Numeric code specifying the problem. This value can be passed to the - :func:`ErrorString` function, or compared to one of the constants defined in the - ``errors`` object. - - -.. attribute:: xmlparser.ErrorColumnNumber - - Column number at which an error occurred. - - -.. attribute:: xmlparser.ErrorLineNumber - - Line number at which an error occurred. - -The following attributes contain values relating to the current parse location -in an :class:`xmlparser` object. During a callback reporting a parse event they -indicate the location of the first of the sequence of characters that generated -the event. When called outside of a callback, the position indicated will be -just past the last parse event (regardless of whether there was an associated -callback). - - -.. attribute:: xmlparser.CurrentByteIndex - - Current byte index in the parser input. - - -.. attribute:: xmlparser.CurrentColumnNumber - - Current column number in the parser input. - - -.. attribute:: xmlparser.CurrentLineNumber - - Current line number in the parser input. - -Here is the list of handlers that can be set. To set a handler on an -:class:`xmlparser` object *o*, use ``o.handlername = func``. *handlername* must -be taken from the following list, and *func* must be a callable object accepting -the correct number of arguments. The arguments are all strings, unless -otherwise stated. - - -.. method:: xmlparser.XmlDeclHandler(version, encoding, standalone) - - Called when the XML declaration is parsed. The XML declaration is the - (optional) declaration of the applicable version of the XML recommendation, the - encoding of the document text, and an optional "standalone" declaration. - *version* and *encoding* will be strings, and *standalone* will be ``1`` if the - document is declared standalone, ``0`` if it is declared not to be standalone, - or ``-1`` if the standalone clause was omitted. This is only available with - Expat version 1.95.0 or newer. - - -.. method:: xmlparser.StartDoctypeDeclHandler(doctypeName, systemId, publicId, has_internal_subset) - - Called when Expat begins parsing the document type declaration (``'``. - - -.. method:: xmlparser.StartCdataSectionHandler() - - Called at the start of a CDATA section. This and :attr:`EndCdataSectionHandler` - are needed to be able to identify the syntactical start and end for CDATA - sections. - - -.. method:: xmlparser.EndCdataSectionHandler() - - Called at the end of a CDATA section. - - -.. method:: xmlparser.DefaultHandler(data) - - Called for any characters in the XML document for which no applicable handler - has been specified. This means characters that are part of a construct which - could be reported, but for which no handler has been supplied. - - -.. method:: xmlparser.DefaultHandlerExpand(data) - - This is the same as the :func:`DefaultHandler`, but doesn't inhibit expansion - of internal entities. The entity reference will not be passed to the default - handler. - - -.. method:: xmlparser.NotStandaloneHandler() - - Called if the XML document hasn't been declared as being a standalone document. - This happens when there is an external subset or a reference to a parameter - entity, but the XML declaration does not set standalone to ``yes`` in an XML - declaration. If this handler returns ``0``, then the parser will raise an - :const:`XML_ERROR_NOT_STANDALONE` error. If this handler is not set, no - exception is raised by the parser for this condition. - - -.. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) - - Called for references to external entities. *base* is the current base, as set - by a previous call to :meth:`SetBase`. The public and system identifiers, - *systemId* and *publicId*, are strings if given; if the public identifier is not - given, *publicId* will be ``None``. The *context* value is opaque and should - only be used as described below. - - For external entities to be parsed, this handler must be implemented. It is - responsible for creating the sub-parser using - ``ExternalEntityParserCreate(context)``, initializing it with the appropriate - callbacks, and parsing the entity. This handler should return an integer; if it - returns ``0``, the parser will raise an - :const:`XML_ERROR_EXTERNAL_ENTITY_HANDLING` error, otherwise parsing will - continue. - - If this handler is not provided, external entities are reported by the - :attr:`DefaultHandler` callback, if provided. - - -.. _expaterror-objects: - -ExpatError Exceptions ---------------------- - -.. sectionauthor:: Fred L. Drake, Jr. - - -:exc:`ExpatError` exceptions have a number of interesting attributes: - - -.. attribute:: ExpatError.code - - Expat's internal error number for the specific error. The - :data:`errors.messages ` dictionary maps - these error numbers to Expat's error messages. For example:: - - from xml.parsers.expat import ParserCreate, ExpatError, errors - - p = ParserCreate() - try: - p.Parse(some_xml_document) - except ExpatError as err: - print("Error:", errors.messages[err.code]) - - The :mod:`~xml.parsers.expat.errors` module also provides error message - constants and a dictionary :data:`~xml.parsers.expat.errors.codes` mapping - these messages back to the error codes, see below. - - -.. attribute:: ExpatError.lineno - - Line number on which the error was detected. The first line is numbered ``1``. - - -.. attribute:: ExpatError.offset - - Character offset into the line where the error occurred. The first column is - numbered ``0``. - - -.. _expat-example: - -Example -------- - -The following program defines three handlers that just print out their -arguments. :: - - import xml.parsers.expat - - # 3 handler functions - def start_element(name, attrs): - print('Start element:', name, attrs) - def end_element(name): - print('End element:', name) - def char_data(data): - print('Character data:', repr(data)) - - p = xml.parsers.expat.ParserCreate() - - p.StartElementHandler = start_element - p.EndElementHandler = end_element - p.CharacterDataHandler = char_data - - p.Parse(""" - Text goes here - More text - """, 1) - -The output from this program is:: - - Start element: parent {'id': 'top'} - Start element: child1 {'name': 'paul'} - Character data: 'Text goes here' - End element: child1 - Character data: '\n' - Start element: child2 {'name': 'fred'} - Character data: 'More text' - End element: child2 - Character data: '\n' - End element: parent - - -.. _expat-content-models: - -Content Model Descriptions --------------------------- - -.. module:: xml.parsers.expat.model - -.. sectionauthor:: Fred L. Drake, Jr. - -Content models are described using nested tuples. Each tuple contains four -values: the type, the quantifier, the name, and a tuple of children. Children -are simply additional content model descriptions. - -The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two -groups: the model type group and the quantifier group. - -The constants in the model type group are: - - -.. data:: XML_CTYPE_ANY - :noindex: - - The element named by the model name was declared to have a content model of - ``ANY``. - - -.. data:: XML_CTYPE_CHOICE - :noindex: - - The named element allows a choice from a number of options; this is used for - content models such as ``(A | B | C)``. - - -.. data:: XML_CTYPE_EMPTY - :noindex: - - Elements which are declared to be ``EMPTY`` have this model type. - - -.. data:: XML_CTYPE_MIXED - :noindex: - - -.. data:: XML_CTYPE_NAME - :noindex: - - -.. data:: XML_CTYPE_SEQ - :noindex: - - Models which represent a series of models which follow one after the other are - indicated with this model type. This is used for models such as ``(A, B, C)``. - -The constants in the quantifier group are: - - -.. data:: XML_CQUANT_NONE - :noindex: - - No modifier is given, so it can appear exactly once, as for ``A``. - - -.. data:: XML_CQUANT_OPT - :noindex: - - The model is optional: it can appear once or not at all, as for ``A?``. - - -.. data:: XML_CQUANT_PLUS - :noindex: - - The model must occur one or more times (like ``A+``). - - -.. data:: XML_CQUANT_REP - :noindex: - - The model must occur zero or more times, as for ``A*``. - - -.. _expat-errors: - -Expat error constants ---------------------- - -.. module:: xml.parsers.expat.errors - -The following constants are provided in the :mod:`xml.parsers.expat.errors` -module. These constants are useful in interpreting some of the attributes of -the :exc:`ExpatError` exception objects raised when an error has occurred. -Since for backwards compatibility reasons, the constants' value is the error -*message* and not the numeric error *code*, you do this by comparing its -:attr:`code` attribute with -:samp:`errors.codes[errors.XML_ERROR_{CONSTANT_NAME}]`. - -The ``errors`` module has the following attributes: - -.. data:: codes - - A dictionary mapping numeric error codes to their string descriptions. - - .. versionadded:: 3.2 - - -.. data:: messages - - A dictionary mapping string descriptions to their error codes. - - .. versionadded:: 3.2 - - -.. data:: XML_ERROR_ASYNC_ENTITY - - -.. data:: XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF - - An entity reference in an attribute value referred to an external entity instead - of an internal entity. - - -.. data:: XML_ERROR_BAD_CHAR_REF - - A character reference referred to a character which is illegal in XML (for - example, character ``0``, or '``�``'). - - -.. data:: XML_ERROR_BINARY_ENTITY_REF - - An entity reference referred to an entity which was declared with a notation, so - cannot be parsed. - - -.. data:: XML_ERROR_DUPLICATE_ATTRIBUTE - - An attribute was used more than once in a start tag. - - -.. data:: XML_ERROR_INCORRECT_ENCODING - - -.. data:: XML_ERROR_INVALID_TOKEN - - Raised when an input byte could not properly be assigned to a character; for - example, a NUL byte (value ``0``) in a UTF-8 input stream. - - -.. data:: XML_ERROR_JUNK_AFTER_DOC_ELEMENT - - Something other than whitespace occurred after the document element. - - -.. data:: XML_ERROR_MISPLACED_XML_PI - - An XML declaration was found somewhere other than the start of the input data. - - -.. data:: XML_ERROR_NO_ELEMENTS - - The document contains no elements (XML requires all documents to contain exactly - one top-level element).. - - -.. data:: XML_ERROR_NO_MEMORY - - Expat was not able to allocate memory internally. - - -.. data:: XML_ERROR_PARAM_ENTITY_REF - - A parameter entity reference was found where it was not allowed. - - -.. data:: XML_ERROR_PARTIAL_CHAR - - An incomplete character was found in the input. - - -.. data:: XML_ERROR_RECURSIVE_ENTITY_REF - - An entity reference contained another reference to the same entity; possibly via - a different name, and possibly indirectly. - - -.. data:: XML_ERROR_SYNTAX - - Some unspecified syntax error was encountered. - - -.. data:: XML_ERROR_TAG_MISMATCH - - An end tag did not match the innermost open start tag. - - -.. data:: XML_ERROR_UNCLOSED_TOKEN - - Some token (such as a start tag) was not closed before the end of the stream or - the next token was encountered. - - -.. data:: XML_ERROR_UNDEFINED_ENTITY - - A reference was made to an entity which was not defined. - - -.. data:: XML_ERROR_UNKNOWN_ENCODING - - The document encoding is not supported by Expat. - - -.. data:: XML_ERROR_UNCLOSED_CDATA_SECTION - - A CDATA marked section was not closed. - - -.. data:: XML_ERROR_EXTERNAL_ENTITY_HANDLING - - -.. data:: XML_ERROR_NOT_STANDALONE - - The parser determined that the document was not "standalone" though it declared - itself to be in the XML declaration, and the :attr:`NotStandaloneHandler` was - set and returned ``0``. - - -.. data:: XML_ERROR_UNEXPECTED_STATE - - -.. data:: XML_ERROR_ENTITY_DECLARED_IN_PE - - -.. data:: XML_ERROR_FEATURE_REQUIRES_XML_DTD - - An operation was requested that requires DTD support to be compiled in, but - Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. - - -.. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING - - A behavioral change was requested after parsing started that can only be changed - before parsing has started. This is (currently) only raised by - :meth:`UseForeignDTD`. - - -.. data:: XML_ERROR_UNBOUND_PREFIX - - An undeclared prefix was found when namespace processing was enabled. - - -.. data:: XML_ERROR_UNDECLARING_PREFIX - - The document attempted to remove the namespace declaration associated with a - prefix. - - -.. data:: XML_ERROR_INCOMPLETE_PE - - A parameter entity contained incomplete markup. - - -.. data:: XML_ERROR_XML_DECL - - The document contained no document element at all. - - -.. data:: XML_ERROR_TEXT_DECL - - There was an error parsing a text declaration in an external entity. - - -.. data:: XML_ERROR_PUBLICID - - Characters were found in the public id that are not allowed. - - -.. data:: XML_ERROR_SUSPENDED - - The requested operation was made on a suspended parser, but isn't allowed. This - includes attempts to provide additional input or to stop the parser. - - -.. data:: XML_ERROR_NOT_SUSPENDED - - An attempt to resume the parser was made when the parser had not been suspended. - - -.. data:: XML_ERROR_ABORTED - - This should not be reported to Python applications. - - -.. data:: XML_ERROR_FINISHED - - The requested operation was made on a parser which was finished parsing input, - but isn't allowed. This includes attempts to provide additional input or to - stop the parser. - - -.. data:: XML_ERROR_SUSPEND_PE - - -.. rubric:: Footnotes - -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. - diff --git a/third_party/python/Doc/library/python.rst b/third_party/python/Doc/library/python.rst deleted file mode 100644 index f307d7db6..000000000 --- a/third_party/python/Doc/library/python.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _python: - -*********************** -Python Runtime Services -*********************** - -The modules described in this chapter provide a wide range of services related -to the Python interpreter and its interaction with its environment. Here's an -overview: - - -.. toctree:: - - sys.rst - sysconfig.rst - builtins.rst - __main__.rst - warnings.rst - contextlib.rst - abc.rst - atexit.rst - traceback.rst - __future__.rst - gc.rst - inspect.rst - site.rst - fpectl.rst diff --git a/third_party/python/Doc/library/queue.rst b/third_party/python/Doc/library/queue.rst deleted file mode 100644 index bd0fc2d8f..000000000 --- a/third_party/python/Doc/library/queue.rst +++ /dev/null @@ -1,203 +0,0 @@ -:mod:`queue` --- A synchronized queue class -=========================================== - -.. module:: queue - :synopsis: A synchronized queue class. - -**Source code:** :source:`Lib/queue.py` - --------------- - -The :mod:`queue` module implements multi-producer, multi-consumer queues. -It is especially useful in threaded programming when information must be -exchanged safely between multiple threads. The :class:`Queue` class in this -module implements all the required locking semantics. It depends on the -availability of thread support in Python; see the :mod:`threading` -module. - -The module implements three types of queue, which differ only in the order in -which the entries are retrieved. In a :abbr:`FIFO (first-in, first-out)` -queue, the first tasks added are the first retrieved. In a -:abbr:`LIFO (last-in, first-out)` queue, the most recently added entry is -the first retrieved (operating like a stack). With a priority queue, -the entries are kept sorted (using the :mod:`heapq` module) and the -lowest valued entry is retrieved first. - -Internally, the module uses locks to temporarily block competing threads; -however, it is not designed to handle reentrancy within a thread. - -The :mod:`queue` module defines the following classes and exceptions: - -.. class:: Queue(maxsize=0) - - Constructor for a :abbr:`FIFO (first-in, first-out)` queue. *maxsize* is - an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - -.. class:: LifoQueue(maxsize=0) - - Constructor for a :abbr:`LIFO (last-in, first-out)` queue. *maxsize* is - an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - - -.. class:: PriorityQueue(maxsize=0) - - Constructor for a priority queue. *maxsize* is an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - - The lowest valued entries are retrieved first (the lowest valued entry is the - one returned by ``sorted(list(entries))[0]``). A typical pattern for entries - is a tuple in the form: ``(priority_number, data)``. - - -.. exception:: Empty - - Exception raised when non-blocking :meth:`~Queue.get` (or - :meth:`~Queue.get_nowait`) is called - on a :class:`Queue` object which is empty. - - -.. exception:: Full - - Exception raised when non-blocking :meth:`~Queue.put` (or - :meth:`~Queue.put_nowait`) is called - on a :class:`Queue` object which is full. - - -.. _queueobjects: - -Queue Objects -------------- - -Queue objects (:class:`Queue`, :class:`LifoQueue`, or :class:`PriorityQueue`) -provide the public methods described below. - - -.. method:: Queue.qsize() - - Return the approximate size of the queue. Note, qsize() > 0 doesn't - guarantee that a subsequent get() will not block, nor will qsize() < maxsize - guarantee that put() will not block. - - -.. method:: Queue.empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. If empty() - returns ``True`` it doesn't guarantee that a subsequent call to put() - will not block. Similarly, if empty() returns ``False`` it doesn't - guarantee that a subsequent call to get() will not block. - - -.. method:: Queue.full() - - Return ``True`` if the queue is full, ``False`` otherwise. If full() - returns ``True`` it doesn't guarantee that a subsequent call to get() - will not block. Similarly, if full() returns ``False`` it doesn't - guarantee that a subsequent call to put() will not block. - - -.. method:: Queue.put(item, block=True, timeout=None) - - Put *item* into the queue. If optional args *block* is true and *timeout* is - ``None`` (the default), block if necessary until a free slot is available. If - *timeout* is a positive number, it blocks at most *timeout* seconds and raises - the :exc:`Full` exception if no free slot was available within that time. - Otherwise (*block* is false), put an item on the queue if a free slot is - immediately available, else raise the :exc:`Full` exception (*timeout* is - ignored in that case). - - -.. method:: Queue.put_nowait(item) - - Equivalent to ``put(item, False)``. - - -.. method:: Queue.get(block=True, timeout=None) - - Remove and return an item from the queue. If optional args *block* is true and - *timeout* is ``None`` (the default), block if necessary until an item is available. - If *timeout* is a positive number, it blocks at most *timeout* seconds and - raises the :exc:`Empty` exception if no item was available within that time. - Otherwise (*block* is false), return an item if one is immediately available, - else raise the :exc:`Empty` exception (*timeout* is ignored in that case). - - -.. method:: Queue.get_nowait() - - Equivalent to ``get(False)``. - -Two methods are offered to support tracking whether enqueued tasks have been -fully processed by daemon consumer threads. - - -.. method:: Queue.task_done() - - Indicate that a formerly enqueued task is complete. Used by queue consumer - threads. For each :meth:`get` used to fetch a task, a subsequent call to - :meth:`task_done` tells the queue that the processing on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items have been - processed (meaning that a :meth:`task_done` call was received for every item - that had been :meth:`put` into the queue). - - Raises a :exc:`ValueError` if called more times than there were items placed in - the queue. - - -.. method:: Queue.join() - - Blocks until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the queue. - The count goes down whenever a consumer thread calls :meth:`task_done` to - indicate that the item was retrieved and all work on it is complete. When the - count of unfinished tasks drops to zero, :meth:`join` unblocks. - - -Example of how to wait for enqueued tasks to be completed:: - - def worker(): - while True: - item = q.get() - if item is None: - break - do_work(item) - q.task_done() - - q = queue.Queue() - threads = [] - for i in range(num_worker_threads): - t = threading.Thread(target=worker) - t.start() - threads.append(t) - - for item in source(): - q.put(item) - - # block until all tasks are done - q.join() - - # stop workers - for i in range(num_worker_threads): - q.put(None) - for t in threads: - t.join() - - -.. seealso:: - - Class :class:`multiprocessing.Queue` - A queue class for use in a multi-processing (rather than multi-threading) - context. - - :class:`collections.deque` is an alternative implementation of unbounded - queues with fast atomic :meth:`~collections.deque.append` and - :meth:`~collections.deque.popleft` operations that do not require locking. - diff --git a/third_party/python/Doc/library/quopri.rst b/third_party/python/Doc/library/quopri.rst deleted file mode 100644 index 86717c00c..000000000 --- a/third_party/python/Doc/library/quopri.rst +++ /dev/null @@ -1,63 +0,0 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== - -.. module:: quopri - :synopsis: Encode and decode files using the MIME quoted-printable encoding. - -**Source code:** :source:`Lib/quopri.py` - -.. index:: - pair: quoted-printable; encoding - single: MIME; quoted-printable encoding - --------------- - -This module performs quoted-printable transport encoding and decoding, as -defined in :rfc:`1521`: "MIME (Multipurpose Internet Mail Extensions) Part One: -Mechanisms for Specifying and Describing the Format of Internet Message Bodies". -The quoted-printable encoding is designed for data where there are relatively -few nonprintable characters; the base64 encoding scheme available via the -:mod:`base64` module is more compact if there are many such characters, as when -sending a graphics file. - -.. function:: decode(input, output, header=False) - - Decode the contents of the *input* file and write the resulting decoded binary - data to the *output* file. *input* and *output* must be :term:`binary file objects - `. If the optional argument *header* is present and true, underscore - will be decoded as space. This is used to decode "Q"-encoded headers as - described in :rfc:`1522`: "MIME (Multipurpose Internet Mail Extensions) - Part Two: Message Header Extensions for Non-ASCII Text". - - -.. function:: encode(input, output, quotetabs, header=False) - - Encode the contents of the *input* file and write the resulting quoted-printable - data to the *output* file. *input* and *output* must be - :term:`binary file objects `. *quotetabs*, a - non-optional flag which controls whether to encode embedded spaces - and tabs; when true it encodes such embedded whitespace, and when - false it leaves them unencoded. - Note that spaces and tabs appearing at the end of lines are always encoded, - as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded - as underscores as per :rfc:`1522`. - - -.. function:: decodestring(s, header=False) - - Like :func:`decode`, except that it accepts a source :class:`bytes` and - returns the corresponding decoded :class:`bytes`. - - -.. function:: encodestring(s, quotetabs=False, header=False) - - Like :func:`encode`, except that it accepts a source :class:`bytes` and - returns the corresponding encoded :class:`bytes`. By default, it sends a - ``False`` value to *quotetabs* parameter of the :func:`encode` function. - - - -.. seealso:: - - Module :mod:`base64` - Encode and decode MIME base64 data diff --git a/third_party/python/Doc/library/random.rst b/third_party/python/Doc/library/random.rst deleted file mode 100644 index 4f251574a..000000000 --- a/third_party/python/Doc/library/random.rst +++ /dev/null @@ -1,479 +0,0 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ - -.. module:: random - :synopsis: Generate pseudo-random numbers with various common distributions. - -**Source code:** :source:`Lib/random.py` - --------------- - -This module implements pseudo-random number generators for various -distributions. - -For integers, there is uniform selection from a range. For sequences, there is -uniform selection of a random element, a function to generate a random -permutation of a list in-place, and a function for random sampling without -replacement. - -On the real line, there are functions to compute uniform, normal (Gaussian), -lognormal, negative exponential, gamma, and beta distributions. For generating -distributions of angles, the von Mises distribution is available. - -Almost all module functions depend on the basic function :func:`.random`, which -generates a random float uniformly in the semi-open range [0.0, 1.0). Python -uses the Mersenne Twister as the core generator. It produces 53-bit precision -floats and has a period of 2\*\*19937-1. The underlying implementation in C is -both fast and threadsafe. The Mersenne Twister is one of the most extensively -tested random number generators in existence. However, being completely -deterministic, it is not suitable for all purposes, and is completely unsuitable -for cryptographic purposes. - -The functions supplied by this module are actually bound methods of a hidden -instance of the :class:`random.Random` class. You can instantiate your own -instances of :class:`Random` to get generators that don't share state. - -Class :class:`Random` can also be subclassed if you want to use a different -basic generator of your own devising: in that case, override the :meth:`~Random.random`, -:meth:`~Random.seed`, :meth:`~Random.getstate`, and :meth:`~Random.setstate` methods. -Optionally, a new generator can supply a :meth:`~Random.getrandbits` method --- this -allows :meth:`randrange` to produce selections over an arbitrarily large range. - -The :mod:`random` module also provides the :class:`SystemRandom` class which -uses the system function :func:`os.urandom` to generate random numbers -from sources provided by the operating system. - -.. warning:: - - The pseudo-random generators of this module should not be used for - security purposes. For security or cryptographic uses, see the - :mod:`secrets` module. - -.. seealso:: - - M. Matsumoto and T. Nishimura, "Mersenne Twister: A 623-dimensionally - equidistributed uniform pseudorandom number generator", ACM Transactions on - Modeling and Computer Simulation Vol. 8, No. 1, January pp.3--30 1998. - - - `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative - random number generator with a long period and comparatively simple update - operations. - - -Bookkeeping functions ---------------------- - -.. function:: seed(a=None, version=2) - - Initialize the random number generator. - - If *a* is omitted or ``None``, the current system time is used. If - randomness sources are provided by the operating system, they are used - instead of the system time (see the :func:`os.urandom` function for details - on availability). - - If *a* is an int, it is used directly. - - With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` - object gets converted to an :class:`int` and all of its bits are used. - - With version 1 (provided for reproducing random sequences from older versions - of Python), the algorithm for :class:`str` and :class:`bytes` generates a - narrower range of seeds. - - .. versionchanged:: 3.2 - Moved to the version 2 scheme which uses all of the bits in a string seed. - -.. function:: getstate() - - Return an object capturing the current internal state of the generator. This - object can be passed to :func:`setstate` to restore the state. - - -.. function:: setstate(state) - - *state* should have been obtained from a previous call to :func:`getstate`, and - :func:`setstate` restores the internal state of the generator to what it was at - the time :func:`getstate` was called. - - -.. function:: getrandbits(k) - - Returns a Python integer with *k* random bits. This method is supplied with - the MersenneTwister generator and some other generators may also provide it - as an optional part of the API. When available, :meth:`getrandbits` enables - :meth:`randrange` to handle arbitrarily large ranges. - - -Functions for integers ----------------------- - -.. function:: randrange(stop) - randrange(start, stop[, step]) - - Return a randomly selected element from ``range(start, stop, step)``. This is - equivalent to ``choice(range(start, stop, step))``, but doesn't actually build a - range object. - - The positional argument pattern matches that of :func:`range`. Keyword arguments - should not be used because the function may use them in unexpected ways. - - .. versionchanged:: 3.2 - :meth:`randrange` is more sophisticated about producing equally distributed - values. Formerly it used a style like ``int(random()*n)`` which could produce - slightly uneven distributions. - -.. function:: randint(a, b) - - Return a random integer *N* such that ``a <= N <= b``. Alias for - ``randrange(a, b+1)``. - - -Functions for sequences ------------------------ - -.. function:: choice(seq) - - Return a random element from the non-empty sequence *seq*. If *seq* is empty, - raises :exc:`IndexError`. - -.. function:: choices(population, weights=None, *, cum_weights=None, k=1) - - Return a *k* sized list of elements chosen from the *population* with replacement. - If the *population* is empty, raises :exc:`IndexError`. - - If a *weights* sequence is specified, selections are made according to the - relative weights. Alternatively, if a *cum_weights* sequence is given, the - selections are made according to the cumulative weights (perhaps computed - using :func:`itertools.accumulate`). For example, the relative weights - ``[10, 5, 30, 5]`` are equivalent to the cumulative weights - ``[10, 15, 45, 50]``. Internally, the relative weights are converted to - cumulative weights before making selections, so supplying the cumulative - weights saves work. - - If neither *weights* nor *cum_weights* are specified, selections are made - with equal probability. If a weights sequence is supplied, it must be - the same length as the *population* sequence. It is a :exc:`TypeError` - to specify both *weights* and *cum_weights*. - - The *weights* or *cum_weights* can use any numeric type that interoperates - with the :class:`float` values returned by :func:`random` (that includes - integers, floats, and fractions but excludes decimals). - - .. versionadded:: 3.6 - - -.. function:: shuffle(x[, random]) - - Shuffle the sequence *x* in place. - - The optional argument *random* is a 0-argument function returning a random - float in [0.0, 1.0); by default, this is the function :func:`.random`. - - To shuffle an immutable sequence and return a new shuffled list, use - ``sample(x, k=len(x))`` instead. - - Note that even for small ``len(x)``, the total number of permutations of *x* - can quickly grow larger than the period of most random number generators. - This implies that most permutations of a long sequence can never be - generated. For example, a sequence of length 2080 is the largest that - can fit within the period of the Mersenne Twister random number generator. - - -.. function:: sample(population, k) - - Return a *k* length list of unique elements chosen from the population sequence - or set. Used for random sampling without replacement. - - Returns a new list containing elements from the population while leaving the - original population unchanged. The resulting list is in selection order so that - all sub-slices will also be valid random samples. This allows raffle winners - (the sample) to be partitioned into grand prize and second place winners (the - subslices). - - Members of the population need not be :term:`hashable` or unique. If the population - contains repeats, then each occurrence is a possible selection in the sample. - - To choose a sample from a range of integers, use a :func:`range` object as an - argument. This is especially fast and space efficient for sampling from a large - population: ``sample(range(10000000), k=60)``. - - If the sample size is larger than the population size, a :exc:`ValueError` - is raised. - -Real-valued distributions -------------------------- - -The following functions generate specific real-valued distributions. Function -parameters are named after the corresponding variables in the distribution's -equation, as used in common mathematical practice; most of these equations can -be found in any statistics text. - - -.. function:: random() - - Return the next random floating point number in the range [0.0, 1.0). - - -.. function:: uniform(a, b) - - Return a random floating point number *N* such that ``a <= N <= b`` for - ``a <= b`` and ``b <= N <= a`` for ``b < a``. - - The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. - - -.. function:: triangular(low, high, mode) - - Return a random floating point number *N* such that ``low <= N <= high`` and - with the specified *mode* between those bounds. The *low* and *high* bounds - default to zero and one. The *mode* argument defaults to the midpoint - between the bounds, giving a symmetric distribution. - - -.. function:: betavariate(alpha, beta) - - Beta distribution. Conditions on the parameters are ``alpha > 0`` and - ``beta > 0``. Returned values range between 0 and 1. - - -.. function:: expovariate(lambd) - - Exponential distribution. *lambd* is 1.0 divided by the desired - mean. It should be nonzero. (The parameter would be called - "lambda", but that is a reserved word in Python.) Returned values - range from 0 to positive infinity if *lambd* is positive, and from - negative infinity to 0 if *lambd* is negative. - - -.. function:: gammavariate(alpha, beta) - - Gamma distribution. (*Not* the gamma function!) Conditions on the - parameters are ``alpha > 0`` and ``beta > 0``. - - The probability distribution function is:: - - x ** (alpha - 1) * math.exp(-x / beta) - pdf(x) = -------------------------------------- - math.gamma(alpha) * beta ** alpha - - -.. function:: gauss(mu, sigma) - - Gaussian distribution. *mu* is the mean, and *sigma* is the standard - deviation. This is slightly faster than the :func:`normalvariate` function - defined below. - - -.. function:: lognormvariate(mu, sigma) - - Log normal distribution. If you take the natural logarithm of this - distribution, you'll get a normal distribution with mean *mu* and standard - deviation *sigma*. *mu* can have any value, and *sigma* must be greater than - zero. - - -.. function:: normalvariate(mu, sigma) - - Normal distribution. *mu* is the mean, and *sigma* is the standard deviation. - - -.. function:: vonmisesvariate(mu, kappa) - - *mu* is the mean angle, expressed in radians between 0 and 2\*\ *pi*, and *kappa* - is the concentration parameter, which must be greater than or equal to zero. If - *kappa* is equal to zero, this distribution reduces to a uniform random angle - over the range 0 to 2\*\ *pi*. - - -.. function:: paretovariate(alpha) - - Pareto distribution. *alpha* is the shape parameter. - - -.. function:: weibullvariate(alpha, beta) - - Weibull distribution. *alpha* is the scale parameter and *beta* is the shape - parameter. - - -Alternative Generator ---------------------- - -.. class:: SystemRandom([seed]) - - Class that uses the :func:`os.urandom` function for generating random numbers - from sources provided by the operating system. Not available on all systems. - Does not rely on software state, and sequences are not reproducible. Accordingly, - the :meth:`seed` method has no effect and is ignored. - The :meth:`getstate` and :meth:`setstate` methods raise - :exc:`NotImplementedError` if called. - - -Notes on Reproducibility ------------------------- - -Sometimes it is useful to be able to reproduce the sequences given by a pseudo -random number generator. By re-using a seed value, the same sequence should be -reproducible from run to run as long as multiple threads are not running. - -Most of the random module's algorithms and seeding functions are subject to -change across Python versions, but two aspects are guaranteed not to change: - -* If a new seeding method is added, then a backward compatible seeder will be - offered. - -* The generator's :meth:`~Random.random` method will continue to produce the same - sequence when the compatible seeder is given the same seed. - -.. _random-examples: - -Examples and Recipes --------------------- - -Basic examples:: - - >>> random() # Random float: 0.0 <= x < 1.0 - 0.37444887175646646 - - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x < 10.0 - 3.1800146073117523 - - >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds - 5.148957571865031 - - >>> randrange(10) # Integer from 0 to 9 inclusive - 7 - - >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive - 26 - - >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence - 'draw' - - >>> deck = 'ace two three four'.split() - >>> shuffle(deck) # Shuffle a list - >>> deck - ['four', 'two', 'ace', 'three'] - - >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement - [40, 10, 50, 30] - -Simulations:: - - >>> # Six roulette wheel spins (weighted sampling with replacement) - >>> choices(['red', 'black', 'green'], [18, 18, 2], k=6) - ['red', 'green', 'black', 'black', 'red', 'black'] - - >>> # Deal 20 cards without replacement from a deck of 52 playing cards - >>> # and determine the proportion of cards with a ten-value - >>> # (a ten, jack, queen, or king). - >>> deck = collections.Counter(tens=16, low_cards=36) - >>> seen = sample(list(deck.elements()), k=20) - >>> seen.count('tens') / 20 - 0.15 - - >>> # Estimate the probability of getting 5 or more heads from 7 spins - >>> # of a biased coin that settles on heads 60% of the time. - >>> trial = lambda: choices('HT', cum_weights=(0.60, 1.00), k=7).count('H') >= 5 - >>> sum(trial() for i in range(10000)) / 10000 - 0.4169 - - >>> # Probability of the median of 5 samples being in middle two quartiles - >>> trial = lambda : 2500 <= sorted(choices(range(10000), k=5))[2] < 7500 - >>> sum(trial() for i in range(10000)) / 10000 - 0.7958 - -Example of `statistical bootstrapping -`_ using resampling -with replacement to estimate a confidence interval for the mean of a sample of -size five:: - - # http://statistics.about.com/od/Applications/a/Example-Of-Bootstrapping.htm - from statistics import mean - from random import choices - - data = 1, 2, 4, 4, 10 - means = sorted(mean(choices(data, k=5)) for i in range(20)) - print(f'The sample mean of {mean(data):.1f} has a 90% confidence ' - f'interval from {means[1]:.1f} to {means[-2]:.1f}') - -Example of a `resampling permutation test -`_ -to determine the statistical significance or `p-value -`_ of an observed difference -between the effects of a drug versus a placebo:: - - # Example from "Statistics is Easy" by Dennis Shasha and Manda Wilson - from statistics import mean - from random import shuffle - - drug = [54, 73, 53, 70, 73, 68, 52, 65, 65] - placebo = [54, 51, 58, 44, 55, 52, 42, 47, 58, 46] - observed_diff = mean(drug) - mean(placebo) - - n = 10000 - count = 0 - combined = drug + placebo - for i in range(n): - shuffle(combined) - new_diff = mean(combined[:len(drug)]) - mean(combined[len(drug):]) - count += (new_diff >= observed_diff) - - print(f'{n} label reshufflings produced only {count} instances with a difference') - print(f'at least as extreme as the observed difference of {observed_diff:.1f}.') - print(f'The one-sided p-value of {count / n:.4f} leads us to reject the null') - print(f'hypothesis that there is no difference between the drug and the placebo.') - -Simulation of arrival times and service deliveries in a single server queue:: - - from random import expovariate, gauss - from statistics import mean, median, stdev - - average_arrival_interval = 5.6 - average_service_time = 5.0 - stdev_service_time = 0.5 - - num_waiting = 0 - arrivals = [] - starts = [] - arrival = service_end = 0.0 - for i in range(20000): - if arrival <= service_end: - num_waiting += 1 - arrival += expovariate(1.0 / average_arrival_interval) - arrivals.append(arrival) - else: - num_waiting -= 1 - service_start = service_end if num_waiting else arrival - service_time = gauss(average_service_time, stdev_service_time) - service_end = service_start + service_time - starts.append(service_start) - - waits = [start - arrival for arrival, start in zip(arrivals, starts)] - print(f'Mean wait: {mean(waits):.1f}. Stdev wait: {stdev(waits):.1f}.') - print(f'Median wait: {median(waits):.1f}. Max wait: {max(waits):.1f}.') - -.. seealso:: - - `Statistics for Hackers `_ - a video tutorial by - `Jake Vanderplas `_ - on statistical analysis using just a few fundamental concepts - including simulation, sampling, shuffling, and cross-validation. - - `Economics Simulation - `_ - a simulation of a marketplace by - `Peter Norvig `_ that shows effective - use of many of the tools and distributions provided by this module - (gauss, uniform, sample, betavariate, choice, triangular, and randrange). - - `A Concrete Introduction to Probability (using Python) - `_ - a tutorial by `Peter Norvig `_ covering - the basics of probability theory, how to write simulations, and - how to perform data analysis using Python. diff --git a/third_party/python/Doc/library/re.rst b/third_party/python/Doc/library/re.rst deleted file mode 100644 index 89d36a9fe..000000000 --- a/third_party/python/Doc/library/re.rst +++ /dev/null @@ -1,1655 +0,0 @@ -:mod:`re` --- Regular expression operations -=========================================== - -.. module:: re - :synopsis: Regular expression operations. - -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Andrew M. Kuchling - -**Source code:** :source:`Lib/re.py` - --------------- - -This module provides regular expression matching operations similar to -those found in Perl. - -Both patterns and strings to be searched can be Unicode strings (:class:`str`) -as well as 8-bit strings (:class:`bytes`). -However, Unicode strings and 8-bit strings cannot be mixed: -that is, you cannot match a Unicode string with a byte pattern or -vice-versa; similarly, when asking for a substitution, the replacement -string must be of the same type as both the pattern and the search string. - -Regular expressions use the backslash character (``'\'``) to indicate -special forms or to allow special characters to be used without invoking -their special meaning. This collides with Python's usage of the same -character for the same purpose in string literals; for example, to match -a literal backslash, one might have to write ``'\\\\'`` as the pattern -string, because the regular expression must be ``\\``, and each -backslash must be expressed as ``\\`` inside a regular Python string -literal. - -The solution is to use Python's raw string notation for regular expression -patterns; backslashes are not handled in any special way in a string literal -prefixed with ``'r'``. So ``r"\n"`` is a two-character string containing -``'\'`` and ``'n'``, while ``"\n"`` is a one-character string containing a -newline. Usually patterns will be expressed in Python code using this raw -string notation. - -It is important to note that most regular expression operations are available as -module-level functions and methods on -:ref:`compiled regular expressions `. The functions are shortcuts -that don't require you to compile a regex object first, but miss some -fine-tuning parameters. - -.. seealso:: - - The third-party `regex `_ module, - which has an API compatible with the standard library :mod:`re` module, - but offers additional functionality and a more thorough Unicode support. - - -.. _re-syntax: - -Regular Expression Syntax -------------------------- - -A regular expression (or RE) specifies a set of strings that matches it; the -functions in this module let you check if a particular string matches a given -regular expression (or if a given regular expression matches a particular -string, which comes down to the same thing). - -Regular expressions can be concatenated to form new regular expressions; if *A* -and *B* are both regular expressions, then *AB* is also a regular expression. -In general, if a string *p* matches *A* and another string *q* matches *B*, the -string *pq* will match AB. This holds unless *A* or *B* contain low precedence -operations; boundary conditions between *A* and *B*; or have numbered group -references. Thus, complex expressions can easily be constructed from simpler -primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book [Frie09]_, -or almost any textbook about compiler construction. - -A brief explanation of the format of regular expressions follows. For further -information and a gentler presentation, consult the :ref:`regex-howto`. - -Regular expressions can contain both special and ordinary characters. Most -ordinary characters, like ``'A'``, ``'a'``, or ``'0'``, are the simplest regular -expressions; they simply match themselves. You can concatenate ordinary -characters, so ``last`` matches the string ``'last'``. (In the rest of this -section, we'll write RE's in ``this special style``, usually without quotes, and -strings to be matched ``'in single quotes'``.) - -Some characters, like ``'|'`` or ``'('``, are special. Special -characters either stand for classes of ordinary characters, or affect -how the regular expressions around them are interpreted. - -Repetition qualifiers (``*``, ``+``, ``?``, ``{m,n}``, etc) cannot be -directly nested. This avoids ambiguity with the non-greedy modifier suffix -``?``, and with other modifiers in other implementations. To apply a second -repetition to an inner repetition, parentheses may be used. For example, -the expression ``(?:a{6})*`` matches any multiple of six ``'a'`` characters. - - -The special characters are: - -.. index:: single: . (dot); in regular expressions - -``.`` - (Dot.) In the default mode, this matches any character except a newline. If - the :const:`DOTALL` flag has been specified, this matches any character - including a newline. - -.. index:: single: ^ (caret); in regular expressions - -``^`` - (Caret.) Matches the start of the string, and in :const:`MULTILINE` mode also - matches immediately after each newline. - -.. index:: single: $ (dollar); in regular expressions - -``$`` - Matches the end of the string or just before the newline at the end of the - string, and in :const:`MULTILINE` mode also matches before a newline. ``foo`` - matches both 'foo' and 'foobar', while the regular expression ``foo$`` matches - only 'foo'. More interestingly, searching for ``foo.$`` in ``'foo1\nfoo2\n'`` - matches 'foo2' normally, but 'foo1' in :const:`MULTILINE` mode; searching for - a single ``$`` in ``'foo\n'`` will find two (empty) matches: one just before - the newline, and one at the end of the string. - -.. index:: single: * (asterisk); in regular expressions - -``*`` - Causes the resulting RE to match 0 or more repetitions of the preceding RE, as - many repetitions as are possible. ``ab*`` will match 'a', 'ab', or 'a' followed - by any number of 'b's. - -.. index:: single: + (plus); in regular expressions - -``+`` - Causes the resulting RE to match 1 or more repetitions of the preceding RE. - ``ab+`` will match 'a' followed by any non-zero number of 'b's; it will not - match just 'a'. - -.. index:: single: ? (question mark); in regular expressions - -``?`` - Causes the resulting RE to match 0 or 1 repetitions of the preceding RE. - ``ab?`` will match either 'a' or 'ab'. - -.. index:: - single: *?; in regular expressions - single: +?; in regular expressions - single: ??; in regular expressions - -``*?``, ``+?``, ``??`` - The ``'*'``, ``'+'``, and ``'?'`` qualifiers are all :dfn:`greedy`; they match - as much text as possible. Sometimes this behaviour isn't desired; if the RE - ``<.*>`` is matched against ``' b '``, it will match the entire - string, and not just ``''``. Adding ``?`` after the qualifier makes it - perform the match in :dfn:`non-greedy` or :dfn:`minimal` fashion; as *few* - characters as possible will be matched. Using the RE ``<.*?>`` will match - only ``''``. - -.. index:: - single: {} (curly brackets); in regular expressions - -``{m}`` - Specifies that exactly *m* copies of the previous RE should be matched; fewer - matches cause the entire RE not to match. For example, ``a{6}`` will match - exactly six ``'a'`` characters, but not five. - -``{m,n}`` - Causes the resulting RE to match from *m* to *n* repetitions of the preceding - RE, attempting to match as many repetitions as possible. For example, - ``a{3,5}`` will match from 3 to 5 ``'a'`` characters. Omitting *m* specifies a - lower bound of zero, and omitting *n* specifies an infinite upper bound. As an - example, ``a{4,}b`` will match ``'aaaab'`` or a thousand ``'a'`` characters - followed by a ``'b'``, but not ``'aaab'``. The comma may not be omitted or the - modifier would be confused with the previously described form. - -``{m,n}?`` - Causes the resulting RE to match from *m* to *n* repetitions of the preceding - RE, attempting to match as *few* repetitions as possible. This is the - non-greedy version of the previous qualifier. For example, on the - 6-character string ``'aaaaaa'``, ``a{3,5}`` will match 5 ``'a'`` characters, - while ``a{3,5}?`` will only match 3 characters. - -.. index:: single: \ (backslash); in regular expressions - -``\`` - Either escapes special characters (permitting you to match characters like - ``'*'``, ``'?'``, and so forth), or signals a special sequence; special - sequences are discussed below. - - If you're not using a raw string to express the pattern, remember that Python - also uses the backslash as an escape sequence in string literals; if the escape - sequence isn't recognized by Python's parser, the backslash and subsequent - character are included in the resulting string. However, if Python would - recognize the resulting sequence, the backslash should be repeated twice. This - is complicated and hard to understand, so it's highly recommended that you use - raw strings for all but the simplest expressions. - -.. index:: - single: [] (square brackets); in regular expressions - -``[]`` - Used to indicate a set of characters. In a set: - - * Characters can be listed individually, e.g. ``[amk]`` will match ``'a'``, - ``'m'``, or ``'k'``. - - .. index:: single: - (minus); in regular expressions - - * Ranges of characters can be indicated by giving two characters and separating - them by a ``'-'``, for example ``[a-z]`` will match any lowercase ASCII letter, - ``[0-5][0-9]`` will match all the two-digits numbers from ``00`` to ``59``, and - ``[0-9A-Fa-f]`` will match any hexadecimal digit. If ``-`` is escaped (e.g. - ``[a\-z]``) or if it's placed as the first or last character - (e.g. ``[-a]`` or ``[a-]``), it will match a literal ``'-'``. - - * Special characters lose their special meaning inside sets. For example, - ``[(+*)]`` will match any of the literal characters ``'('``, ``'+'``, - ``'*'``, or ``')'``. - - .. index:: single: \ (backslash); in regular expressions - - * Character classes such as ``\w`` or ``\S`` (defined below) are also accepted - inside a set, although the characters they match depends on whether - :const:`ASCII` or :const:`LOCALE` mode is in force. - - .. index:: single: ^ (caret); in regular expressions - - * Characters that are not within a range can be matched by :dfn:`complementing` - the set. If the first character of the set is ``'^'``, all the characters - that are *not* in the set will be matched. For example, ``[^5]`` will match - any character except ``'5'``, and ``[^^]`` will match any character except - ``'^'``. ``^`` has no special meaning if it's not the first character in - the set. - - * To match a literal ``']'`` inside a set, precede it with a backslash, or - place it at the beginning of the set. For example, both ``[()[\]{}]`` and - ``[]()[{}]`` will both match a parenthesis. - -.. index:: single: | (vertical bar); in regular expressions - -``|`` - ``A|B``, where *A* and *B* can be arbitrary REs, creates a regular expression that - will match either *A* or *B*. An arbitrary number of REs can be separated by the - ``'|'`` in this way. This can be used inside groups (see below) as well. As - the target string is scanned, REs separated by ``'|'`` are tried from left to - right. When one pattern completely matches, that branch is accepted. This means - that once *A* matches, *B* will not be tested further, even if it would - produce a longer overall match. In other words, the ``'|'`` operator is never - greedy. To match a literal ``'|'``, use ``\|``, or enclose it inside a - character class, as in ``[|]``. - -.. index:: - single: () (parentheses); in regular expressions - -``(...)`` - Matches whatever regular expression is inside the parentheses, and indicates the - start and end of a group; the contents of a group can be retrieved after a match - has been performed, and can be matched later in the string with the ``\number`` - special sequence, described below. To match the literals ``'('`` or ``')'``, - use ``\(`` or ``\)``, or enclose them inside a character class: ``[(]``, ``[)]``. - -.. index:: single: (?; in regular expressions - -``(?...)`` - This is an extension notation (a ``'?'`` following a ``'('`` is not meaningful - otherwise). The first character after the ``'?'`` determines what the meaning - and further syntax of the construct is. Extensions usually do not create a new - group; ``(?P...)`` is the only exception to this rule. Following are the - currently supported extensions. - -``(?aiLmsux)`` - (One or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``.) The group matches the empty string; the - letters set the corresponding flags: :const:`re.A` (ASCII-only matching), - :const:`re.I` (ignore case), :const:`re.L` (locale dependent), - :const:`re.M` (multi-line), :const:`re.S` (dot matches all), - :const:`re.U` (Unicode matching), and :const:`re.X` (verbose), - for the entire regular expression. - (The flags are described in :ref:`contents-of-module-re`.) - This is useful if you wish to include the flags as part of the - regular expression, instead of passing a *flag* argument to the - :func:`re.compile` function. Flags should be used first in the - expression string. - -.. index:: single: (?:; in regular expressions - -``(?:...)`` - A non-capturing version of regular parentheses. Matches whatever regular - expression is inside the parentheses, but the substring matched by the group - *cannot* be retrieved after performing a match or referenced later in the - pattern. - -``(?imsx-imsx:...)`` - (Zero or more letters from the set ``'i'``, ``'m'``, ``'s'``, ``'x'``, - optionally followed by ``'-'`` followed by one or more letters from the - same set.) The letters set or removes the corresponding flags: - :const:`re.I` (ignore case), :const:`re.M` (multi-line), :const:`re.S` - (dot matches all), and :const:`re.X` (verbose), for the part of the - expression. (The flags are described in :ref:`contents-of-module-re`.) - - .. versionadded:: 3.6 - -.. index:: single: (?P<; in regular expressions - -``(?P...)`` - Similar to regular parentheses, but the substring matched by the group is - accessible via the symbolic group name *name*. Group names must be valid - Python identifiers, and each group name must be defined only once within a - regular expression. A symbolic group is also a numbered group, just as if - the group were not named. - - Named groups can be referenced in three contexts. If the pattern is - ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either - single or double quotes): - - +---------------------------------------+----------------------------------+ - | Context of reference to group "quote" | Ways to reference it | - +=======================================+==================================+ - | in the same pattern itself | * ``(?P=quote)`` (as shown) | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ - | when processing match object *m* | * ``m.group('quote')`` | - | | * ``m.end('quote')`` (etc.) | - +---------------------------------------+----------------------------------+ - | in a string passed to the *repl* | * ``\g`` | - | argument of ``re.sub()`` | * ``\g<1>`` | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ - -.. index:: single: (?P=; in regular expressions - -``(?P=name)`` - A backreference to a named group; it matches whatever text was matched by the - earlier group named *name*. - -.. index:: single: (?#; in regular expressions - -``(?#...)`` - A comment; the contents of the parentheses are simply ignored. - -``(?=...)`` - Matches if ``...`` matches next, but doesn't consume any of the string. This is - called a :dfn:`lookahead assertion`. For example, ``Isaac (?=Asimov)`` will match - ``'Isaac '`` only if it's followed by ``'Asimov'``. - -.. index:: single: (?!; in regular expressions - -``(?!...)`` - Matches if ``...`` doesn't match next. This is a :dfn:`negative lookahead assertion`. - For example, ``Isaac (?!Asimov)`` will match ``'Isaac '`` only if it's *not* - followed by ``'Asimov'``. - -.. index:: single: (?<=; in regular expressions - -``(?<=...)`` - Matches if the current position in the string is preceded by a match for ``...`` - that ends at the current position. This is called a :dfn:`positive lookbehind - assertion`. ``(?<=abc)def`` will find a match in ``'abcdef'``, since the - lookbehind will back up 3 characters and check if the contained pattern matches. - The contained pattern must only match strings of some fixed length, meaning that - ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Note that - patterns which start with positive lookbehind assertions will not match at the - beginning of the string being searched; you will most likely want to use the - :func:`search` function rather than the :func:`match` function: - - >>> import re - >>> m = re.search('(?<=abc)def', 'abcdef') - >>> m.group(0) - 'def' - - This example looks for a word following a hyphen: - - >>> m = re.search(r'(?<=-)\w+', 'spam-egg') - >>> m.group(0) - 'egg' - - .. versionchanged:: 3.5 - Added support for group references of fixed length. - -.. index:: single: (?|$)`` is a poor email matching pattern, which - will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. - - -The special sequences consist of ``'\'`` and a character from the list below. -If the ordinary character is not an ASCII digit or an ASCII letter, then the -resulting RE will match the second character. For example, ``\$`` matches the -character ``'$'``. - -.. index:: single: \ (backslash); in regular expressions - -``\number`` - Matches the contents of the group of the same number. Groups are numbered - starting from 1. For example, ``(.+) \1`` matches ``'the the'`` or ``'55 55'``, - but not ``'thethe'`` (note the space after the group). This special sequence - can only be used to match one of the first 99 groups. If the first digit of - *number* is 0, or *number* is 3 octal digits long, it will not be interpreted as - a group match, but as the character with octal value *number*. Inside the - ``'['`` and ``']'`` of a character class, all numeric escapes are treated as - characters. - -.. index:: single: \A; in regular expressions - -``\A`` - Matches only at the start of the string. - -.. index:: single: \b; in regular expressions - -``\b`` - Matches the empty string, but only at the beginning or end of a word. - A word is defined as a sequence of word characters. Note that formally, - ``\b`` is defined as the boundary between a ``\w`` and a ``\W`` character - (or vice versa), or between ``\w`` and the beginning/end of the string. - This means that ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, - ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - - By default Unicode alphanumerics are the ones used in Unicode patterns, but - this can be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - Inside a character range, ``\b`` represents the backspace character, for - compatibility with Python's string literals. - -.. index:: single: \B; in regular expressions - -``\B`` - Matches the empty string, but only when it is *not* at the beginning or end - of a word. This means that ``r'py\B'`` matches ``'python'``, ``'py3'``, - ``'py2'``, but not ``'py'``, ``'py.'``, or ``'py!'``. - ``\B`` is just the opposite of ``\b``, so word characters in Unicode - patterns are Unicode alphanumerics or the underscore, although this can - be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - -.. index:: single: \d; in regular expressions - -``\d`` - For Unicode (str) patterns: - Matches any Unicode decimal digit (that is, any character in - Unicode character category [Nd]). This includes ``[0-9]``, and - also many other digit characters. If the :const:`ASCII` flag is - used only ``[0-9]`` is matched (but the flag affects the entire - regular expression, so in such cases using an explicit ``[0-9]`` - may be a better choice). - - For 8-bit (bytes) patterns: - Matches any decimal digit; this is equivalent to ``[0-9]``. - -.. index:: single: \D; in regular expressions - -``\D`` - Matches any character which is not a decimal digit. This is - the opposite of ``\d``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^0-9]`` (but the flag affects the entire - regular expression, so in such cases using an explicit ``[^0-9]`` may - be a better choice). - -.. index:: single: \s; in regular expressions - -``\s`` - For Unicode (str) patterns: - Matches Unicode whitespace characters (which includes - ``[ \t\n\r\f\v]``, and also many other characters, for example the - non-breaking spaces mandated by typography rules in many - languages). If the :const:`ASCII` flag is used, only - ``[ \t\n\r\f\v]`` is matched (but the flag affects the entire - regular expression, so in such cases using an explicit - ``[ \t\n\r\f\v]`` may be a better choice). - - For 8-bit (bytes) patterns: - Matches characters considered whitespace in the ASCII character set; - this is equivalent to ``[ \t\n\r\f\v]``. - -.. index:: single: \S; in regular expressions - -``\S`` - Matches any character which is not a whitespace character. This is - the opposite of ``\s``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^ \t\n\r\f\v]`` (but the flag affects the entire - regular expression, so in such cases using an explicit ``[^ \t\n\r\f\v]`` may - be a better choice). - -.. index:: single: \w; in regular expressions - -``\w`` - For Unicode (str) patterns: - Matches Unicode word characters; this includes most characters - that can be part of a word in any language, as well as numbers and - the underscore. If the :const:`ASCII` flag is used, only - ``[a-zA-Z0-9_]`` is matched (but the flag affects the entire - regular expression, so in such cases using an explicit - ``[a-zA-Z0-9_]`` may be a better choice). - - For 8-bit (bytes) patterns: - Matches characters considered alphanumeric in the ASCII character set; - this is equivalent to ``[a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. - -.. index:: single: \W; in regular expressions - -``\W`` - Matches any character which is not a word character. This is - the opposite of ``\w``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^a-zA-Z0-9_]`` (but the flag affects the - entire regular expression, so in such cases using an explicit - ``[^a-zA-Z0-9_]`` may be a better choice). If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. - -.. index:: single: \Z; in regular expressions - -``\Z`` - Matches only at the end of the string. - -.. index:: - single: \a; in regular expressions - single: \b; in regular expressions - single: \f; in regular expressions - single: \n; in regular expressions - single: \N; in regular expressions - single: \r; in regular expressions - single: \t; in regular expressions - single: \u; in regular expressions - single: \U; in regular expressions - single: \v; in regular expressions - single: \x; in regular expressions - single: \\; in regular expressions - -Most of the standard escapes supported by Python string literals are also -accepted by the regular expression parser:: - - \a \b \f \n - \r \t \u \U - \v \x \\ - -(Note that ``\b`` is used to represent word boundaries, and means "backspace" -only inside character classes.) - -``'\u'`` and ``'\U'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are errors. - -Octal escapes are included in a limited form. If the first digit is a 0, or if -there are three octal digits, it is considered an octal escape. Otherwise, it is -a group reference. As for string literals, octal escapes are always at most -three digits in length. - -.. versionchanged:: 3.3 - The ``'\u'`` and ``'\U'`` escape sequences have been added. - -.. versionchanged:: 3.6 - Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. - - - -.. _contents-of-module-re: - -Module Contents ---------------- - -The module defines several functions, constants, and an exception. Some of the -functions are simplified versions of the full featured methods for compiled -regular expressions. Most non-trivial applications always use the compiled -form. - -.. versionchanged:: 3.6 - Flag constants are now instances of :class:`RegexFlag`, which is a subclass of - :class:`enum.IntFlag`. - -.. function:: compile(pattern, flags=0) - - Compile a regular expression pattern into a :ref:`regular expression object - `, which can be used for matching using its - :func:`~regex.match`, :func:`~regex.search` and other methods, described - below. - - The expression's behaviour can be modified by specifying a *flags* value. - Values can be any of the following variables, combined using bitwise OR (the - ``|`` operator). - - The sequence :: - - prog = re.compile(pattern) - result = prog.match(string) - - is equivalent to :: - - result = re.match(pattern, string) - - but using :func:`re.compile` and saving the resulting regular expression - object for reuse is more efficient when the expression will be used several - times in a single program. - - .. note:: - - The compiled versions of the most recent patterns passed to - :func:`re.compile` and the module-level matching functions are cached, so - programs that use only a few regular expressions at a time needn't worry - about compiling regular expressions. - - -.. data:: A - ASCII - - Make ``\w``, ``\W``, ``\b``, ``\B``, ``\d``, ``\D``, ``\s`` and ``\S`` - perform ASCII-only matching instead of full Unicode matching. This is only - meaningful for Unicode patterns, and is ignored for byte patterns. - Corresponds to the inline flag ``(?a)``. - - Note that for backward compatibility, the :const:`re.U` flag still - exists (as well as its synonym :const:`re.UNICODE` and its embedded - counterpart ``(?u)``), but these are redundant in Python 3 since - matches are Unicode by default for strings (and Unicode matching - isn't allowed for bytes). - - -.. data:: DEBUG - - Display debug information about compiled expression. - No corresponding inline flag. - - -.. data:: I - IGNORECASE - - Perform case-insensitive matching; expressions like ``[A-Z]`` will also - match lowercase letters. Full Unicode matching (such as ``Ü`` matching - ``ü``) also works unless the :const:`re.ASCII` flag is used to disable - non-ASCII matches. The current locale does not change the effect of this - flag unless the :const:`re.LOCALE` flag is also used. - Corresponds to the inline flag ``(?i)``. - - Note that when the Unicode patterns ``[a-z]`` or ``[A-Z]`` are used in - combination with the :const:`IGNORECASE` flag, they will match the 52 ASCII - letters and 4 additional non-ASCII letters: 'İ' (U+0130, Latin capital - letter I with dot above), 'ı' (U+0131, Latin small letter dotless i), - 'ſ' (U+017F, Latin small letter long s) and 'K' (U+212A, Kelvin sign). - If the :const:`ASCII` flag is used, only letters 'a' to 'z' - and 'A' to 'Z' are matched (but the flag affects the entire regular - expression, so in such cases using an explicit ``(?-i:[a-zA-Z])`` may be - a better choice). - -.. data:: L - LOCALE - - Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching - dependent on the current locale. This flag can be used only with bytes - patterns. The use of this flag is discouraged as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. - Corresponds to the inline flag ``(?L)``. - - .. versionchanged:: 3.6 - :const:`re.LOCALE` can be used only with bytes patterns and is - not compatible with :const:`re.ASCII`. - - -.. data:: M - MULTILINE - - When specified, the pattern character ``'^'`` matches at the beginning of the - string and at the beginning of each line (immediately following each newline); - and the pattern character ``'$'`` matches at the end of the string and at the - end of each line (immediately preceding each newline). By default, ``'^'`` - matches only at the beginning of the string, and ``'$'`` only at the end of the - string and immediately before the newline (if any) at the end of the string. - Corresponds to the inline flag ``(?m)``. - - -.. data:: S - DOTALL - - Make the ``'.'`` special character match any character at all, including a - newline; without this flag, ``'.'`` will match anything *except* a newline. - Corresponds to the inline flag ``(?s)``. - - -.. data:: X - VERBOSE - - .. index:: single: # (hash); in regular expressions - - This flag allows you to write regular expressions that look nicer and are - more readable by allowing you to visually separate logical sections of the - pattern and add comments. Whitespace within the pattern is ignored, except - when in a character class, or when preceded by an unescaped backslash, - or within tokens like ``*?``, ``(?:`` or ``(?P<...>``. - When a line contains a ``#`` that is not in a character class and is not - preceded by an unescaped backslash, all characters from the leftmost such - ``#`` through the end of the line are ignored. - - This means that the two following regular expression objects that match a - decimal number are functionally equal:: - - a = re.compile(r"""\d + # the integral part - \. # the decimal point - \d * # some fractional digits""", re.X) - b = re.compile(r"\d+\.\d*") - - Corresponds to the inline flag ``(?x)``. - - -.. function:: search(pattern, string, flags=0) - - Scan through *string* looking for the first location where the regular expression - *pattern* produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. - - -.. function:: match(pattern, string, flags=0) - - If zero or more characters at the beginning of *string* match the regular - expression *pattern*, return a corresponding :ref:`match object - `. Return ``None`` if the string does not match the pattern; - note that this is different from a zero-length match. - - Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match - at the beginning of the string and not at the beginning of each line. - - If you want to locate a match anywhere in *string*, use :func:`search` - instead (see also :ref:`search-vs-match`). - - -.. function:: fullmatch(pattern, string, flags=0) - - If the whole *string* matches the regular expression *pattern*, return a - corresponding :ref:`match object `. Return ``None`` if the - string does not match the pattern; note that this is different from a - zero-length match. - - .. versionadded:: 3.4 - - -.. function:: split(pattern, string, maxsplit=0, flags=0) - - Split *string* by the occurrences of *pattern*. If capturing parentheses are - used in *pattern*, then the text of all groups in the pattern are also returned - as part of the resulting list. If *maxsplit* is nonzero, at most *maxsplit* - splits occur, and the remainder of the string is returned as the final element - of the list. :: - - >>> re.split(r'\W+', 'Words, words, words.') - ['Words', 'words', 'words', ''] - >>> re.split(r'(\W+)', 'Words, words, words.') - ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'\W+', 'Words, words, words.', 1) - ['Words', 'words, words.'] - >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) - ['0', '3', '9'] - - If there are capturing groups in the separator and it matches at the start of - the string, the result will start with an empty string. The same holds for - the end of the string:: - - >>> re.split(r'(\W+)', '...words, words...') - ['', '...', 'words', ', ', 'words', '...', ''] - - That way, separator components are always found at the same relative - indices within the result list. - - .. note:: - - :func:`split` doesn't currently split a string on an empty pattern match. - For example:: - - >>> re.split('x*', 'axbc') - ['a', 'bc'] - - Even though ``'x*'`` also matches 0 'x' before 'a', between 'b' and 'c', - and after 'c', currently these matches are ignored. The correct behavior - (i.e. splitting on empty matches too and returning ``['', 'a', 'b', 'c', - '']``) will be implemented in future versions of Python, but since this - is a backward incompatible change, a :exc:`FutureWarning` will be raised - in the meanwhile. - - Patterns that can only match empty strings currently never split the - string. Since this doesn't match the expected behavior, a - :exc:`ValueError` will be raised starting from Python 3.5:: - - >>> re.split("^$", "foo\n\nbar\n", flags=re.M) - Traceback (most recent call last): - File "", line 1, in - ... - ValueError: split() requires a non-empty pattern match. - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Splitting on a pattern that could match an empty string now raises - a warning. Patterns that can only match empty strings are now rejected. - - -.. function:: findall(pattern, string, flags=0) - - Return all non-overlapping matches of *pattern* in *string*, as a list of - strings. The *string* is scanned left-to-right, and matches are returned in - the order found. If one or more groups are present in the pattern, return a - list of groups; this will be a list of tuples if the pattern has more than - one group. Empty matches are included in the result. - - .. note:: - - Due to the limitation of the current implementation the character - following an empty match is not included in a next match, so - ``findall(r'^|\w+', 'two words')`` returns ``['', 'wo', 'words']`` - (note missed "t"). This is changed in Python 3.7. - - -.. function:: finditer(pattern, string, flags=0) - - Return an :term:`iterator` yielding :ref:`match objects ` over - all non-overlapping matches for the RE *pattern* in *string*. The *string* - is scanned left-to-right, and matches are returned in the order found. Empty - matches are included in the result. See also the note about :func:`findall`. - - -.. function:: sub(pattern, repl, string, count=0, flags=0) - - Return the string obtained by replacing the leftmost non-overlapping occurrences - of *pattern* in *string* by the replacement *repl*. If the pattern isn't found, - *string* is returned unchanged. *repl* can be a string or a function; if it is - a string, any backslash escapes in it are processed. That is, ``\n`` is - converted to a single newline character, ``\r`` is converted to a carriage return, and - so forth. Unknown escapes such as ``\&`` are left alone. Backreferences, such - as ``\6``, are replaced with the substring matched by group 6 in the pattern. - For example:: - - >>> re.sub(r'def\s+([a-zA-Z_][a-zA-Z_0-9]*)\s*\(\s*\):', - ... r'static PyObject*\npy_\1(void)\n{', - ... 'def myfunc():') - 'static PyObject*\npy_myfunc(void)\n{' - - If *repl* is a function, it is called for every non-overlapping occurrence of - *pattern*. The function takes a single :ref:`match object ` - argument, and returns the replacement string. For example:: - - >>> def dashrepl(matchobj): - ... if matchobj.group(0) == '-': return ' ' - ... else: return '-' - >>> re.sub('-{1,2}', dashrepl, 'pro----gram-files') - 'pro--gram files' - >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) - 'Baked Beans & Spam' - - The pattern may be a string or a :ref:`pattern object `. - - The optional argument *count* is the maximum number of pattern occurrences to be - replaced; *count* must be a non-negative integer. If omitted or zero, all - occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous match, so ``sub('x*', '-', 'abc')`` returns - ``'-a-b-c-'``. - - .. index:: single: \g; in regular expressions - - In string-type *repl* arguments, in addition to the character escapes and - backreferences described above, - ``\g`` will use the substring matched by the group named ``name``, as - defined by the ``(?P...)`` syntax. ``\g`` uses the corresponding - group number; ``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous - in a replacement such as ``\g<2>0``. ``\20`` would be interpreted as a - reference to group 20, not a reference to group 2 followed by the literal - character ``'0'``. The backreference ``\g<0>`` substitutes in the entire - substring matched by the RE. - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - - .. versionchanged:: 3.6 - Unknown escapes in *pattern* consisting of ``'\'`` and an ASCII letter - now are errors. - - .. deprecated-removed:: 3.5 3.7 - Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now raise - a deprecation warning and will be forbidden in Python 3.7. - - -.. function:: subn(pattern, repl, string, count=0, flags=0) - - Perform the same operation as :func:`sub`, but return a tuple ``(new_string, - number_of_subs_made)``. - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - - -.. function:: escape(pattern) - - Escape all the characters in *pattern* except ASCII letters, numbers and ``'_'``. - This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. For example:: - - >>> print(re.escape('python.exe')) - python\.exe - - >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" - >>> print('[%s]+' % re.escape(legal_chars)) - [abcdefghijklmnopqrstuvwxyz0123456789\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\:]+ - - >>> operators = ['+', '-', '*', '/', '**'] - >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) - \/|\-|\+|\*\*|\* - - This functions must not be used for the replacement string in :func:`sub` - and :func:`subn`, only backslashes should be escaped. For example:: - - >>> digits_re = r'\d+' - >>> sample = '/usr/sbin/sendmail - 0 errors, 12 warnings' - >>> print(re.sub(digits_re, digits_re.replace('\\', r'\\'), sample)) - /usr/sbin/sendmail - \d+ errors, \d+ warnings - - .. versionchanged:: 3.3 - The ``'_'`` character is no longer escaped. - - -.. function:: purge() - - Clear the regular expression cache. - - -.. exception:: error(msg, pattern=None, pos=None) - - Exception raised when a string passed to one of the functions here is not a - valid regular expression (for example, it might contain unmatched parentheses) - or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has - the following additional attributes: - - .. attribute:: msg - - The unformatted error message. - - .. attribute:: pattern - - The regular expression pattern. - - .. attribute:: pos - - The index in *pattern* where compilation failed (may be ``None``). - - .. attribute:: lineno - - The line corresponding to *pos* (may be ``None``). - - .. attribute:: colno - - The column corresponding to *pos* (may be ``None``). - - .. versionchanged:: 3.5 - Added additional attributes. - -.. _re-objects: - -Regular Expression Objects --------------------------- - -Compiled regular expression objects support the following methods and -attributes: - -.. method:: regex.search(string[, pos[, endpos]]) - - Scan through *string* looking for the first location where this regular - expression produces a match, and return a corresponding :ref:`match object - `. Return ``None`` if no position in the string matches the - pattern; note that this is different from finding a zero-length match at some - point in the string. - - The optional second parameter *pos* gives an index in the string where the - search is to start; it defaults to ``0``. This is not completely equivalent to - slicing the string; the ``'^'`` pattern character matches at the real beginning - of the string and at positions just after a newline, but not necessarily at the - index where the search is to start. - - The optional parameter *endpos* limits how far the string will be searched; it - will be as if the string is *endpos* characters long, so only the characters - from *pos* to ``endpos - 1`` will be searched for a match. If *endpos* is less - than *pos*, no match will be found; otherwise, if *rx* is a compiled regular - expression object, ``rx.search(string, 0, 50)`` is equivalent to - ``rx.search(string[:50], 0)``. :: - - >>> pattern = re.compile("d") - >>> pattern.search("dog") # Match at index 0 - <_sre.SRE_Match object; span=(0, 1), match='d'> - >>> pattern.search("dog", 1) # No match; search doesn't include the "d" - - -.. method:: regex.match(string[, pos[, endpos]]) - - If zero or more characters at the *beginning* of *string* match this regular - expression, return a corresponding :ref:`match object `. - Return ``None`` if the string does not match the pattern; note that this is - different from a zero-length match. - - The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~regex.search` method. :: - - >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". - <_sre.SRE_Match object; span=(1, 2), match='o'> - - If you want to locate a match anywhere in *string*, use - :meth:`~regex.search` instead (see also :ref:`search-vs-match`). - - -.. method:: regex.fullmatch(string[, pos[, endpos]]) - - If the whole *string* matches this regular expression, return a corresponding - :ref:`match object `. Return ``None`` if the string does not - match the pattern; note that this is different from a zero-length match. - - The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~regex.search` method. :: - - >>> pattern = re.compile("o[gh]") - >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". - >>> pattern.fullmatch("ogre") # No match as not the full string matches. - >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. - <_sre.SRE_Match object; span=(1, 3), match='og'> - - .. versionadded:: 3.4 - - -.. method:: regex.split(string, maxsplit=0) - - Identical to the :func:`split` function, using the compiled pattern. - - -.. method:: regex.findall(string[, pos[, endpos]]) - - Similar to the :func:`findall` function, using the compiled pattern, but - also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`search`. - - -.. method:: regex.finditer(string[, pos[, endpos]]) - - Similar to the :func:`finditer` function, using the compiled pattern, but - also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`search`. - - -.. method:: regex.sub(repl, string, count=0) - - Identical to the :func:`sub` function, using the compiled pattern. - - -.. method:: regex.subn(repl, string, count=0) - - Identical to the :func:`subn` function, using the compiled pattern. - - -.. attribute:: regex.flags - - The regex matching flags. This is a combination of the flags given to - :func:`.compile`, any ``(?...)`` inline flags in the pattern, and implicit - flags such as :data:`UNICODE` if the pattern is a Unicode string. - - -.. attribute:: regex.groups - - The number of capturing groups in the pattern. - - -.. attribute:: regex.groupindex - - A dictionary mapping any symbolic group names defined by ``(?P)`` to group - numbers. The dictionary is empty if no symbolic groups were used in the - pattern. - - -.. attribute:: regex.pattern - - The pattern string from which the RE object was compiled. - - -.. _match-objects: - -Match Objects -------------- - -Match objects always have a boolean value of ``True``. -Since :meth:`~regex.match` and :meth:`~regex.search` return ``None`` -when there is no match, you can test whether there was a match with a simple -``if`` statement:: - - match = re.search(pattern, string) - if match: - process(match) - -Match objects support the following methods and attributes: - - -.. method:: match.expand(template) - - Return the string obtained by doing backslash substitution on the template - string *template*, as done by the :meth:`~regex.sub` method. - Escapes such as ``\n`` are converted to the appropriate characters, - and numeric backreferences (``\1``, ``\2``) and named backreferences - (``\g<1>``, ``\g``) are replaced by the contents of the - corresponding group. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - -.. method:: match.group([group1, ...]) - - Returns one or more subgroups of the match. If there is a single argument, the - result is a single string; if there are multiple arguments, the result is a - tuple with one item per argument. Without arguments, *group1* defaults to zero - (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a - part of the pattern that did not match, the corresponding result is ``None``. - If a group is contained in a part of the pattern that matched multiple times, - the last match is returned. :: - - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") - >>> m.group(0) # The entire match - 'Isaac Newton' - >>> m.group(1) # The first parenthesized subgroup. - 'Isaac' - >>> m.group(2) # The second parenthesized subgroup. - 'Newton' - >>> m.group(1, 2) # Multiple arguments give us a tuple. - ('Isaac', 'Newton') - - If the regular expression uses the ``(?P...)`` syntax, the *groupN* - arguments may also be strings identifying groups by their group name. If a - string argument is not used as a group name in the pattern, an :exc:`IndexError` - exception is raised. - - A moderately complicated example:: - - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.group('first_name') - 'Malcolm' - >>> m.group('last_name') - 'Reynolds' - - Named groups can also be referred to by their index:: - - >>> m.group(1) - 'Malcolm' - >>> m.group(2) - 'Reynolds' - - If a group matches multiple times, only the last match is accessible:: - - >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. - >>> m.group(1) # Returns only the last match. - 'c3' - - -.. method:: match.__getitem__(g) - - This is identical to ``m.group(g)``. This allows easier access to - an individual group from a match:: - - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") - >>> m[0] # The entire match - 'Isaac Newton' - >>> m[1] # The first parenthesized subgroup. - 'Isaac' - >>> m[2] # The second parenthesized subgroup. - 'Newton' - - .. versionadded:: 3.6 - - -.. method:: match.groups(default=None) - - Return a tuple containing all the subgroups of the match, from 1 up to however - many groups are in the pattern. The *default* argument is used for groups that - did not participate in the match; it defaults to ``None``. - - For example:: - - >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") - >>> m.groups() - ('24', '1632') - - If we make the decimal place and everything after it optional, not all groups - might participate in the match. These groups will default to ``None`` unless - the *default* argument is given:: - - >>> m = re.match(r"(\d+)\.?(\d+)?", "24") - >>> m.groups() # Second group defaults to None. - ('24', None) - >>> m.groups('0') # Now, the second group defaults to '0'. - ('24', '0') - - -.. method:: match.groupdict(default=None) - - Return a dictionary containing all the *named* subgroups of the match, keyed by - the subgroup name. The *default* argument is used for groups that did not - participate in the match; it defaults to ``None``. For example:: - - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.groupdict() - {'first_name': 'Malcolm', 'last_name': 'Reynolds'} - - -.. method:: match.start([group]) - match.end([group]) - - Return the indices of the start and end of the substring matched by *group*; - *group* defaults to zero (meaning the whole matched substring). Return ``-1`` if - *group* exists but did not contribute to the match. For a match object *m*, and - a group *g* that did contribute to the match, the substring matched by group *g* - (equivalent to ``m.group(g)``) is :: - - m.string[m.start(g):m.end(g)] - - Note that ``m.start(group)`` will equal ``m.end(group)`` if *group* matched a - null string. For example, after ``m = re.search('b(c?)', 'cba')``, - ``m.start(0)`` is 1, ``m.end(0)`` is 2, ``m.start(1)`` and ``m.end(1)`` are both - 2, and ``m.start(2)`` raises an :exc:`IndexError` exception. - - An example that will remove *remove_this* from email addresses:: - - >>> email = "tony@tiremove_thisger.net" - >>> m = re.search("remove_this", email) - >>> email[:m.start()] + email[m.end():] - 'tony@tiger.net' - - -.. method:: match.span([group]) - - For a match *m*, return the 2-tuple ``(m.start(group), m.end(group))``. Note - that if *group* did not contribute to the match, this is ``(-1, -1)``. - *group* defaults to zero, the entire match. - - -.. attribute:: match.pos - - The value of *pos* which was passed to the :meth:`~regex.search` or - :meth:`~regex.match` method of a :ref:`regex object `. This is - the index into the string at which the RE engine started looking for a match. - - -.. attribute:: match.endpos - - The value of *endpos* which was passed to the :meth:`~regex.search` or - :meth:`~regex.match` method of a :ref:`regex object `. This is - the index into the string beyond which the RE engine will not go. - - -.. attribute:: match.lastindex - - The integer index of the last matched capturing group, or ``None`` if no group - was matched at all. For example, the expressions ``(a)b``, ``((a)(b))``, and - ``((ab))`` will have ``lastindex == 1`` if applied to the string ``'ab'``, while - the expression ``(a)(b)`` will have ``lastindex == 2``, if applied to the same - string. - - -.. attribute:: match.lastgroup - - The name of the last matched capturing group, or ``None`` if the group didn't - have a name, or if no group was matched at all. - - -.. attribute:: match.re - - The :ref:`regular expression object ` whose :meth:`~regex.match` or - :meth:`~regex.search` method produced this match instance. - - -.. attribute:: match.string - - The string passed to :meth:`~regex.match` or :meth:`~regex.search`. - - -.. _re-examples: - -Regular Expression Examples ---------------------------- - - -Checking for a Pair -^^^^^^^^^^^^^^^^^^^ - -In this example, we'll use the following helper function to display match -objects a little more gracefully: - -.. testcode:: - - def displaymatch(match): - if match is None: - return None - return '' % (match.group(), match.groups()) - -Suppose you are writing a poker program where a player's hand is represented as -a 5-character string with each character representing a card, "a" for ace, "k" -for king, "q" for queen, "j" for jack, "t" for 10, and "2" through "9" -representing the card with that value. - -To see if a given string is a valid hand, one could do the following:: - - >>> valid = re.compile(r"^[a2-9tjqk]{5}$") - >>> displaymatch(valid.match("akt5q")) # Valid. - "" - >>> displaymatch(valid.match("akt5e")) # Invalid. - >>> displaymatch(valid.match("akt")) # Invalid. - >>> displaymatch(valid.match("727ak")) # Valid. - "" - -That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. -To match this with a regular expression, one could use backreferences as such:: - - >>> pair = re.compile(r".*(.).*\1") - >>> displaymatch(pair.match("717ak")) # Pair of 7s. - "" - >>> displaymatch(pair.match("718ak")) # No pairs. - >>> displaymatch(pair.match("354aa")) # Pair of aces. - "" - -To find out what card the pair consists of, one could use the -:meth:`~match.group` method of the match object in the following manner: - -.. doctest:: - - >>> pair.match("717ak").group(1) - '7' - - # Error because re.match() returns None, which doesn't have a group() method: - >>> pair.match("718ak").group(1) - Traceback (most recent call last): - File "", line 1, in - re.match(r".*(.).*\1", "718ak").group(1) - AttributeError: 'NoneType' object has no attribute 'group' - - >>> pair.match("354aa").group(1) - 'a' - - -Simulating scanf() -^^^^^^^^^^^^^^^^^^ - -.. index:: single: scanf() - -Python does not currently have an equivalent to :c:func:`scanf`. Regular -expressions are generally more powerful, though also more verbose, than -:c:func:`scanf` format strings. The table below offers some more-or-less -equivalent mappings between :c:func:`scanf` format tokens and regular -expressions. - -+--------------------------------+---------------------------------------------+ -| :c:func:`scanf` Token | Regular Expression | -+================================+=============================================+ -| ``%c`` | ``.`` | -+--------------------------------+---------------------------------------------+ -| ``%5c`` | ``.{5}`` | -+--------------------------------+---------------------------------------------+ -| ``%d`` | ``[-+]?\d+`` | -+--------------------------------+---------------------------------------------+ -| ``%e``, ``%E``, ``%f``, ``%g`` | ``[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?`` | -+--------------------------------+---------------------------------------------+ -| ``%i`` | ``[-+]?(0[xX][\dA-Fa-f]+|0[0-7]*|\d+)`` | -+--------------------------------+---------------------------------------------+ -| ``%o`` | ``[-+]?[0-7]+`` | -+--------------------------------+---------------------------------------------+ -| ``%s`` | ``\S+`` | -+--------------------------------+---------------------------------------------+ -| ``%u`` | ``\d+`` | -+--------------------------------+---------------------------------------------+ -| ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | -+--------------------------------+---------------------------------------------+ - -To extract the filename and numbers from a string like :: - - /usr/sbin/sendmail - 0 errors, 4 warnings - -you would use a :c:func:`scanf` format like :: - - %s - %d errors, %d warnings - -The equivalent regular expression would be :: - - (\S+) - (\d+) errors, (\d+) warnings - - -.. _search-vs-match: - -search() vs. match() -^^^^^^^^^^^^^^^^^^^^ - -.. sectionauthor:: Fred L. Drake, Jr. - -Python offers two different primitive operations based on regular expressions: -:func:`re.match` checks for a match only at the beginning of the string, while -:func:`re.search` checks for a match anywhere in the string (this is what Perl -does by default). - -For example:: - - >>> re.match("c", "abcdef") # No match - >>> re.search("c", "abcdef") # Match - <_sre.SRE_Match object; span=(2, 3), match='c'> - -Regular expressions beginning with ``'^'`` can be used with :func:`search` to -restrict the match at the beginning of the string:: - - >>> re.match("c", "abcdef") # No match - >>> re.search("^c", "abcdef") # No match - >>> re.search("^a", "abcdef") # Match - <_sre.SRE_Match object; span=(0, 1), match='a'> - -Note however that in :const:`MULTILINE` mode :func:`match` only matches at the -beginning of the string, whereas using :func:`search` with a regular expression -beginning with ``'^'`` will match at the beginning of each line. :: - - >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match - >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match - <_sre.SRE_Match object; span=(4, 5), match='X'> - - -Making a Phonebook -^^^^^^^^^^^^^^^^^^ - -:func:`split` splits a string into a list delimited by the passed pattern. The -method is invaluable for converting textual data into data structures that can be -easily read and modified by Python as demonstrated in the following example that -creates a phonebook. - -First, here is the input. Normally it may come from a file, here we are using -triple-quoted string syntax:: - - >>> text = """Ross McFluff: 834.345.1254 155 Elm Street - ... - ... Ronald Heathmore: 892.345.3428 436 Finley Avenue - ... Frank Burger: 925.541.7625 662 South Dogwood Way - ... - ... - ... Heather Albrecht: 548.326.4584 919 Park Place""" - -The entries are separated by one or more newlines. Now we convert the string -into a list with each nonempty line having its own entry: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> entries = re.split("\n+", text) - >>> entries - ['Ross McFluff: 834.345.1254 155 Elm Street', - 'Ronald Heathmore: 892.345.3428 436 Finley Avenue', - 'Frank Burger: 925.541.7625 662 South Dogwood Way', - 'Heather Albrecht: 548.326.4584 919 Park Place'] - -Finally, split each entry into a list with first name, last name, telephone -number, and address. We use the ``maxsplit`` parameter of :func:`split` -because the address has spaces, our splitting pattern, in it: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> [re.split(":? ", entry, 3) for entry in entries] - [['Ross', 'McFluff', '834.345.1254', '155 Elm Street'], - ['Ronald', 'Heathmore', '892.345.3428', '436 Finley Avenue'], - ['Frank', 'Burger', '925.541.7625', '662 South Dogwood Way'], - ['Heather', 'Albrecht', '548.326.4584', '919 Park Place']] - -The ``:?`` pattern matches the colon after the last name, so that it does not -occur in the result list. With a ``maxsplit`` of ``4``, we could separate the -house number from the street name: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> [re.split(":? ", entry, 4) for entry in entries] - [['Ross', 'McFluff', '834.345.1254', '155', 'Elm Street'], - ['Ronald', 'Heathmore', '892.345.3428', '436', 'Finley Avenue'], - ['Frank', 'Burger', '925.541.7625', '662', 'South Dogwood Way'], - ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] - - -Text Munging -^^^^^^^^^^^^ - -:func:`sub` replaces every occurrence of a pattern with a string or the -result of a function. This example demonstrates using :func:`sub` with -a function to "munge" text, or randomize the order of all the characters -in each word of a sentence except for the first and last characters:: - - >>> def repl(m): - ... inner_word = list(m.group(2)) - ... random.shuffle(inner_word) - ... return m.group(1) + "".join(inner_word) + m.group(3) - >>> text = "Professor Abdolmalek, please report your absences promptly." - >>> re.sub(r"(\w)(\w+)(\w)", repl, text) - 'Poefsrosr Aealmlobdk, pslaee reorpt your abnseces plmrptoy.' - >>> re.sub(r"(\w)(\w+)(\w)", repl, text) - 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' - - -Finding all Adverbs -^^^^^^^^^^^^^^^^^^^ - -:func:`findall` matches *all* occurrences of a pattern, not just the first -one as :func:`search` does. For example, if a writer wanted to -find all of the adverbs in some text, they might use :func:`findall` in -the following manner:: - - >>> text = "He was carefully disguised but captured quickly by police." - >>> re.findall(r"\w+ly", text) - ['carefully', 'quickly'] - - -Finding all Adverbs and their Positions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If one wants more information about all matches of a pattern than the matched -text, :func:`finditer` is useful as it provides :ref:`match objects -` instead of strings. Continuing with the previous example, if -a writer wanted to find all of the adverbs *and their positions* in -some text, they would use :func:`finditer` in the following manner:: - - >>> text = "He was carefully disguised but captured quickly by police." - >>> for m in re.finditer(r"\w+ly", text): - ... print('%02d-%02d: %s' % (m.start(), m.end(), m.group(0))) - 07-16: carefully - 40-47: quickly - - -Raw String Notation -^^^^^^^^^^^^^^^^^^^ - -Raw string notation (``r"text"``) keeps regular expressions sane. Without it, -every backslash (``'\'``) in a regular expression would have to be prefixed with -another one to escape it. For example, the two following lines of code are -functionally identical:: - - >>> re.match(r"\W(.)\1\W", " ff ") - <_sre.SRE_Match object; span=(0, 4), match=' ff '> - >>> re.match("\\W(.)\\1\\W", " ff ") - <_sre.SRE_Match object; span=(0, 4), match=' ff '> - -When one wants to match a literal backslash, it must be escaped in the regular -expression. With raw string notation, this means ``r"\\"``. Without raw string -notation, one must use ``"\\\\"``, making the following lines of code -functionally identical:: - - >>> re.match(r"\\", r"\\") - <_sre.SRE_Match object; span=(0, 1), match='\\'> - >>> re.match("\\\\", r"\\") - <_sre.SRE_Match object; span=(0, 1), match='\\'> - - -Writing a Tokenizer -^^^^^^^^^^^^^^^^^^^ - -A `tokenizer or scanner `_ -analyzes a string to categorize groups of characters. This is a useful first -step in writing a compiler or interpreter. - -The text categories are specified with regular expressions. The technique is -to combine those into a single master regular expression and to loop over -successive matches:: - - import collections - import re - - Token = collections.namedtuple('Token', ['type', 'value', 'line', 'column']) - - def tokenize(code): - keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} - token_specification = [ - ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number - ('ASSIGN', r':='), # Assignment operator - ('END', r';'), # Statement terminator - ('ID', r'[A-Za-z]+'), # Identifiers - ('OP', r'[+\-*/]'), # Arithmetic operators - ('NEWLINE', r'\n'), # Line endings - ('SKIP', r'[ \t]+'), # Skip over spaces and tabs - ('MISMATCH', r'.'), # Any other character - ] - tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification) - line_num = 1 - line_start = 0 - for mo in re.finditer(tok_regex, code): - kind = mo.lastgroup - value = mo.group() - column = mo.start() - line_start - if kind == 'NUMBER': - value = float(value) if '.' in value else int(value) - elif kind == 'ID' and value in keywords: - kind = value - elif kind == 'NEWLINE': - line_start = mo.end() - line_num += 1 - continue - elif kind == 'SKIP': - continue - elif kind == 'MISMATCH': - raise RuntimeError(f'{value!r} unexpected on line {line_num}') - yield Token(kind, value, line_num, column) - - statements = ''' - IF quantity THEN - total := total + price * quantity; - tax := price * 0.05; - ENDIF; - ''' - - for token in tokenize(statements): - print(token) - -The tokenizer produces the following output:: - - Token(type='IF', value='IF', line=2, column=4) - Token(type='ID', value='quantity', line=2, column=7) - Token(type='THEN', value='THEN', line=2, column=16) - Token(type='ID', value='total', line=3, column=8) - Token(type='ASSIGN', value=':=', line=3, column=14) - Token(type='ID', value='total', line=3, column=17) - Token(type='OP', value='+', line=3, column=23) - Token(type='ID', value='price', line=3, column=25) - Token(type='OP', value='*', line=3, column=31) - Token(type='ID', value='quantity', line=3, column=33) - Token(type='END', value=';', line=3, column=41) - Token(type='ID', value='tax', line=4, column=8) - Token(type='ASSIGN', value=':=', line=4, column=12) - Token(type='ID', value='price', line=4, column=15) - Token(type='OP', value='*', line=4, column=21) - Token(type='NUMBER', value=0.05, line=4, column=23) - Token(type='END', value=';', line=4, column=27) - Token(type='ENDIF', value='ENDIF', line=5, column=4) - Token(type='END', value=';', line=5, column=9) - - -.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly - Media, 2009. The third edition of the book no longer covers Python at all, - but the first edition covered writing good regular expression patterns in - great detail. diff --git a/third_party/python/Doc/library/readline.rst b/third_party/python/Doc/library/readline.rst deleted file mode 100644 index 837816e95..000000000 --- a/third_party/python/Doc/library/readline.rst +++ /dev/null @@ -1,351 +0,0 @@ -:mod:`readline` --- GNU readline interface -========================================== - -.. module:: readline - :platform: Unix - :synopsis: GNU readline support for Python. - -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`readline` module defines a number of functions to facilitate -completion and reading/writing of history files from the Python interpreter. -This module can be used directly, or via the :mod:`rlcompleter` module, which -supports completion of Python identifiers at the interactive prompt. Settings -made using this module affect the behaviour of both the interpreter's -interactive prompt and the prompts offered by the built-in :func:`input` -function. - -.. note:: - - The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. - On MacOS X the :mod:`readline` module detects which library is being used - at run time. - - The configuration file for ``libedit`` is different from that - of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. - -Readline keybindings may be configured via an initialization file, typically -``.inputrc`` in your home directory. See `Readline Init File -`_ -in the GNU Readline manual for information about the format and -allowable constructs of that file, and the capabilities of the -Readline library in general. - - -Init file ---------- - -The following functions relate to the init file and user configuration: - - -.. function:: parse_and_bind(string) - - Execute the init line provided in the *string* argument. This calls - :c:func:`rl_parse_and_bind` in the underlying library. - - -.. function:: read_init_file([filename]) - - Execute a readline initialization file. The default filename is the last filename - used. This calls :c:func:`rl_read_init_file` in the underlying library. - - -Line buffer ------------ - -The following functions operate on the line buffer: - - -.. function:: get_line_buffer() - - Return the current contents of the line buffer (:c:data:`rl_line_buffer` - in the underlying library). - - -.. function:: insert_text(string) - - Insert text into the line buffer at the cursor position. This calls - :c:func:`rl_insert_text` in the underlying library, but ignores - the return value. - - -.. function:: redisplay() - - Change what's displayed on the screen to reflect the current contents of the - line buffer. This calls :c:func:`rl_redisplay` in the underlying library. - - -History file ------------- - -The following functions operate on a history file: - - -.. function:: read_history_file([filename]) - - Load a readline history file, and append it to the history list. - The default filename is :file:`~/.history`. This calls - :c:func:`read_history` in the underlying library. - - -.. function:: write_history_file([filename]) - - Save the history list to a readline history file, overwriting any - existing file. The default filename is :file:`~/.history`. This calls - :c:func:`write_history` in the underlying library. - - -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* items of history to a file. The default filename is - :file:`~/.history`. The file must already exist. This calls - :c:func:`append_history` in the underlying library. This function - only exists if Python was compiled for a version of the library - that supports it. - - .. versionadded:: 3.5 - - -.. function:: get_history_length() - set_history_length(length) - - Set or return the desired number of lines to save in the history file. - The :func:`write_history_file` function uses this value to truncate - the history file, by calling :c:func:`history_truncate_file` in - the underlying library. Negative values imply - unlimited history file size. - - -History list ------------- - -The following functions operate on a global history list: - - -.. function:: clear_history() - - Clear the current history. This calls :c:func:`clear_history` in the - underlying library. The Python function only exists if Python was - compiled for a version of the library that supports it. - - -.. function:: get_current_history_length() - - Return the number of items currently in the history. (This is different from - :func:`get_history_length`, which returns the maximum number of lines that will - be written to a history file.) - - -.. function:: get_history_item(index) - - Return the current contents of history item at *index*. The item index - is one-based. This calls :c:func:`history_get` in the underlying library. - - -.. function:: remove_history_item(pos) - - Remove history item specified by its position from the history. - The position is zero-based. This calls :c:func:`remove_history` in - the underlying library. - - -.. function:: replace_history_item(pos, line) - - Replace history item specified by its position with *line*. - The position is zero-based. This calls :c:func:`replace_history_entry` - in the underlying library. - - -.. function:: add_history(line) - - Append *line* to the history buffer, as if it was the last line typed. - This calls :c:func:`add_history` in the underlying library. - - -.. function:: set_auto_history(enabled) - - Enable or disable automatic calls to :c:func:`add_history` when reading - input via readline. The *enabled* argument should be a Boolean value - that when true, enables auto history, and that when false, disables - auto history. - - .. versionadded:: 3.6 - - .. impl-detail:: - Auto history is enabled by default, and changes to this do not persist - across multiple sessions. - - -Startup hooks -------------- - - -.. function:: set_startup_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_startup_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any function - already installed is removed. The hook is called with no - arguments just before readline prints the first prompt. - - -.. function:: set_pre_input_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_pre_input_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any - function already installed is removed. The hook is called - with no arguments after the first prompt has been printed and just before - readline starts reading input characters. This function only exists - if Python was compiled for a version of the library that supports it. - - -Completion ----------- - -The following functions relate to implementing a custom word completion -function. This is typically operated by the Tab key, and can suggest and -automatically complete a word being typed. By default, Readline is set up -to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used -with a custom completer, a different set of word delimiters should be set. - - -.. function:: set_completer([function]) - - Set or remove the completer function. If *function* is specified, it will be - used as the new completer function; if omitted or ``None``, any completer - function already installed is removed. The completer function is called as - ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it - returns a non-string value. It should return the next possible completion - starting with *text*. - - The installed completer function is invoked by the *entry_func* callback - passed to :c:func:`rl_completion_matches` in the underlying library. - The *text* string comes from the first parameter to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: get_completer() - - Get the completer function, or ``None`` if no completer function has been set. - - -.. function:: get_completion_type() - - Get the type of completion being attempted. This returns the - :c:data:`rl_completion_type` variable in the underlying library as - an integer. - - -.. function:: get_begidx() - get_endidx() - - Get the beginning or ending index of the completion scope. - These indexes are the *start* and *end* arguments passed to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: set_completer_delims(string) - get_completer_delims() - - Set or get the word delimiters for completion. These determine the - start of the word to be considered for completion (the completion scope). - These functions access the :c:data:`rl_completer_word_break_characters` - variable in the underlying library. - - -.. function:: set_completion_display_matches_hook([function]) - - Set or remove the completion display function. If *function* is - specified, it will be used as the new completion display function; - if omitted or ``None``, any completion display function already - installed is removed. This sets or clears the - :c:data:`rl_completion_display_matches_hook` callback in the - underlying library. The completion display function is called as - ``function(substitution, [matches], longest_match_length)`` once - each time matches need to be displayed. - - -.. _readline-example: - -Example -------- - -The following example demonstrates how to use the :mod:`readline` module's -history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: - - import atexit - import os - import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - try: - readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) - except FileNotFoundError: - pass - - atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_current_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_current_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) - -The following example extends the :class:`code.InteractiveConsole` class to -support history save/restore. :: - - import atexit - import code - import os - import readline - - class HistoryConsole(code.InteractiveConsole): - def __init__(self, locals=None, filename="", - histfile=os.path.expanduser("~/.console-history")): - code.InteractiveConsole.__init__(self, locals, filename) - self.init_history(histfile) - - def init_history(self, histfile): - readline.parse_and_bind("tab: complete") - if hasattr(readline, "read_history_file"): - try: - readline.read_history_file(histfile) - except FileNotFoundError: - pass - atexit.register(self.save_history, histfile) - - def save_history(self, histfile): - readline.set_history_length(1000) - readline.write_history_file(histfile) diff --git a/third_party/python/Doc/library/reprlib.rst b/third_party/python/Doc/library/reprlib.rst deleted file mode 100644 index dbe2a7182..000000000 --- a/third_party/python/Doc/library/reprlib.rst +++ /dev/null @@ -1,162 +0,0 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== - -.. module:: reprlib - :synopsis: Alternate repr() implementation with size limits. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/reprlib.py` - --------------- - -The :mod:`reprlib` module provides a means for producing object representations -with limits on the size of the resulting strings. This is used in the Python -debugger and may be useful in other contexts as well. - -This module provides a class, an instance, and a function: - - -.. class:: Repr() - - Class which provides formatting services useful in implementing functions - similar to the built-in :func:`repr`; size limits for different object types - are added to avoid the generation of representations which are excessively long. - - -.. data:: aRepr - - This is an instance of :class:`Repr` which is used to provide the - :func:`.repr` function described below. Changing the attributes of this - object will affect the size limits used by :func:`.repr` and the Python - debugger. - - -.. function:: repr(obj) - - This is the :meth:`~Repr.repr` method of ``aRepr``. It returns a string - similar to that returned by the built-in function of the same name, but with - limits on most sizes. - -In addition to size-limiting tools, the module also provides a decorator for -detecting recursive calls to :meth:`__repr__` and substituting a placeholder -string instead. - - -.. index:: single: ...; placeholder - -.. decorator:: recursive_repr(fillvalue="...") - - Decorator for :meth:`__repr__` methods to detect recursive calls within the - same thread. If a recursive call is made, the *fillvalue* is returned, - otherwise, the usual :meth:`__repr__` call is made. For example: - - >>> class MyList(list): - ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' - ... - >>> m = MyList('abc') - >>> m.append(m) - >>> m.append('x') - >>> print(m) - <'a'|'b'|'c'|...|'x'> - - .. versionadded:: 3.2 - - -.. _repr-objects: - -Repr Objects ------------- - -:class:`Repr` instances provide several attributes which can be used to provide -size limits for the representations of different object types, and methods -which format specific object types. - - -.. attribute:: Repr.maxlevel - - Depth limit on the creation of recursive representations. The default is ``6``. - - -.. attribute:: Repr.maxdict - Repr.maxlist - Repr.maxtuple - Repr.maxset - Repr.maxfrozenset - Repr.maxdeque - Repr.maxarray - - Limits on the number of entries represented for the named object type. The - default is ``4`` for :attr:`maxdict`, ``5`` for :attr:`maxarray`, and ``6`` for - the others. - - -.. attribute:: Repr.maxlong - - Maximum number of characters in the representation for an integer. Digits - are dropped from the middle. The default is ``40``. - - -.. attribute:: Repr.maxstring - - Limit on the number of characters in the representation of the string. Note - that the "normal" representation of the string is used as the character source: - if escape sequences are needed in the representation, these may be mangled when - the representation is shortened. The default is ``30``. - - -.. attribute:: Repr.maxother - - This limit is used to control the size of object types for which no specific - formatting method is available on the :class:`Repr` object. It is applied in a - similar manner as :attr:`maxstring`. The default is ``20``. - - -.. method:: Repr.repr(obj) - - The equivalent to the built-in :func:`repr` that uses the formatting imposed by - the instance. - - -.. method:: Repr.repr1(obj, level) - - Recursive implementation used by :meth:`.repr`. This uses the type of *obj* to - determine which formatting method to call, passing it *obj* and *level*. The - type-specific methods should call :meth:`repr1` to perform recursive formatting, - with ``level - 1`` for the value of *level* in the recursive call. - - -.. method:: Repr.repr_TYPE(obj, level) - :noindex: - - Formatting methods for specific types are implemented as methods with a name - based on the type name. In the method name, **TYPE** is replaced by - ``'_'.join(type(obj).__name__.split())``. Dispatch to these methods is - handled by :meth:`repr1`. Type-specific methods which need to recursively - format a value should call ``self.repr1(subobj, level - 1)``. - - -.. _subclassing-reprs: - -Subclassing Repr Objects ------------------------- - -The use of dynamic dispatching by :meth:`Repr.repr1` allows subclasses of -:class:`Repr` to add support for additional built-in object types or to modify -the handling of types already supported. This example shows how special support -for file objects could be added:: - - import reprlib - import sys - - class MyRepr(reprlib.Repr): - - def repr_TextIOWrapper(self, obj, level): - if obj.name in {'', '', ''}: - return obj.name - return repr(obj) - - aRepr = MyRepr() - print(aRepr.repr(sys.stdin)) # prints '' diff --git a/third_party/python/Doc/library/resource.rst b/third_party/python/Doc/library/resource.rst deleted file mode 100644 index bdfe5f6d4..000000000 --- a/third_party/python/Doc/library/resource.rst +++ /dev/null @@ -1,350 +0,0 @@ -:mod:`resource` --- Resource usage information -============================================== - -.. module:: resource - :platform: Unix - :synopsis: An interface to provide resource usage information on the current process. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Jeremy Hylton - --------------- - -This module provides basic mechanisms for measuring and controlling system -resources utilized by a program. - -Symbolic constants are used to specify particular system resources and to -request usage information about either the current process or its children. - -An :exc:`OSError` is raised on syscall failure. - - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -Resource Limits ---------------- - -Resources usage can be limited using the :func:`setrlimit` function described -below. Each resource is controlled by a pair of limits: a soft limit and a hard -limit. The soft limit is the current limit, and may be lowered or raised by a -process over time. The soft limit can never exceed the hard limit. The hard -limit can be lowered to any value greater than the soft limit, but not raised. -(Only processes with the effective UID of the super-user can raise a hard -limit.) - -The specific resources that can be limited are system dependent. They are -described in the :manpage:`getrlimit(2)` man page. The resources listed below -are supported when the underlying operating system supports them; resources -which cannot be checked or controlled by the operating system are not defined in -this module for those platforms. - - -.. data:: RLIM_INFINITY - - Constant used to represent the limit for an unlimited resource. - - -.. function:: getrlimit(resource) - - Returns a tuple ``(soft, hard)`` with the current soft and hard limits of - *resource*. Raises :exc:`ValueError` if an invalid resource is specified, or - :exc:`error` if the underlying system call fails unexpectedly. - - -.. function:: setrlimit(resource, limits) - - Sets new limits of consumption of *resource*. The *limits* argument must be a - tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is - unlimited. - - Raises :exc:`ValueError` if an invalid resource is specified, if the new soft - limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or - system limit for that resource is not unlimited will result in a - :exc:`ValueError`. A process with the effective UID of super-user can - request any valid limit value, including unlimited, but :exc:`ValueError` - will still be raised if the requested limit exceeds the system imposed - limit. - - ``setrlimit`` may also raise :exc:`error` if the underlying system call - fails. - -.. function:: prlimit(pid, resource[, limits]) - - Combines :func:`setrlimit` and :func:`getrlimit` in one function and - supports to get and set the resources limits of an arbitrary process. If - *pid* is 0, then the call applies to the current process. *resource* and - *limits* have the same meaning as in :func:`setrlimit`, except that - *limits* is optional. - - When *limits* is not given the function returns the *resource* limit of the - process *pid*. When *limits* is given the *resource* limit of the process is - set and the former resource limit is returned. - - Raises :exc:`ProcessLookupError` when *pid* can't be found and - :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for - the process. - - Availability: Linux 2.6.36 or later with glibc 2.13 or later - - .. versionadded:: 3.4 - - -These symbols define resources whose consumption can be controlled using the -:func:`setrlimit` and :func:`getrlimit` functions described below. The values of -these symbols are exactly the constants used by C programs. - -The Unix man page for :manpage:`getrlimit(2)` lists the available resources. -Note that not all systems use the same symbol or same value to denote the same -resource. This module does not attempt to mask platform differences --- symbols -not defined for a platform will not be available from this module on that -platform. - - -.. data:: RLIMIT_CORE - - The maximum size (in bytes) of a core file that the current process can create. - This may result in the creation of a partial core file if a larger core would be - required to contain the entire process image. - - -.. data:: RLIMIT_CPU - - The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See - the :mod:`signal` module documentation for information about how to catch this - signal and do something useful, e.g. flush open files to disk.) - - -.. data:: RLIMIT_FSIZE - - The maximum size of a file which the process may create. - - -.. data:: RLIMIT_DATA - - The maximum size (in bytes) of the process's heap. - - -.. data:: RLIMIT_STACK - - The maximum size (in bytes) of the call stack for the current process. This only - affects the stack of the main thread in a multi-threaded process. - - -.. data:: RLIMIT_RSS - - The maximum resident set size that should be made available to the process. - - -.. data:: RLIMIT_NPROC - - The maximum number of processes the current process may create. - - -.. data:: RLIMIT_NOFILE - - The maximum number of open file descriptors for the current process. - - -.. data:: RLIMIT_OFILE - - The BSD name for :const:`RLIMIT_NOFILE`. - - -.. data:: RLIMIT_MEMLOCK - - The maximum address space which may be locked in memory. - - -.. data:: RLIMIT_VMEM - - The largest area of mapped memory which the process may occupy. - - -.. data:: RLIMIT_AS - - The maximum area (in bytes) of address space which may be taken by the process. - - -.. data:: RLIMIT_MSGQUEUE - - The number of bytes that can be allocated for POSIX message queues. - - Availability: Linux 2.6.8 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_NICE - - The ceiling for the process's nice level (calculated as 20 - rlim_cur). - - Availability: Linux 2.6.12 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTPRIO - - The ceiling of the real-time priority. - - Availability: Linux 2.6.12 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTTIME - - The time limit (in microseconds) on CPU time that a process can spend - under real-time scheduling without making a blocking syscall. - - Availability: Linux 2.6.25 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_SIGPENDING - - The number of signals which the process may queue. - - Availability: Linux 2.6.8 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SBSIZE - - The maximum size (in bytes) of socket buffer usage for this user. - This limits the amount of network memory, and hence the amount of mbufs, - that this user may hold at any time. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SWAP - - The maximum size (in bytes) of the swap space that may be reserved or - used by all of this user id's processes. - This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. - Please see :manpage:`tuning(7)` for a complete description of this sysctl. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_NPTS - - The maximum number of pseudo-terminals created by this user id. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - -Resource Usage --------------- - -These functions are used to retrieve resource usage information: - - -.. function:: getrusage(who) - - This function returns an object that describes the resources consumed by either - the current process or its children, as specified by the *who* parameter. The - *who* parameter should be specified using one of the :const:`RUSAGE_\*` - constants described below. - - The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times - the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. - - For backward compatibility, the return value is also accessible as a tuple of 16 - elements. - - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are - floating point values representing the amount of time spent executing in user - mode and the amount of time spent executing in system mode, respectively. The - remaining values are integers. Consult the :manpage:`getrusage(2)` man page for - detailed information about these values. A brief summary is presented here: - - +--------+---------------------+-------------------------------+ - | Index | Field | Resource | - +========+=====================+===============================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float) | - +--------+---------------------+-------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float) | - +--------+---------------------+-------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+-------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+-------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+-------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+-------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+-------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+-------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+-------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+-------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+-------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+-------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+-------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+-------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+-------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+-------------------------------+ - - This function will raise a :exc:`ValueError` if an invalid *who* parameter is - specified. It may also raise :exc:`error` exception in unusual circumstances. - - -.. function:: getpagesize() - - Returns the number of bytes in a system page. (This need not be the same as the - hardware page size.) - -The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` -function to specify which processes information should be provided for. - - -.. data:: RUSAGE_SELF - - Pass to :func:`getrusage` to request resources consumed by the calling - process, which is the sum of resources used by all threads in the process. - - -.. data:: RUSAGE_CHILDREN - - Pass to :func:`getrusage` to request resources consumed by child processes - of the calling process which have been terminated and waited for. - - -.. data:: RUSAGE_BOTH - - Pass to :func:`getrusage` to request resources consumed by both the current - process and child processes. May not be available on all systems. - - -.. data:: RUSAGE_THREAD - - Pass to :func:`getrusage` to request resources consumed by the current - thread. May not be available on all systems. - - .. versionadded:: 3.2 diff --git a/third_party/python/Doc/library/rlcompleter.rst b/third_party/python/Doc/library/rlcompleter.rst deleted file mode 100644 index 40b09ce89..000000000 --- a/third_party/python/Doc/library/rlcompleter.rst +++ /dev/null @@ -1,61 +0,0 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== - -.. module:: rlcompleter - :synopsis: Python identifier completion, suitable for the GNU readline library. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/rlcompleter.py` - --------------- - -The :mod:`rlcompleter` module defines a completion function suitable for the -:mod:`readline` module by completing valid Python identifiers and keywords. - -When this module is imported on a Unix platform with the :mod:`readline` module -available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. - -Example:: - - >>> import rlcompleter - >>> import readline - >>> readline.parse_and_bind("tab: complete") - >>> readline. - readline.__doc__ readline.get_line_buffer( readline.read_init_file( - readline.__file__ readline.insert_text( readline.set_completer( - readline.__name__ readline.parse_and_bind( - >>> readline. - -The :mod:`rlcompleter` module is designed for use with Python's -:ref:`interactive mode `. Unless Python is run with the -:option:`-S` option, the module is automatically imported and configured -(see :ref:`rlcompleter-config`). - -On platforms without :mod:`readline`, the :class:`Completer` class defined by -this module can still be used for custom purposes. - - -.. _completer-objects: - -Completer Objects ------------------ - -Completer objects have the following method: - - -.. method:: Completer.complete(text, state) - - Return the *state*\ th completion for *text*. - - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. - diff --git a/third_party/python/Doc/library/runpy.rst b/third_party/python/Doc/library/runpy.rst deleted file mode 100644 index af35e81a2..000000000 --- a/third_party/python/Doc/library/runpy.rst +++ /dev/null @@ -1,179 +0,0 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== - -.. module:: runpy - :synopsis: Locate and run Python modules without importing them first. - -.. moduleauthor:: Nick Coghlan - -**Source code:** :source:`Lib/runpy.py` - --------------- - -The :mod:`runpy` module is used to locate and run Python modules without -importing them first. Its main use is to implement the :option:`-m` command -line switch that allows scripts to be located using the Python module -namespace rather than the filesystem. - -Note that this is *not* a sandbox module - all code is executed in the -current process, and any side effects (such as cached imports of other -modules) will remain in place after the functions have returned. - -Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. -If that limitation is not acceptable for a given use case, :mod:`importlib` -is likely to be a more suitable choice than this module. - -The :mod:`runpy` module provides two functions: - - -.. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) - - .. index:: - module: __main__ - - Execute the code of the specified module and return the resulting module - globals dictionary. The module's code is first located using the standard - import mechanism (refer to :pep:`302` for details) and then executed in a - fresh module namespace. - - The *mod_name* argument should be an absolute module name. - If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within - that package is then executed and the resulting module globals dictionary - returned. - - The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are - overridden by :func:`run_module`. - - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). - - ``__name__`` is set to *run_name* if this optional argument is not - :const:`None`, to ``mod_name + '.__main__'`` if the named module is a - package and to the *mod_name* argument otherwise. - - ``__spec__`` will be set appropriately for the *actually* imported - module (that is, ``__spec__.name`` will always be *mod_name* or - ``mod_name + '.__main__``, never *run_name*). - - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are - :ref:`set as normal ` based on the module spec. - - If the argument *alter_sys* is supplied and evaluates to :const:`True`, - then ``sys.argv[0]`` is updated with the value of ``__file__`` and - ``sys.modules[__name__]`` is updated with a temporary module object for the - module being executed. Both ``sys.argv[0]`` and ``sys.modules[__name__]`` - are restored to their original values before the function returns. - - Note that this manipulation of :mod:`sys` is not thread-safe. Other threads - may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when - invoking this function from threaded code. - - .. seealso:: - The :option:`-m` option offering equivalent functionality from the - command line. - - .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. - - .. versionchanged:: 3.2 - Added ``__cached__`` global variable (see :pep:`3147`). - - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly for modules - run this way, as well as ensuring the real module name is always - accessible as ``__spec__.name``. - -.. function:: run_path(file_path, init_globals=None, run_name=None) - - .. index:: - module: __main__ - - Execute the code at the named filesystem location and return the resulting - module globals dictionary. As with a script name supplied to the CPython - command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). - - For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or - directory), the entry is first added to the beginning of ``sys.path``. The - function then looks for and executes a :mod:`__main__` module using the - updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if - there is no such module at the specified location. - - The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are - overridden by :func:`run_path`. - - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). - - ``__name__`` is set to *run_name* if this optional argument is not - :const:`None` and to ``''`` otherwise. - - If the supplied path directly references a script file (whether as source - or as precompiled byte code), then ``__file__`` will be set to the - supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and - ``__package__`` will all be set to :const:`None`. - - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` - module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be - :ref:`set as normal ` based on the module spec. - - A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated - with the value of ``file_path`` and ``sys.modules[__name__]`` is updated - with a temporary module object for the module being executed. All - modifications to items in :mod:`sys` are reverted before the function - returns. - - Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` - are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety - limitations still apply, use of this function in threaded code should be - either serialised with the import lock or delegated to a separate process. - - .. seealso:: - :ref:`using-on-interface-options` for equivalent functionality on the - command line (``python path/to/script``). - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather - than being executed directly. - -.. seealso:: - - :pep:`338` -- Executing modules as scripts - PEP written and implemented by Nick Coghlan. - - :pep:`366` -- Main module explicit relative imports - PEP written and implemented by Nick Coghlan. - - :pep:`451` -- A ModuleSpec Type for the Import System - PEP written and implemented by Eric Snow - - :ref:`using-on-general` - CPython command line details - - The :func:`importlib.import_module` function diff --git a/third_party/python/Doc/library/sched.rst b/third_party/python/Doc/library/sched.rst deleted file mode 100644 index 6094a7b87..000000000 --- a/third_party/python/Doc/library/sched.rst +++ /dev/null @@ -1,138 +0,0 @@ -:mod:`sched` --- Event scheduler -================================ - -.. module:: sched - :synopsis: General purpose event scheduler. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/sched.py` - -.. index:: single: event scheduling - --------------- - -The :mod:`sched` module defines a class which implements a general purpose event -scheduler: - -.. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) - - The :class:`scheduler` class defines a generic interface to scheduling events. - It needs two functions to actually deal with the "outside world" --- *timefunc* - should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one - argument, compatible with the output of *timefunc*, and should delay that many - time units. *delayfunc* will also be called with the argument ``0`` after each - event is run to allow other threads an opportunity to run in multi-threaded - applications. - - .. versionchanged:: 3.3 - *timefunc* and *delayfunc* parameters are optional. - - .. versionchanged:: 3.3 - :class:`scheduler` class can be safely used in multi-threaded - environments. - -Example:: - - >>> import sched, time - >>> s = sched.scheduler(time.time, time.sleep) - >>> def print_time(a='default'): - ... print("From print_time", time.time(), a) - ... - >>> def print_some_times(): - ... print(time.time()) - ... s.enter(10, 1, print_time) - ... s.enter(5, 2, print_time, argument=('positional',)) - ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) - ... s.run() - ... print(time.time()) - ... - >>> print_some_times() - 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default - 930343700.276 - -.. _scheduler-objects: - -Scheduler Objects ------------------ - -:class:`scheduler` instances have the following methods and attributes: - - -.. method:: scheduler.enterabs(time, priority, action, argument=(), kwargs={}) - - Schedule a new event. The *time* argument should be a numeric type compatible - with the return value of the *timefunc* function passed to the constructor. - Events scheduled for the same *time* will be executed in the order of their - *priority*. A lower number represents a higher priority. - - Executing the event means executing ``action(*argument, **kwargs)``. - *argument* is a sequence holding the positional arguments for *action*. - *kwargs* is a dictionary holding the keyword arguments for *action*. - - Return value is an event which may be used for later cancellation of the event - (see :meth:`cancel`). - - .. versionchanged:: 3.3 - *argument* parameter is optional. - - .. versionadded:: 3.3 - *kwargs* parameter was added. - - -.. method:: scheduler.enter(delay, priority, action, argument=(), kwargs={}) - - Schedule an event for *delay* more time units. Other than the relative time, the - other arguments, the effect and the return value are the same as those for - :meth:`enterabs`. - - .. versionchanged:: 3.3 - *argument* parameter is optional. - - .. versionadded:: 3.3 - *kwargs* parameter was added. - -.. method:: scheduler.cancel(event) - - Remove the event from the queue. If *event* is not an event currently in the - queue, this method will raise a :exc:`ValueError`. - - -.. method:: scheduler.empty() - - Return true if the event queue is empty. - - -.. method:: scheduler.run(blocking=True) - - Run all scheduled events. This method will wait (using the :func:`delayfunc` - function passed to the constructor) for the next event, then execute it and so - on until there are no more scheduled events. - - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). - - Either *action* or *delayfunc* can raise an exception. In either case, the - scheduler will maintain a consistent state and propagate the exception. If an - exception is raised by *action*, the event will not be attempted in future calls - to :meth:`run`. - - If a sequence of events takes longer to run than the time available before the - next event, the scheduler will simply fall behind. No events will be dropped; - the calling code is responsible for canceling events which are no longer - pertinent. - - .. versionadded:: 3.3 - *blocking* parameter was added. - -.. attribute:: scheduler.queue - - Read-only attribute returning a list of upcoming events in the order they - will be run. Each event is shown as a :term:`named tuple` with the - following fields: time, priority, action, argument, kwargs. diff --git a/third_party/python/Doc/library/secrets.rst b/third_party/python/Doc/library/secrets.rst deleted file mode 100644 index 9bf848f91..000000000 --- a/third_party/python/Doc/library/secrets.rst +++ /dev/null @@ -1,198 +0,0 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== - -.. module:: secrets - :synopsis: Generate secure random numbers for managing secrets. - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano -.. versionadded:: 3.6 - -.. testsetup:: - - from secrets import * - __name__ = '' - -**Source code:** :source:`Lib/secrets.py` - -------------- - -The :mod:`secrets` module is used for generating cryptographically strong -random numbers suitable for managing data such as passwords, account -authentication, security tokens, and related secrets. - -In particularly, :mod:`secrets` should be used in preference to the -default pseudo-random number generator in the :mod:`random` module, which -is designed for modelling and simulation, not security or cryptography. - -.. seealso:: - - :pep:`506` - - -Random numbers --------------- - -The :mod:`secrets` module provides access to the most secure source of -randomness that your operating system provides. - -.. class:: SystemRandom - - A class for generating random numbers using the highest-quality - sources provided by the operating system. See - :class:`random.SystemRandom` for additional details. - -.. function:: choice(sequence) - - Return a randomly-chosen element from a non-empty sequence. - -.. function:: randbelow(n) - - Return a random int in the range [0, *n*). - -.. function:: randbits(k) - - Return an int with *k* random bits. - - -Generating tokens ------------------ - -The :mod:`secrets` module provides functions for generating secure -tokens, suitable for applications such as password resets, -hard-to-guess URLs, and similar. - -.. function:: token_bytes([nbytes=None]) - - Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. - - .. doctest:: - - >>> token_bytes(16) #doctest:+SKIP - b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' - - -.. function:: token_hex([nbytes=None]) - - Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. - - .. doctest:: - - >>> token_hex(16) #doctest:+SKIP - 'f9bf78b9a18ce6d46a0cd2b0b86df9da' - -.. function:: token_urlsafe([nbytes=None]) - - Return a random URL-safe text string, containing *nbytes* random - bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. - - .. doctest:: - - >>> token_urlsafe(16) #doctest:+SKIP - 'Drmhze6EPcv0fN_81Bj-nA' - - -How many bytes should tokens use? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To be secure against -`brute-force attacks `_, -tokens need to have sufficient randomness. Unfortunately, what is -considered sufficient will necessarily increase as computers get more -powerful and able to make more guesses in a shorter period. As of 2015, -it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. - -For those who want to manage their own token length, you can explicitly -specify how much randomness is used for tokens by giving an :class:`int` -argument to the various ``token_*`` functions. That argument is taken -as the number of bytes of randomness to use. - -Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. - -.. note:: - - That default is subject to change at any time, including during - maintenance releases. - - -Other functions ---------------- - -.. function:: compare_digest(a, b) - - Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, - in such a way as to reduce the risk of - `timing attacks `_. - See :func:`hmac.compare_digest` for additional details. - - -Recipes and best practices --------------------------- - -This section shows recipes and best practices for using :mod:`secrets` -to manage a basic level of security. - -Generate an eight-character alphanumeric password: - -.. testcode:: - - import string - alphabet = string.ascii_letters + string.digits - password = ''.join(choice(alphabet) for i in range(8)) - - -.. note:: - - Applications should not - `store passwords in a recoverable format `_, - whether plain text or encrypted. They should be salted and hashed - using a cryptographically-strong one-way (irreversible) hash function. - - -Generate a ten-character alphanumeric password with at least one -lowercase character, at least one uppercase character, and at least -three digits: - -.. testcode:: - - import string - alphabet = string.ascii_letters + string.digits - while True: - password = ''.join(choice(alphabet) for i in range(10)) - if (any(c.islower() for c in password) - and any(c.isupper() for c in password) - and sum(c.isdigit() for c in password) >= 3): - break - - -Generate an `XKCD-style passphrase `_: - -.. testcode:: - - # On standard Linux systems, use a convenient dictionary file. - # Other platforms may need to provide their own word-list. - with open('/usr/share/dict/words') as f: - words = [word.strip() for word in f] - password = ' '.join(choice(words) for i in range(4)) - - -Generate a hard-to-guess temporary URL containing a security token -suitable for password recovery applications: - -.. testcode:: - - url = 'https://mydomain.com/reset=' + token_urlsafe() - - - -.. - # This modeline must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/third_party/python/Doc/library/select.rst b/third_party/python/Doc/library/select.rst deleted file mode 100644 index 413ec3579..000000000 --- a/third_party/python/Doc/library/select.rst +++ /dev/null @@ -1,640 +0,0 @@ -:mod:`select` --- Waiting for I/O completion -============================================ - -.. module:: select - :synopsis: Wait for I/O completion on multiple streams. - --------------- - -This module provides access to the :c:func:`select` and :c:func:`poll` functions -available in most operating systems, :c:func:`devpoll` available on -Solaris and derivatives, :c:func:`epoll` available on Linux 2.5+ and -:c:func:`kqueue` available on most BSD. -Note that on Windows, it only works for sockets; on other operating systems, -it also works for other file types (in particular, on Unix, it works on pipes). -It cannot be used on regular files to determine whether a file has grown since -it was last read. - -.. note:: - - The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are - encouraged to use the :mod:`selectors` module instead, unless they want - precise control over the OS-level primitives used. - - -The module defines the following: - - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -.. function:: devpoll() - - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` - polling object; see section :ref:`devpoll-objects` below for the - methods supported by devpoll objects. - - :c:func:`devpoll` objects are linked to the number of file - descriptors allowed at the time of instantiation. If your program - reduces this value, :c:func:`devpoll` will fail. If your program - increases this value, :c:func:`devpoll` may return an - incomplete list of active file descriptors. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - -.. function:: epoll(sizehint=-1, flags=0) - - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, - which can be used as Edge or Level Triggered interface for I/O - events. - - *sizehint* informs epoll about the expected number of events to be - registered. It must be positive, or `-1` to use the default. It is only - used on older systems where :c:func:`epoll_create1` is not available; - otherwise it has no effect (though its value is still checked). - - *flags* is deprecated and completely ignored. However, when supplied, its - value must be ``0`` or ``select.EPOLL_CLOEXEC``, otherwise ``OSError`` is - raised. - - See the :ref:`epoll-objects` section below for the methods supported by - epolling objects. - - ``epoll`` objects support the context management protocol: when used in a - :keyword:`with` statement, the new file descriptor is automatically closed - at the end of the block. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionchanged:: 3.3 - Added the *flags* parameter. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - The new file descriptor is now non-inheritable. - - .. deprecated:: 3.4 - The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. - Use :func:`os.set_inheritable` to make the file descriptor inheritable. - - -.. function:: poll() - - (Not supported by all operating systems.) Returns a polling object, which - supports registering and unregistering file descriptors, and then polling them - for I/O events; see section :ref:`poll-objects` below for the methods supported - by polling objects. - - -.. function:: kqueue() - - (Only supported on BSD.) Returns a kernel queue object; see section - :ref:`kqueue-objects` below for the methods supported by kqueue objects. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - -.. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - - (Only supported on BSD.) Returns a kernel event object; see section - :ref:`kevent-objects` below for the methods supported by kevent objects. - - -.. function:: select(rlist, wlist, xlist[, timeout]) - - This is a straightforward interface to the Unix :c:func:`select` system call. - The first three arguments are sequences of 'waitable objects': either - integers representing file descriptors or objects with a parameterless method - named :meth:`~io.IOBase.fileno` returning such an integer: - - * *rlist*: wait until ready for reading - * *wlist*: wait until ready for writing - * *xlist*: wait for an "exceptional condition" (see the manual page for what - your system considers such a condition) - - Empty sequences are allowed, but acceptance of three empty sequences is - platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating point number - in seconds. When the *timeout* argument is omitted the function blocks until - at least one file descriptor is ready. A time-out value of zero specifies a - poll and never blocks. - - The return value is a triple of lists of objects that are ready: subsets of the - first three arguments. When the time-out is reached without a file descriptor - becoming ready, three empty lists are returned. - - .. index:: - single: socket() (in module socket) - single: popen() (in module os) - - Among the acceptable object types in the sequences are Python :term:`file - objects ` (e.g. ``sys.stdin``, or objects returned by - :func:`open` or :func:`os.popen`), socket objects returned by - :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, - as long as it has an appropriate :meth:`~io.IOBase.fileno` method (that - really returns a file descriptor, not just a random integer). - - .. note:: - - .. index:: single: WinSock - - File objects on Windows are not acceptable, but sockets are. On Windows, - the underlying :c:func:`select` function is provided by the WinSock - library, and does not handle file descriptors that don't originate from - WinSock. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. attribute:: PIPE_BUF - - The minimum number of bytes which can be written without blocking to a pipe - when the pipe has been reported as ready for writing by :func:`~select.select`, - :func:`poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. - - This value is guaranteed by POSIX to be at least 512. Availability: Unix. - - .. versionadded:: 3.2 - - -.. _devpoll-objects: - -``/dev/poll`` Polling Objects ------------------------------ - -Solaris and derivatives have ``/dev/poll``. While :c:func:`select` is -O(highest file descriptor) and :c:func:`poll` is O(number of file -descriptors), ``/dev/poll`` is O(active file descriptors). - -``/dev/poll`` behaviour is very close to the standard :c:func:`poll` -object. - - -.. method:: devpoll.close() - - Close the file descriptor of the polling object. - - .. versionadded:: 3.4 - - -.. attribute:: devpoll.closed - - ``True`` if the polling object is closed. - - .. versionadded:: 3.4 - - -.. method:: devpoll.fileno() - - Return the file descriptor number of the polling object. - - .. versionadded:: 3.4 - - -.. method:: devpoll.register(fd[, eventmask]) - - Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. - - *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`poll` - object. The default value is a combination of the constants :const:`POLLIN`, - :const:`POLLPRI`, and :const:`POLLOUT`. - - .. warning:: - - Registering a file descriptor that's already registered is not an - error, but the result is undefined. The appropriate action is to - unregister or modify it first. This is an important difference - compared with :c:func:`poll`. - - -.. method:: devpoll.modify(fd[, eventmask]) - - This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same - explicitly. - - -.. method:: devpoll.unregister(fd) - - Remove a file descriptor being tracked by a polling object. Just like the - :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. - - Attempting to remove a file descriptor that was never registered is - safely ignored. - - -.. method:: devpoll.poll([timeout]) - - Polls the set of registered file descriptors, and returns a possibly-empty list - containing ``(fd, event)`` 2-tuples for the descriptors that have events or - errors to report. *fd* is the file descriptor, and *event* is a bitmask with - bits set for the reported events for that descriptor --- :const:`POLLIN` for - waiting input, :const:`POLLOUT` to indicate that the descriptor can be written - to, and so forth. An empty list indicates that the call timed out and no file - descriptors had any events to report. If *timeout* is given, it specifies the - length of time in milliseconds which the system will wait for events before - returning. If *timeout* is omitted, -1, or :const:`None`, the call will - block until there is an event for this poll object. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _epoll-objects: - -Edge and Level Trigger Polling (epoll) Objects ----------------------------------------------- - - http://linux.die.net/man/4/epoll - - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - - -.. method:: epoll.close() - - Close the control file descriptor of the epoll object. - - -.. attribute:: epoll.closed - - ``True`` if the epoll object is closed. - - -.. method:: epoll.fileno() - - Return the file descriptor number of the control fd. - - -.. method:: epoll.fromfd(fd) - - Create an epoll object from a given file descriptor. - - -.. method:: epoll.register(fd[, eventmask]) - - Register a fd descriptor with the epoll object. - - -.. method:: epoll.modify(fd, eventmask) - - Modify a registered file descriptor. - - -.. method:: epoll.unregister(fd) - - Remove a registered file descriptor from the epoll object. - - -.. method:: epoll.poll(timeout=-1, maxevents=-1) - - Wait for events. timeout in seconds (float) - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _poll-objects: - -Polling Objects ---------------- - -The :c:func:`poll` system call, supported on most Unix systems, provides better -scalability for network servers that service many, many clients at the same -time. :c:func:`poll` scales better because the system call only requires listing -the file descriptors of interest, while :c:func:`select` builds a bitmap, turns -on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`select` is O(highest file descriptor), while -:c:func:`poll` is O(number of file descriptors). - - -.. method:: poll.register(fd[, eventmask]) - - Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. - - *eventmask* is an optional bitmask describing the type of events you want to - check for, and can be a combination of the constants :const:`POLLIN`, - :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not - specified, the default value used will check for all 3 types of events. - - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ - - Registering a file descriptor that's already registered is not an error, and has - the same effect as registering the descriptor exactly once. - - -.. method:: poll.modify(fd, eventmask) - - Modifies an already registered fd. This has the same effect as - ``register(fd, eventmask)``. Attempting to modify a file descriptor - that was never registered causes an :exc:`OSError` exception with errno - :const:`ENOENT` to be raised. - - -.. method:: poll.unregister(fd) - - Remove a file descriptor being tracked by a polling object. Just like the - :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. - - Attempting to remove a file descriptor that was never registered causes a - :exc:`KeyError` exception to be raised. - - -.. method:: poll.poll([timeout]) - - Polls the set of registered file descriptors, and returns a possibly-empty list - containing ``(fd, event)`` 2-tuples for the descriptors that have events or - errors to report. *fd* is the file descriptor, and *event* is a bitmask with - bits set for the reported events for that descriptor --- :const:`POLLIN` for - waiting input, :const:`POLLOUT` to indicate that the descriptor can be written - to, and so forth. An empty list indicates that the call timed out and no file - descriptors had any events to report. If *timeout* is given, it specifies the - length of time in milliseconds which the system will wait for events before - returning. If *timeout* is omitted, negative, or :const:`None`, the call will - block until there is an event for this poll object. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _kqueue-objects: - -Kqueue Objects --------------- - -.. method:: kqueue.close() - - Close the control file descriptor of the kqueue object. - - -.. attribute:: kqueue.closed - - ``True`` if the kqueue object is closed. - - -.. method:: kqueue.fileno() - - Return the file descriptor number of the control fd. - - -.. method:: kqueue.fromfd(fd) - - Create a kqueue object from a given file descriptor. - - -.. method:: kqueue.control(changelist, max_events[, timeout=None]) -> eventlist - - Low level interface to kevent - - - changelist must be an iterable of kevent object or ``None`` - - max_events must be 0 or a positive integer - - timeout in seconds (floats possible) - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _kevent-objects: - -Kevent Objects --------------- - -https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 - -.. attribute:: kevent.ident - - Value used to identify the event. The interpretation depends on the filter - but it's usually the file descriptor. In the constructor ident can either - be an int or an object with a :meth:`~io.IOBase.fileno` method. kevent - stores the integer internally. - -.. attribute:: kevent.filter - - Name of the kernel filter. - - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on Mac OS X] | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | - +---------------------------+---------------------------------------------+ - -.. attribute:: kevent.flags - - Filter action. - - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ - - -.. attribute:: kevent.fflags - - Filter specific flags. - - :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_VNODE` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_PROC` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_NETDEV` filter flags (not available on Mac OS X): - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | - +----------------------------+--------------------------------------------+ - - -.. attribute:: kevent.data - - Filter specific data. - - -.. attribute:: kevent.udata - - User defined value. diff --git a/third_party/python/Doc/library/selectors.rst b/third_party/python/Doc/library/selectors.rst deleted file mode 100644 index 6d864a836..000000000 --- a/third_party/python/Doc/library/selectors.rst +++ /dev/null @@ -1,277 +0,0 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ - -.. module:: selectors - :synopsis: High-level I/O multiplexing. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/selectors.py` - --------------- - -Introduction ------------- - -This module allows high-level and efficient I/O multiplexing, built upon the -:mod:`select` module primitives. Users are encouraged to use this module -instead, unless they want precise control over the OS-level primitives used. - -It defines a :class:`BaseSelector` abstract base class, along with several -concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), -that can be used to wait for I/O readiness notification on multiple file -objects. In the following, "file object" refers to any object with a -:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. - -:class:`DefaultSelector` is an alias to the most efficient implementation -available on the current platform: this should be the default choice for most -users. - -.. note:: - The type of file objects supported depends on the platform: on Windows, - sockets are supported, but not pipes, whereas on Unix, both are supported - (some other types may be supported as well, such as fifos or special file - devices). - -.. seealso:: - - :mod:`select` - Low-level I/O multiplexing module. - - -Classes -------- - -Classes hierarchy:: - - BaseSelector - +-- SelectSelector - +-- PollSelector - +-- EpollSelector - +-- DevpollSelector - +-- KqueueSelector - - -In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules -constants below: - - +-----------------------+-----------------------------------------------+ - | Constant | Meaning | - +=======================+===============================================+ - | :const:`EVENT_READ` | Available for read | - +-----------------------+-----------------------------------------------+ - | :const:`EVENT_WRITE` | Available for write | - +-----------------------+-----------------------------------------------+ - - -.. class:: SelectorKey - - A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file descriptor, selected event - mask and attached data. It is returned by several :class:`BaseSelector` - methods. - - .. attribute:: fileobj - - File object registered. - - .. attribute:: fd - - Underlying file descriptor. - - .. attribute:: events - - Events that must be waited for on this file object. - - .. attribute:: data - - Optional opaque data associated to this file object: for example, this - could be used to store a per-client session ID. - - -.. class:: BaseSelector - - A :class:`BaseSelector` is used to wait for I/O event readiness on multiple - file objects. It supports file stream registration, unregistration, and a - method to wait for I/O events on those streams, with an optional timeout. - It's an abstract base class, so cannot be instantiated. Use - :class:`DefaultSelector` instead, or one of :class:`SelectSelector`, - :class:`KqueueSelector` etc. if you want to specifically use an - implementation, and your platform supports it. - :class:`BaseSelector` and its concrete implementations support the - :term:`context manager` protocol. - - .. abstractmethod:: register(fileobj, events, data=None) - - Register a file object for selection, monitoring it for I/O events. - - *fileobj* is the file object to monitor. It may either be an integer - file descriptor or an object with a ``fileno()`` method. - *events* is a bitwise mask of events to monitor. - *data* is an opaque object. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is already registered. - - .. abstractmethod:: unregister(fileobj) - - Unregister a file object from selection, removing it from monitoring. A - file object shall be unregistered prior to being closed. - - *fileobj* must be a file object previously registered. - - This returns the associated :class:`SelectorKey` instance, or raises a - :exc:`KeyError` if *fileobj* is not registered. It will raise - :exc:`ValueError` if *fileobj* is invalid (e.g. it has no ``fileno()`` - method or its ``fileno()`` method has an invalid return value). - - .. method:: modify(fileobj, events, data=None) - - Change a registered file object's monitored events or attached data. - - This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed - by :meth:`BaseSelector.register(fileobj, events, data)`, except that it - can be implemented more efficiently. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: select(timeout=None) - - Wait until some registered file objects become ready, or the timeout - expires. - - If ``timeout > 0``, this specifies the maximum wait time, in seconds. - If ``timeout <= 0``, the call won't block, and will report the currently - ready file objects. - If *timeout* is ``None``, the call will block until a monitored file object - becomes ready. - - This returns a list of ``(key, events)`` tuples, one for each ready file - object. - - *key* is the :class:`SelectorKey` instance corresponding to a ready file - object. - *events* is a bitmask of events ready on this file object. - - .. note:: - This method can return before any file object becomes ready or the - timeout has elapsed if the current process receives a signal: in this - case, an empty list will be returned. - - .. versionchanged:: 3.5 - The selector is now retried with a recomputed timeout when interrupted - by a signal if the signal handler did not raise an exception (see - :pep:`475` for the rationale), instead of returning an empty list - of events before the timeout. - - .. method:: close() - - Close the selector. - - This must be called to make sure that any underlying resource is freed. - The selector shall not be used once it has been closed. - - .. method:: get_key(fileobj) - - Return the key associated with a registered file object. - - This returns the :class:`SelectorKey` instance associated to this file - object, or raises :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: get_map() - - Return a mapping of file objects to selector keys. - - This returns a :class:`~collections.abc.Mapping` instance mapping - registered file objects to their associated :class:`SelectorKey` - instance. - - -.. class:: DefaultSelector() - - The default selector class, using the most efficient implementation - available on the current platform. This should be the default choice for - most users. - - -.. class:: SelectSelector() - - :func:`select.select`-based selector. - - -.. class:: PollSelector() - - :func:`select.poll`-based selector. - - -.. class:: EpollSelector() - - :func:`select.epoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.epoll` object. - -.. class:: DevpollSelector() - - :func:`select.devpoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.devpoll` object. - - .. versionadded:: 3.5 - -.. class:: KqueueSelector() - - :func:`select.kqueue`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.kqueue` object. - - -Examples --------- - -Here is a simple echo server implementation:: - - import selectors - import socket - - sel = selectors.DefaultSelector() - - def accept(sock, mask): - conn, addr = sock.accept() # Should be ready - print('accepted', conn, 'from', addr) - conn.setblocking(False) - sel.register(conn, selectors.EVENT_READ, read) - - def read(conn, mask): - data = conn.recv(1000) # Should be ready - if data: - print('echoing', repr(data), 'to', conn) - conn.send(data) # Hope it won't block - else: - print('closing', conn) - sel.unregister(conn) - conn.close() - - sock = socket.socket() - sock.bind(('localhost', 1234)) - sock.listen(100) - sock.setblocking(False) - sel.register(sock, selectors.EVENT_READ, accept) - - while True: - events = sel.select() - for key, mask in events: - callback = key.data - callback(key.fileobj, mask) diff --git a/third_party/python/Doc/library/shelve.rst b/third_party/python/Doc/library/shelve.rst deleted file mode 100644 index f08c58179..000000000 --- a/third_party/python/Doc/library/shelve.rst +++ /dev/null @@ -1,203 +0,0 @@ -:mod:`shelve` --- Python object persistence -=========================================== - -.. module:: shelve - :synopsis: Python object persistence. - -**Source code:** :source:`Lib/shelve.py` - -.. index:: module: pickle - --------------- - -A "shelf" is a persistent, dictionary-like object. The difference with "dbm" -databases is that the values (not the keys!) in a shelf can be essentially -arbitrary Python objects --- anything that the :mod:`pickle` module can handle. -This includes most class instances, recursive data types, and objects containing -lots of shared sub-objects. The keys are ordinary strings. - - -.. function:: open(filename, flag='c', protocol=None, writeback=False) - - Open a persistent dictionary. The filename specified is the base filename for - the underlying database. As a side-effect, an extension may be added to the - filename and more than one file may be created. By default, the underlying - database file is opened for reading and writing. The optional *flag* parameter - has the same interpretation as the *flag* parameter of :func:`dbm.open`. - - By default, version 3 pickles are used to serialize values. The version of the - pickle protocol can be specified with the *protocol* parameter. - - Because of Python semantics, a shelf cannot know when a mutable - persistent-dictionary entry is modified. By default modified objects are - written *only* when assigned to the shelf (see :ref:`shelve-example`). If the - optional *writeback* parameter is set to ``True``, all entries accessed are also - cached in memory, and written back on :meth:`~Shelf.sync` and - :meth:`~Shelf.close`; this can make it handier to mutate mutable entries in - the persistent dictionary, but, if many entries are accessed, it can consume - vast amounts of memory for the cache, and it can make the close operation - very slow since all accessed entries are written back (there is no way to - determine which accessed entries are mutable, nor which ones were actually - mutated). - - .. note:: - - Do not rely on the shelf being closed automatically; always call - :meth:`~Shelf.close` explicitly when you don't need it any more, or - use :func:`shelve.open` as a context manager:: - - with shelve.open('spam') as db: - db['eggs'] = 'eggs' - -.. warning:: - - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure - to load a shelf from an untrusted source. Like with pickle, loading a shelf - can execute arbitrary code. - -Shelf objects support all methods supported by dictionaries. This eases the -transition from dictionary based scripts to those requiring persistent storage. - -Two additional methods are supported: - -.. method:: Shelf.sync() - - Write back all entries in the cache if the shelf was opened with *writeback* - set to :const:`True`. Also empty the cache and synchronize the persistent - dictionary on disk, if feasible. This is called automatically when the shelf - is closed with :meth:`close`. - -.. method:: Shelf.close() - - Synchronize and close the persistent *dict* object. Operations on a closed - shelf will fail with a :exc:`ValueError`. - - -.. seealso:: - - `Persistent dictionary recipe `_ - with widely supported storage formats and having the speed of native - dictionaries. - - -Restrictions ------------- - - .. index:: - module: dbm.ndbm - module: dbm.gnu - -* The choice of which database package will be used (such as :mod:`dbm.ndbm` or - :mod:`dbm.gnu`) depends on which interface is available. Therefore it is not - safe to open the database directly using :mod:`dbm`. The database is also - (unfortunately) subject to the limitations of :mod:`dbm`, if it is used --- - this means that (the pickled representation of) the objects stored in the - database should be fairly small, and in rare cases key collisions may cause - the database to refuse updates. - -* The :mod:`shelve` module does not support *concurrent* read/write access to - shelved objects. (Multiple simultaneous read accesses are safe.) When a - program has a shelf open for writing, no other program should have it open for - reading or writing. Unix file locking can be used to solve this, but this - differs across Unix versions and requires knowledge about the database - implementation used. - - -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - - A subclass of :class:`collections.abc.MutableMapping` which stores pickled - values in the *dict* object. - - By default, version 3 pickles are used to serialize values. The version of the - pickle protocol can be specified with the *protocol* parameter. See the - :mod:`pickle` documentation for a discussion of the pickle protocols. - - If the *writeback* parameter is ``True``, the object will hold a cache of all - entries accessed and write them back to the *dict* at sync and close times. - This allows natural operations on mutable entries, but can consume much more - memory and make sync and close take a long time. - - The *keyencoding* parameter is the encoding used to encode keys before they - are used with the underlying dict. - - A :class:`Shelf` object can also be used as a context manager, in which - case it will be automatically closed when the :keyword:`with` block ends. - - .. versionchanged:: 3.2 - Added the *keyencoding* parameter; previously, keys were always encoded in - UTF-8. - - .. versionchanged:: 3.4 - Added context manager support. - - -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - - A subclass of :class:`Shelf` which exposes :meth:`first`, :meth:`!next`, - :meth:`previous`, :meth:`last` and :meth:`set_location` which are available - in the third-party :mod:`bsddb` module from `pybsddb - `_ but not in other database - modules. The *dict* object passed to the constructor must support those - methods. This is generally accomplished by calling one of - :func:`bsddb.hashopen`, :func:`bsddb.btopen` or :func:`bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. - - -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) - - A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like - object. The underlying file will be opened using :func:`dbm.open`. By - default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. - - -.. _shelve-example: - -Example -------- - -To summarize the interface (``key`` is a string, ``data`` is an arbitrary -object):: - - import shelve - - d = shelve.open(filename) # open -- file may get suffix added by low-level - # library - - d[key] = data # store data at key (overwrites old data if - # using an existing key) - data = d[key] # retrieve a COPY of data at key (raise KeyError - # if no such key) - del d[key] # delete data stored at key (raises KeyError - # if no such key) - - flag = key in d # true if the key exists - klist = list(d.keys()) # a list of all existing keys (slow!) - - # as d was opened WITHOUT writeback=True, beware: - d['xx'] = [0, 1, 2] # this works as expected, but... - d['xx'].append(3) # *this doesn't!* -- d['xx'] is STILL [0, 1, 2]! - - # having opened d without writeback=True, you need to code carefully: - temp = d['xx'] # extracts the copy - temp.append(5) # mutates the copy - d['xx'] = temp # stores the copy right back, to persist it - - # or, d=shelve.open(filename,writeback=True) would let you just code - # d['xx'].append(5) and have it work as expected, BUT it would also - # consume more memory and make the d.close() operation slower. - - d.close() # close it - - -.. seealso:: - - Module :mod:`dbm` - Generic interface to ``dbm``-style databases. - - Module :mod:`pickle` - Object serialization used by :mod:`shelve`. - diff --git a/third_party/python/Doc/library/shlex.rst b/third_party/python/Doc/library/shlex.rst deleted file mode 100644 index 55012f80e..000000000 --- a/third_party/python/Doc/library/shlex.rst +++ /dev/null @@ -1,416 +0,0 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== - -.. module:: shlex - :synopsis: Simple lexical analysis for Unix shell-like languages. - -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Gustavo Niemeyer -.. sectionauthor:: Eric S. Raymond -.. sectionauthor:: Gustavo Niemeyer - -**Source code:** :source:`Lib/shlex.py` - --------------- - -The :class:`~shlex.shlex` class makes it easy to write lexical analyzers for -simple syntaxes resembling that of the Unix shell. This will often be useful -for writing minilanguages, (for example, in run control files for Python -applications) or for parsing quoted strings. - -The :mod:`shlex` module defines the following functions: - - -.. function:: split(s, comments=False, posix=True) - - Split the string *s* using shell-like syntax. If *comments* is :const:`False` - (the default), the parsing of comments in the given string will be disabled - (setting the :attr:`~shlex.commenters` attribute of the - :class:`~shlex.shlex` instance to the empty string). This function operates - in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is - false. - - .. note:: - - Since the :func:`split` function instantiates a :class:`~shlex.shlex` - instance, passing ``None`` for *s* will read the string to split from - standard input. - - -.. function:: quote(s) - - Return a shell-escaped version of the string *s*. The returned value is a - string that can safely be used as one token in a shell command line, for - cases where you cannot use a list. - - This idiom would be unsafe:: - - >>> filename = 'somefile; rm -rf ~' - >>> command = 'ls -l {}'.format(filename) - >>> print(command) # executed by a shell: boom! - ls -l somefile; rm -rf ~ - - :func:`quote` lets you plug the security hole:: - - >>> command = 'ls -l {}'.format(quote(filename)) - >>> print(command) - ls -l 'somefile; rm -rf ~' - >>> remote_command = 'ssh home {}'.format(quote(command)) - >>> print(remote_command) - ssh home 'ls -l '"'"'somefile; rm -rf ~'"'"'' - - The quoting is compatible with UNIX shells and with :func:`split`: - - >>> remote_command = split(remote_command) - >>> remote_command - ['ssh', 'home', "ls -l 'somefile; rm -rf ~'"] - >>> command = split(remote_command[-1]) - >>> command - ['ls', '-l', 'somefile; rm -rf ~'] - - .. versionadded:: 3.3 - -The :mod:`shlex` module defines the following class: - - -.. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) - - A :class:`~shlex.shlex` instance or subclass instance is a lexical analyzer - object. The initialization argument, if present, specifies where to read - characters from. It must be a file-/stream-like object with - :meth:`~io.TextIOBase.read` and :meth:`~io.TextIOBase.readline` methods, or - a string. If no argument is given, input will be taken from ``sys.stdin``. - The second optional argument is a filename string, which sets the initial - value of the :attr:`~shlex.infile` attribute. If the *instream* - argument is omitted or equal to ``sys.stdin``, this second argument - defaults to "stdin". The *posix* argument defines the operational mode: - when *posix* is not true (default), the :class:`~shlex.shlex` instance will - operate in compatibility mode. When operating in POSIX mode, - :class:`~shlex.shlex` will try to be as close as possible to the POSIX shell - parsing rules. The *punctuation_chars* argument provides a way to make the - behaviour even closer to how real shells parse. This can take a number of - values: the default value, ``False``, preserves the behaviour seen under - Python 3.5 and earlier. If set to ``True``, then parsing of the characters - ``();<>|&`` is changed: any run of these characters (considered punctuation - characters) is returned as a single token. If set to a non-empty string of - characters, those characters will be used as the punctuation characters. Any - characters in the :attr:`wordchars` attribute that appear in - *punctuation_chars* will be removed from :attr:`wordchars`. See - :ref:`improved-shell-compatibility` for more information. - - .. versionchanged:: 3.6 - The *punctuation_chars* parameter was added. - -.. seealso:: - - Module :mod:`configparser` - Parser for configuration files similar to the Windows :file:`.ini` files. - - -.. _shlex-objects: - -shlex Objects -------------- - -A :class:`~shlex.shlex` instance has the following methods: - - -.. method:: shlex.get_token() - - Return a token. If tokens have been stacked using :meth:`push_token`, pop a - token off the stack. Otherwise, read one from the input stream. If reading - encounters an immediate end-of-file, :attr:`eof` is returned (the empty - string (``''``) in non-POSIX mode, and ``None`` in POSIX mode). - - -.. method:: shlex.push_token(str) - - Push the argument onto the token stack. - - -.. method:: shlex.read_token() - - Read a raw token. Ignore the pushback stack, and do not interpret source - requests. (This is not ordinarily a useful entry point, and is documented here - only for the sake of completeness.) - - -.. method:: shlex.sourcehook(filename) - - When :class:`~shlex.shlex` detects a source request (see :attr:`source` - below) this method is given the following token as argument, and expected - to return a tuple consisting of a filename and an open file-like object. - - Normally, this method first strips any quotes off the argument. If the result - is an absolute pathname, or there was no previous source request in effect, or - the previous source was a stream (such as ``sys.stdin``), the result is left - alone. Otherwise, if the result is a relative pathname, the directory part of - the name of the file immediately before it on the source inclusion stack is - prepended (this behavior is like the way the C preprocessor handles ``#include - "file.h"``). - - The result of the manipulations is treated as a filename, and returned as the - first component of the tuple, with :func:`open` called on it to yield the second - component. (Note: this is the reverse of the order of arguments in instance - initialization!) - - This hook is exposed so that you can use it to implement directory search paths, - addition of file extensions, and other namespace hacks. There is no - corresponding 'close' hook, but a shlex instance will call the - :meth:`~io.IOBase.close` method of the sourced input stream when it returns - EOF. - - For more explicit control of source stacking, use the :meth:`push_source` and - :meth:`pop_source` methods. - - -.. method:: shlex.push_source(newstream, newfile=None) - - Push an input source stream onto the input stack. If the filename argument is - specified it will later be available for use in error messages. This is the - same method used internally by the :meth:`sourcehook` method. - - -.. method:: shlex.pop_source() - - Pop the last-pushed input source from the input stack. This is the same method - used internally when the lexer reaches EOF on a stacked input stream. - - -.. method:: shlex.error_leader(infile=None, lineno=None) - - This method generates an error message leader in the format of a Unix C compiler - error label; the format is ``'"%s", line %d: '``, where the ``%s`` is replaced - with the name of the current source file and the ``%d`` with the current input - line number (the optional arguments can be used to override these). - - This convenience is provided to encourage :mod:`shlex` users to generate error - messages in the standard, parseable format understood by Emacs and other Unix - tools. - -Instances of :class:`~shlex.shlex` subclasses have some public instance -variables which either control lexical analysis or can be used for debugging: - - -.. attribute:: shlex.commenters - - The string of characters that are recognized as comment beginners. All - characters from the comment beginner to end of line are ignored. Includes just - ``'#'`` by default. - - -.. attribute:: shlex.wordchars - - The string of characters that will accumulate into multi-character tokens. By - default, includes all ASCII alphanumerics and underscore. In POSIX mode, the - accented characters in the Latin-1 set are also included. If - :attr:`punctuation_chars` is not empty, the characters ``~-./*?=``, which can - appear in filename specifications and command line parameters, will also be - included in this attribute, and any characters which appear in - ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. - - -.. attribute:: shlex.whitespace - - Characters that will be considered whitespace and skipped. Whitespace bounds - tokens. By default, includes space, tab, linefeed and carriage-return. - - -.. attribute:: shlex.escape - - Characters that will be considered as escape. This will be only used in POSIX - mode, and includes just ``'\'`` by default. - - -.. attribute:: shlex.quotes - - Characters that will be considered string quotes. The token accumulates until - the same quote is encountered again (thus, different quote types protect each - other as in the shell.) By default, includes ASCII single and double quotes. - - -.. attribute:: shlex.escapedquotes - - Characters in :attr:`quotes` that will interpret escape characters defined in - :attr:`escape`. This is only used in POSIX mode, and includes just ``'"'`` by - default. - - -.. attribute:: shlex.whitespace_split - - If ``True``, tokens will only be split in whitespaces. This is useful, for - example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. If this attribute is ``True``, - :attr:`punctuation_chars` will have no effect, and splitting will happen - only on whitespaces. When using :attr:`punctuation_chars`, which is - intended to provide parsing closer to that implemented by shells, it is - advisable to leave ``whitespace_split`` as ``False`` (the default value). - - -.. attribute:: shlex.infile - - The name of the current input file, as initially set at class instantiation time - or stacked by later source requests. It may be useful to examine this when - constructing error messages. - - -.. attribute:: shlex.instream - - The input stream from which this :class:`~shlex.shlex` instance is reading - characters. - - -.. attribute:: shlex.source - - This attribute is ``None`` by default. If you assign a string to it, that - string will be recognized as a lexical-level inclusion request similar to the - ``source`` keyword in various shells. That is, the immediately following token - will be opened as a filename and input will be taken from that stream until - EOF, at which point the :meth:`~io.IOBase.close` method of that stream will be - called and the input source will again become the original input stream. Source - requests may be stacked any number of levels deep. - - -.. attribute:: shlex.debug - - If this attribute is numeric and ``1`` or more, a :class:`~shlex.shlex` - instance will print verbose progress output on its behavior. If you need - to use this, you can read the module source code to learn the details. - - -.. attribute:: shlex.lineno - - Source line number (count of newlines seen so far plus one). - - -.. attribute:: shlex.token - - The token buffer. It may be useful to examine this when catching exceptions. - - -.. attribute:: shlex.eof - - Token used to determine end of file. This will be set to the empty string - (``''``), in non-POSIX mode, and to ``None`` in POSIX mode. - - -.. attribute:: shlex.punctuation_chars - - Characters that will be considered punctuation. Runs of punctuation - characters will be returned as a single token. However, note that no - semantic validity checking will be performed: for example, '>>>' could be - returned as a token, even though it may not be recognised as such by shells. - - .. versionadded:: 3.6 - - -.. _shlex-parsing-rules: - -Parsing Rules -------------- - -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the -following rules. - -* Quote characters are not recognized within words (``Do"Not"Separate`` is - parsed as the single word ``Do"Not"Separate``); - -* Escape characters are not recognized; - -* Enclosing characters in quotes preserve the literal value of all characters - within the quotes; - -* Closing quotes separate words (``"Do"Separate`` is parsed as ``"Do"`` and - ``Separate``); - -* If :attr:`~shlex.whitespace_split` is ``False``, any character not - declared to be a word character, whitespace, or a quote will be returned as - a single-character token. If it is ``True``, :class:`~shlex.shlex` will only - split words in whitespaces; - -* EOF is signaled with an empty string (``''``); - -* It's not possible to parse empty strings, even if quoted. - -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the -following parsing rules. - -* Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is - parsed as the single word ``DoNotSeparate``); - -* Non-quoted escape characters (e.g. ``'\'``) preserve the literal value of the - next character that follows; - -* Enclosing characters in quotes which are not part of - :attr:`~shlex.escapedquotes` (e.g. ``"'"``) preserve the literal value - of all characters within the quotes; - -* Enclosing characters in quotes which are part of - :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value - of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its - special meaning only when followed by the quote in use, or the escape - character itself. Otherwise the escape character will be considered a - normal character. - -* EOF is signaled with a :const:`None` value; - -* Quoted empty strings (``''``) are allowed. - -.. _improved-shell-compatibility: - -Improved Compatibility with Shells ----------------------------------- - -.. versionadded:: 3.6 - -The :class:`shlex` class provides compatibility with the parsing performed by -common Unix shells like ``bash``, ``dash``, and ``sh``. To take advantage of -this compatibility, specify the ``punctuation_chars`` argument in the -constructor. This defaults to ``False``, which preserves pre-3.6 behaviour. -However, if it is set to ``True``, then parsing of the characters ``();<>|&`` -is changed: any run of these characters is returned as a single token. While -this is short of a full parser for shells (which would be out of scope for the -standard library, given the multiplicity of shells out there), it does allow -you to perform processing of command lines more easily than you could -otherwise. To illustrate, you can see the difference in the following snippet: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> import shlex - >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> list(shlex.shlex(text)) - ['a', '&', '&', 'b', ';', 'c', '&', '&', 'd', '|', '|', 'e', ';', 'f', '>', - "'abc'", ';', '(', 'def', '"ghi"', ')'] - >>> list(shlex.shlex(text, punctuation_chars=True)) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', "'abc'", - ';', '(', 'def', '"ghi"', ')'] - -Of course, tokens will be returned which are not valid for shells, and you'll -need to implement your own error checks on the returned tokens. - -Instead of passing ``True`` as the value for the punctuation_chars parameter, -you can pass a string with specific characters, which will be used to determine -which characters constitute punctuation. For example:: - - >>> import shlex - >>> s = shlex.shlex("a && b || c", punctuation_chars="|") - >>> list(s) - ['a', '&', '&', 'b', '||', 'c'] - -.. note:: When ``punctuation_chars`` is specified, the :attr:`~shlex.wordchars` - attribute is augmented with the characters ``~-./*?=``. That is because these - characters can appear in file names (including wildcards) and command-line - arguments (e.g. ``--color=auto``). Hence:: - - >>> import shlex - >>> s = shlex.shlex('~/a && b-c --color=auto || d *.py?', - ... punctuation_chars=True) - >>> list(s) - ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] - -For best effect, ``punctuation_chars`` should be set in conjunction with -``posix=True``. (Note that ``posix=False`` is the default for -:class:`~shlex.shlex`.) diff --git a/third_party/python/Doc/library/shutil.rst b/third_party/python/Doc/library/shutil.rst deleted file mode 100644 index c3f7bd664..000000000 --- a/third_party/python/Doc/library/shutil.rst +++ /dev/null @@ -1,656 +0,0 @@ -:mod:`shutil` --- High-level file operations -============================================ - -.. module:: shutil - :synopsis: High-level file operations, including copying. - -.. sectionauthor:: Fred L. Drake, Jr. -.. partly based on the docstrings - -**Source code:** :source:`Lib/shutil.py` - -.. index:: - single: file; copying - single: copying files - --------------- - -The :mod:`shutil` module offers a number of high-level operations on files and -collections of files. In particular, functions are provided which support file -copying and removal. For operations on individual files, see also the -:mod:`os` module. - -.. warning:: - - Even the higher-level file copying functions (:func:`shutil.copy`, - :func:`shutil.copy2`) cannot copy all file metadata. - - On POSIX platforms, this means that file owner and group are lost as well - as ACLs. On Mac OS, the resource fork and other metadata are not used. - This means that resources will be lost and file type and creator codes will - not be correct. On Windows, file owners, ACLs and alternate data streams - are not copied. - - -.. _file-operations: - -Directory and files operations ------------------------------- - -.. function:: copyfileobj(fsrc, fdst[, length]) - - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. - The integer *length*, if given, is the buffer size. In particular, a negative - *length* value means to copy the data without looping over the source data in - chunks; by default the data is read in chunks to avoid uncontrolled memory - consumption. Note that if the current file position of the *fsrc* object is not - 0, only the contents from the current file position to the end of the file will - be copied. - - -.. function:: copyfile(src, dst, *, follow_symlinks=True) - - Copy the contents (no metadata) of the file named *src* to a file named - *dst* and return *dst*. *src* and *dst* are path names given as strings. - *dst* must be the complete target file name; look at :func:`shutil.copy` - for a copy that accepts a target directory path. If *src* and *dst* - specify the same file, :exc:`SameFileError` is raised. - - The destination location must be writable; otherwise, an :exc:`OSError` - exception will be raised. If *dst* already exists, it will be replaced. - Special files such as character or block devices and pipes cannot be - copied with this function. - - If *follow_symlinks* is false and *src* is a symbolic link, - a new symbolic link will be created instead of copying the - file *src* points to. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - Added *follow_symlinks* argument. - Now returns *dst*. - - .. versionchanged:: 3.4 - Raise :exc:`SameFileError` instead of :exc:`Error`. Since the former is - a subclass of the latter, this change is backward compatible. - - -.. exception:: SameFileError - - This exception is raised if source and destination in :func:`copyfile` - are the same file. - - .. versionadded:: 3.4 - - -.. function:: copymode(src, dst, *, follow_symlinks=True) - - Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. - If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, - :func:`copymode` will attempt to modify the mode of *dst* itself (rather - than the file it points to). This functionality is not available on every - platform; please see :func:`copystat` for more information. If - :func:`copymode` cannot modify symbolic links on the local platform, and it - is asked to do so, it will do nothing and return. - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument. - -.. function:: copystat(src, dst, *, follow_symlinks=True) - - Copy the permission bits, last access time, last modification time, and - flags from *src* to *dst*. On Linux, :func:`copystat` also copies the - "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. - - If *follow_symlinks* is false, and *src* and *dst* both - refer to symbolic links, :func:`copystat` will operate on - the symbolic links themselves rather than the files the - symbolic links refer to—reading the information from the - *src* symbolic link, and writing the information to the - *dst* symbolic link. - - .. note:: - - Not all platforms provide the ability to examine and - modify symbolic links. Python itself can tell you what - functionality is locally available. - - * If ``os.chmod in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the permission - bits of a symbolic link. - - * If ``os.utime in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the last access - and modification times of a symbolic link. - - * If ``os.chflags in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the flags of - a symbolic link. (``os.chflags`` is not available on - all platforms.) - - On platforms where some or all of this functionality - is unavailable, when asked to modify a symbolic link, - :func:`copystat` will copy everything it can. - :func:`copystat` never returns failure. - - Please see :data:`os.supports_follow_symlinks` - for more information. - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument and support for Linux extended attributes. - -.. function:: copy(src, dst, *, follow_symlinks=True) - - Copies the file *src* to the file or directory *dst*. *src* and *dst* - should be strings. If *dst* specifies a directory, the file will be - copied into *dst* using the base filename from *src*. Returns the - path to the newly created file. - - If *follow_symlinks* is false, and *src* is a symbolic link, - *dst* will be created as a symbolic link. If *follow_symlinks* - is true and *src* is a symbolic link, *dst* will be a copy of - the file *src* refers to. - - :func:`~shutil.copy` copies the file data and the file's permission - mode (see :func:`os.chmod`). Other metadata, like the - file's creation and modification times, is not preserved. - To preserve all file metadata from the original, use - :func:`~shutil.copy2` instead. - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument. - Now returns path to the newly created file. - -.. function:: copy2(src, dst, *, follow_symlinks=True) - - Identical to :func:`~shutil.copy` except that :func:`copy2` - also attempts to preserve file metadata. - - When *follow_symlinks* is false, and *src* is a symbolic - link, :func:`copy2` attempts to copy all metadata from the - *src* symbolic link to the newly-created *dst* symbolic link. - However, this functionality is not available on all platforms. - On platforms where some or all of this functionality is - unavailable, :func:`copy2` will preserve all the metadata - it can; :func:`copy2` never returns failure. - - :func:`copy2` uses :func:`copystat` to copy the file metadata. - Please see :func:`copystat` for more information - about platform support for modifying symbolic link metadata. - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument, try to copy extended - file system attributes too (currently Linux only). - Now returns path to the newly created file. - -.. function:: ignore_patterns(\*patterns) - - This factory function creates a function that can be used as a callable for - :func:`copytree`\'s *ignore* argument, ignoring files and directories that - match one of the glob-style *patterns* provided. See the example below. - - -.. function:: copytree(src, dst, symlinks=False, ignore=None, \ - copy_function=copy2, ignore_dangling_symlinks=False) - - Recursively copy an entire directory tree rooted at *src*, returning the - destination directory. The destination - directory, named by *dst*, must not already exist; it will be created as - well as missing parent directories. Permissions and times of directories - are copied with :func:`copystat`, individual files are copied using - :func:`shutil.copy2`. - - If *symlinks* is true, symbolic links in the source tree are represented as - symbolic links in the new tree and the metadata of the original links will - be copied as far as the platform allows; if false or omitted, the contents - and metadata of the linked files are copied to the new tree. - - When *symlinks* is false, if the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an :exc:`Error` exception at the end of the copy process. - You can set the optional *ignore_dangling_symlinks* flag to true if you - want to silence this exception. Notice that this option has no effect - on platforms that don't support :func:`os.symlink`. - - If *ignore* is given, it must be a callable that will receive as its - arguments the directory being visited by :func:`copytree`, and a list of its - contents, as returned by :func:`os.listdir`. Since :func:`copytree` is - called recursively, the *ignore* callable will be called once for each - directory that is copied. The callable must return a sequence of directory - and file names relative to the current directory (i.e. a subset of the items - in its second argument); these names will then be ignored in the copy - process. :func:`ignore_patterns` can be used to create such a callable that - ignores names based on glob-style patterns. - - If exception(s) occur, an :exc:`Error` is raised with a list of reasons. - - If *copy_function* is given, it must be a callable that will be used to copy - each file. It will be called with the source path and the destination path - as arguments. By default, :func:`shutil.copy2` is used, but any function - that supports the same signature (like :func:`shutil.copy`) can be used. - - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - - .. versionchanged:: 3.2 - Added the *copy_function* argument to be able to provide a custom copy - function. - Added the *ignore_dangling_symlinks* argument to silent dangling symlinks - errors when *symlinks* is false. - - -.. function:: rmtree(path, ignore_errors=False, onerror=None) - - .. index:: single: directory; deleting - - Delete an entire directory tree; *path* must point to a directory (but not a - symbolic link to a directory). If *ignore_errors* is true, errors resulting - from failed removals will be ignored; if false or omitted, such errors are - handled by calling a handler specified by *onerror* or, if that is omitted, - they raise an exception. - - .. note:: - - On platforms that support the necessary fd-based functions a symlink - attack resistant version of :func:`rmtree` is used by default. On other - platforms, the :func:`rmtree` implementation is susceptible to a symlink - attack: given proper timing and circumstances, attackers can manipulate - symlinks on the filesystem to delete files they wouldn't be able to access - otherwise. Applications can use the :data:`rmtree.avoids_symlink_attacks` - function attribute to determine which case applies. - - If *onerror* is provided, it must be a callable that accepts three - parameters: *function*, *path*, and *excinfo*. - - The first parameter, *function*, is the function which raised the exception; - it depends on the platform and implementation. The second parameter, - *path*, will be the path name passed to *function*. The third parameter, - *excinfo*, will be the exception information returned by - :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. - - .. versionchanged:: 3.3 - Added a symlink attack resistant version that is used automatically - if platform supports fd-based functions. - - .. attribute:: rmtree.avoids_symlink_attacks - - Indicates whether the current platform and implementation provides a - symlink attack resistant version of :func:`rmtree`. Currently this is - only true for platforms supporting fd-based directory access functions. - - .. versionadded:: 3.3 - - -.. function:: move(src, dst, copy_function=copy2) - - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. - - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. - - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. - - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dest* if - :func:`os.rename` cannot be used. If the source is a directory, - :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the - *copy_function* allows the move to succeed when it is not possible to also - copy the metadata, at the expense of not copying any of the metadata. - - .. versionchanged:: 3.3 - Added explicit symlink handling for foreign filesystems, thus adapting - it to the behavior of GNU's :program:`mv`. - Now returns *dst*. - - .. versionchanged:: 3.5 - Added the *copy_function* keyword argument. - -.. function:: disk_usage(path) - - Return disk usage statistics about the given path as a :term:`named tuple` - with the attributes *total*, *used* and *free*, which are the amount of - total, used and free space, in bytes. On Windows, *path* must be a - directory; on Unix, it can be a file or directory. - - .. versionadded:: 3.3 - - Availability: Unix, Windows. - -.. function:: chown(path, user=None, group=None) - - Change owner *user* and/or *group* of the given *path*. - - *user* can be a system user name or a uid; the same applies to *group*. At - least one argument is required. - - See also :func:`os.chown`, the underlying function. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) - - Return the path to an executable which would be run if the given *cmd* was - called. If no *cmd* would be called, return ``None``. - - *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. - - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :attr:`os.defpath`. - - On Windows, the current directory is always prepended to the *path* whether - or not you use the default or provide your own, which is the behavior the - command shell uses when finding executables. Additionally, when finding the - *cmd* in the *path*, the ``PATHEXT`` environment variable is checked. For - example, if you call ``shutil.which("python")``, :func:`which` will search - ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* - directories. For example, on Windows:: - - >>> shutil.which("python") - 'C:\\Python33\\python.EXE' - - .. versionadded:: 3.3 - - -.. exception:: Error - - This exception collects exceptions that are raised during a multi-file - operation. For :func:`copytree`, the exception argument is a list of 3-tuples - (*srcname*, *dstname*, *exception*). - - -.. _shutil-copytree-example: - -copytree example -~~~~~~~~~~~~~~~~ - -This example is the implementation of the :func:`copytree` function, described -above, with the docstring omitted. It demonstrates many of the other functions -provided by this module. :: - - def copytree(src, dst, symlinks=False): - names = os.listdir(src) - os.makedirs(dst) - errors = [] - for name in names: - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except OSError as why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error as err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError as why: - # can't copy file access times on Windows - if why.winerror is None: - errors.extend((src, dst, str(why))) - if errors: - raise Error(errors) - -Another example that uses the :func:`ignore_patterns` helper:: - - from shutil import copytree, ignore_patterns - - copytree(source, destination, ignore=ignore_patterns('*.pyc', 'tmp*')) - -This will copy everything except ``.pyc`` files and files or directories whose -name starts with ``tmp``. - -Another example that uses the *ignore* argument to add a logging call:: - - from shutil import copytree - import logging - - def _logpath(path, names): - logging.info('Working in %s', path) - return [] # nothing will be ignored - - copytree(source, destination, ignore=_logpath) - - -.. _shutil-rmtree-example: - -rmtree example -~~~~~~~~~~~~~~ - -This example shows how to remove a directory tree on Windows where some -of the files have their read-only bit set. It uses the onerror callback -to clear the readonly bit and reattempt the remove. Any subsequent failure -will propagate. :: - - import os, stat - import shutil - - def remove_readonly(func, path, _): - "Clear the readonly bit and reattempt the removal" - os.chmod(path, stat.S_IWRITE) - func(path) - - shutil.rmtree(directory, onerror=remove_readonly) - -.. _archiving-operations: - -Archiving operations --------------------- - -.. versionadded:: 3.2 - -.. versionchanged:: 3.5 - Added support for the *xztar* format. - - -High-level utilities to create and read compressed and archived files are also -provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - -.. function:: make_archive(base_name, format, [root_dir, [base_dir, [verbose, [dry_run, [owner, [group, [logger]]]]]]]) - - Create an archive file (such as zip or tar) and return its name. - - *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of - "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the - :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is - available), or "xztar" (if the :mod:`lzma` module is available). - - *root_dir* is a directory that will be the root directory of the - archive; for example, we typically chdir into *root_dir* before creating the - archive. - - *base_dir* is the directory where we start archiving from; - i.e. *base_dir* will be the common prefix of all files and - directories in the archive. - - *root_dir* and *base_dir* both default to the current directory. - - If *dry_run* is true, no archive is created, but the operations that would be - executed are logged to *logger*. - - *owner* and *group* are used when creating a tar archive. By default, - uses the current owner and group. - - *logger* must be an object compatible with :pep:`282`, usually an instance of - :class:`logging.Logger`. - - The *verbose* argument is unused and deprecated. - - -.. function:: get_archive_formats() - - Return a list of supported formats for archiving. - Each element of the returned sequence is a tuple ``(name, description)``. - - By default :mod:`shutil` provides these formats: - - - *zip*: ZIP file (if the :mod:`zlib` module is available). - - *tar*: uncompressed tar file. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). - - You can register new formats or provide your own archiver for any existing - formats, by using :func:`register_archive_format`. - - -.. function:: register_archive_format(name, function, [extra_args, [description]]) - - Register an archiver for the format *name*. - - *function* is the callable that will be used to unpack archives. The callable - will receive the *base_name* of the file to create, followed by the - *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. - Further arguments are passed as keyword arguments: *owner*, *group*, - *dry_run* and *logger* (as passed in :func:`make_archive`). - - If given, *extra_args* is a sequence of ``(name, value)`` pairs that will be - used as extra keywords arguments when the archiver callable is used. - - *description* is used by :func:`get_archive_formats` which returns the - list of archivers. Defaults to an empty string. - - -.. function:: unregister_archive_format(name) - - Remove the archive format *name* from the list of supported formats. - - -.. function:: unpack_archive(filename[, extract_dir[, format]]) - - Unpack an archive. *filename* is the full path of the archive. - - *extract_dir* is the name of the target directory where the archive is - unpacked. If not provided, the current working directory is used. - - *format* is the archive format: one of "zip", "tar", "gztar", "bztar", or - "xztar". Or any other format registered with - :func:`register_unpack_format`. If not provided, :func:`unpack_archive` - will use the archive file name extension and see if an unpacker was - registered for that extension. In case none is found, - a :exc:`ValueError` is raised. - - -.. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) - - Registers an unpack format. *name* is the name of the format and - *extensions* is a list of extensions corresponding to the format, like - ``.zip`` for Zip files. - - *function* is the callable that will be used to unpack archives. The - callable will receive the path of the archive, followed by the directory - the archive must be extracted to. - - When provided, *extra_args* is a sequence of ``(name, value)`` tuples that - will be passed as keywords arguments to the callable. - - *description* can be provided to describe the format, and will be returned - by the :func:`get_unpack_formats` function. - - -.. function:: unregister_unpack_format(name) - - Unregister an unpack format. *name* is the name of the format. - - -.. function:: get_unpack_formats() - - Return a list of all registered formats for unpacking. - Each element of the returned sequence is a tuple - ``(name, extensions, description)``. - - By default :mod:`shutil` provides these formats: - - - *zip*: ZIP file (unpacking compressed files works only if the corresponding - module is available). - - *tar*: uncompressed tar file. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). - - You can register new formats or provide your own unpacker for any existing - formats, by using :func:`register_unpack_format`. - - -.. _shutil-archiving-example: - -Archiving example -~~~~~~~~~~~~~~~~~ - -In this example, we create a gzip'ed tar-file archive containing all files -found in the :file:`.ssh` directory of the user:: - - >>> from shutil import make_archive - >>> import os - >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) - >>> root_dir = os.path.expanduser(os.path.join('~', '.ssh')) - >>> make_archive(archive_name, 'gztar', root_dir) - '/Users/tarek/myarchive.tar.gz' - -The resulting archive contains: - -.. code-block:: shell-session - - $ tar -tzvf /Users/tarek/myarchive.tar.gz - drwx------ tarek/staff 0 2010-02-01 16:23:40 ./ - -rw-r--r-- tarek/staff 609 2008-06-09 13:26:54 ./authorized_keys - -rwxr-xr-x tarek/staff 65 2008-06-09 13:26:54 ./config - -rwx------ tarek/staff 668 2008-06-09 13:26:54 ./id_dsa - -rwxr-xr-x tarek/staff 609 2008-06-09 13:26:54 ./id_dsa.pub - -rw------- tarek/staff 1675 2008-06-09 13:26:54 ./id_rsa - -rw-r--r-- tarek/staff 397 2008-06-09 13:26:54 ./id_rsa.pub - -rw-r--r-- tarek/staff 37192 2010-02-06 18:23:10 ./known_hosts - - -Querying the size of the output terminal ----------------------------------------- - -.. function:: get_terminal_size(fallback=(columns, lines)) - - Get the size of the terminal window. - - For each of the two dimensions, the environment variable, ``COLUMNS`` - and ``LINES`` respectively, is checked. If the variable is defined and - the value is a positive integer, it is used. - - When ``COLUMNS`` or ``LINES`` is not defined, which is the common case, - the terminal connected to :data:`sys.__stdout__` is queried - by invoking :func:`os.get_terminal_size`. - - If the terminal size cannot be successfully queried, either because - the system doesn't support querying, or because we are not - connected to a terminal, the value given in ``fallback`` parameter - is used. ``fallback`` defaults to ``(80, 24)`` which is the default - size used by many terminal emulators. - - The value returned is a named tuple of type :class:`os.terminal_size`. - - See also: The Single UNIX Specification, Version 2, - `Other Environment Variables`_. - - .. versionadded:: 3.3 - -.. _`Other Environment Variables`: - http://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html#tag_002_003 - diff --git a/third_party/python/Doc/library/signal.rst b/third_party/python/Doc/library/signal.rst deleted file mode 100644 index 46d71def0..000000000 --- a/third_party/python/Doc/library/signal.rst +++ /dev/null @@ -1,466 +0,0 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== - -.. module:: signal - :synopsis: Set handlers for asynchronous events. - --------------- - -This module provides mechanisms to use signal handlers in Python. - - -General rules -------------- - -The :func:`signal.signal` function allows defining custom handlers to be -executed when a signal is received. A small number of default handlers are -installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets -can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception. - -A handler for a particular signal, once set, remains installed until it is -explicitly reset (Python emulates the BSD style interface regardless of the -underlying implementation), with the exception of the handler for -:const:`SIGCHLD`, which follows the underlying implementation. - - -Execution of Python signal handlers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A Python signal handler does not get executed inside the low-level (C) signal -handler. Instead, the low-level signal handler sets a flag which tells the -:term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). -This has consequences: - -* It makes little sense to catch synchronous errors like :const:`SIGFPE` or - :const:`SIGSEGV` that are caused by an invalid operation in C code. Python - will return from the signal handler to the C code, which is likely to raise - the same signal again, causing Python to apparently hang. From Python 3.3 - onwards, you can use the :mod:`faulthandler` module to report on synchronous - errors. - -* A long-running calculation implemented purely in C (such as regular - expression matching on a large body of text) may run uninterrupted for an - arbitrary amount of time, regardless of any signals received. The Python - signal handlers will be called when the calculation finishes. - - -.. _signals-and-threads: - - -Signals and threads -^^^^^^^^^^^^^^^^^^^ - -Python signal handlers are always executed in the main Python thread, -even if the signal was received in another thread. This means that signals -can't be used as a means of inter-thread communication. You can use -the synchronization primitives from the :mod:`threading` module instead. - -Besides, only the main thread is allowed to set a new signal handler. - - -Module contents ---------------- - -.. versionchanged:: 3.5 - signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask - (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`) - related constants listed below were turned into - :class:`enums `. - :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and - :func:`sigwait` functions return human-readable - :class:`enums `. - - -The variables defined in the :mod:`signal` module are: - - -.. data:: SIG_DFL - - This is one of two standard signal handling options; it will simply perform - the default function for the signal. For example, on most systems the - default action for :const:`SIGQUIT` is to dump core and exit, while the - default action for :const:`SIGCHLD` is to simply ignore it. - - -.. data:: SIG_IGN - - This is another standard signal handler, which will simply ignore the given - signal. - - -.. data:: SIG* - - All the signal numbers are defined symbolically. For example, the hangup signal - is defined as :const:`signal.SIGHUP`; the variable names are identical to the - names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is - :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that - not all systems define the same set of signal names; only those names defined by - the system are defined by this module. - - -.. data:: CTRL_C_EVENT - - The signal corresponding to the :kbd:`Ctrl+C` keystroke event. This signal can - only be used with :func:`os.kill`. - - Availability: Windows. - - .. versionadded:: 3.2 - - -.. data:: CTRL_BREAK_EVENT - - The signal corresponding to the :kbd:`Ctrl+Break` keystroke event. This signal can - only be used with :func:`os.kill`. - - Availability: Windows. - - .. versionadded:: 3.2 - - -.. data:: NSIG - - One more than the number of the highest signal number. - - -.. data:: ITIMER_REAL - - Decrements interval timer in real time, and delivers :const:`SIGALRM` upon - expiration. - - -.. data:: ITIMER_VIRTUAL - - Decrements interval timer only when the process is executing, and delivers - SIGVTALRM upon expiration. - - -.. data:: ITIMER_PROF - - Decrements interval timer both when the process executes and when the - system is executing on behalf of the process. Coupled with ITIMER_VIRTUAL, - this timer is usually used to profile the time spent by the application - in user and kernel space. SIGPROF is delivered upon expiration. - - -.. data:: SIG_BLOCK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that signals are to be blocked. - - .. versionadded:: 3.3 - -.. data:: SIG_UNBLOCK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that signals are to be unblocked. - - .. versionadded:: 3.3 - -.. data:: SIG_SETMASK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that the signal mask is to be replaced. - - .. versionadded:: 3.3 - - -The :mod:`signal` module defines one exception: - -.. exception:: ItimerError - - Raised to signal an error from the underlying :func:`setitimer` or - :func:`getitimer` implementation. Expect this error if an invalid - interval timer or a negative time is passed to :func:`setitimer`. - This error is a subtype of :exc:`OSError`. - - .. versionadded:: 3.3 - This error used to be a subtype of :exc:`IOError`, which is now an - alias of :exc:`OSError`. - - -The :mod:`signal` module defines the following functions: - - -.. function:: alarm(time) - - If *time* is non-zero, this function requests that a :const:`SIGALRM` signal be - sent to the process in *time* seconds. Any previously scheduled alarm is - canceled (only one alarm can be scheduled at any time). The returned value is - then the number of seconds before any previously set alarm was to have been - delivered. If *time* is zero, no alarm is scheduled, and any scheduled alarm is - canceled. If the return value is zero, no alarm is currently scheduled. (See - the Unix man page :manpage:`alarm(2)`.) Availability: Unix. - - -.. function:: getsignal(signalnum) - - Return the current signal handler for the signal *signalnum*. The returned value - may be a callable Python object, or one of the special values - :const:`signal.SIG_IGN`, :const:`signal.SIG_DFL` or :const:`None`. Here, - :const:`signal.SIG_IGN` means that the signal was previously ignored, - :const:`signal.SIG_DFL` means that the default way of handling the signal was - previously in use, and ``None`` means that the previous signal handler was not - installed from Python. - - -.. function:: pause() - - Cause the process to sleep until a signal is received; the appropriate handler - will then be called. Returns nothing. Not on Windows. (See the Unix man page - :manpage:`signal(2)`.) - - See also :func:`sigwait`, :func:`sigwaitinfo`, :func:`sigtimedwait` and - :func:`sigpending`. - - -.. function:: pthread_kill(thread_id, signalnum) - - Send the signal *signalnum* to the thread *thread_id*, another thread in the - same process as the caller. The target thread can be executing any code - (Python or not). However, if the target thread is executing the Python - interpreter, the Python signal handlers will be :ref:`executed by the main - thread `. Therefore, the only point of sending a - signal to a particular Python thread would be to force a running system call - to fail with :exc:`InterruptedError`. - - Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` - attribute of :class:`threading.Thread` objects to get a suitable value - for *thread_id*. - - If *signalnum* is 0, then no signal is sent, but error checking is still - performed; this can be used to check if the target thread is still running. - - Availability: Unix (see the man page :manpage:`pthread_kill(3)` for further - information). - - See also :func:`os.kill`. - - .. versionadded:: 3.3 - - -.. function:: pthread_sigmask(how, mask) - - Fetch and/or change the signal mask of the calling thread. The signal mask - is the set of signals whose delivery is currently blocked for the caller. - Return the old signal mask as a set of signals. - - The behavior of the call is dependent on the value of *how*, as follows. - - * :data:`SIG_BLOCK`: The set of blocked signals is the union of the current - set and the *mask* argument. - * :data:`SIG_UNBLOCK`: The signals in *mask* are removed from the current - set of blocked signals. It is permissible to attempt to unblock a - signal which is not blocked. - * :data:`SIG_SETMASK`: The set of blocked signals is set to the *mask* - argument. - - *mask* is a set of signal numbers (e.g. {:const:`signal.SIGINT`, - :const:`signal.SIGTERM`}). Use ``range(1, signal.NSIG)`` for a full mask - including all signals. - - For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the - signal mask of the calling thread. - - Availability: Unix. See the man page :manpage:`sigprocmask(3)` and - :manpage:`pthread_sigmask(3)` for further information. - - See also :func:`pause`, :func:`sigpending` and :func:`sigwait`. - - .. versionadded:: 3.3 - - -.. function:: setitimer(which, seconds[, interval]) - - Sets given interval timer (one of :const:`signal.ITIMER_REAL`, - :const:`signal.ITIMER_VIRTUAL` or :const:`signal.ITIMER_PROF`) specified - by *which* to fire after *seconds* (float is accepted, different from - :func:`alarm`) and after that every *interval* seconds. The interval - timer specified by *which* can be cleared by setting seconds to zero. - - When an interval timer fires, a signal is sent to the process. - The signal sent is dependent on the timer being used; - :const:`signal.ITIMER_REAL` will deliver :const:`SIGALRM`, - :const:`signal.ITIMER_VIRTUAL` sends :const:`SIGVTALRM`, - and :const:`signal.ITIMER_PROF` will deliver :const:`SIGPROF`. - - The old values are returned as a tuple: (delay, interval). - - Attempting to pass an invalid interval timer will cause an - :exc:`ItimerError`. Availability: Unix. - - -.. function:: getitimer(which) - - Returns current value of a given interval timer specified by *which*. - Availability: Unix. - - -.. function:: set_wakeup_fd(fd) - - Set the wakeup file descriptor to *fd*. When a signal is received, the - signal number is written as a single byte into the fd. This can be used by - a library to wakeup a poll or select call, allowing the signal to be fully - processed. - - The old wakeup fd is returned (or -1 if file descriptor wakeup was not - enabled). If *fd* is -1, file descriptor wakeup is disabled. - If not -1, *fd* must be non-blocking. It is up to the library to remove - any bytes from *fd* before calling poll or select again. - - Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the - signal numbers list. - - When threads are enabled, this function can only be called from the main thread; - attempting to call it from other threads will cause a :exc:`ValueError` - exception to be raised. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -.. function:: siginterrupt(signalnum, flag) - - Change system call restart behaviour: if *flag* is :const:`False`, system - calls will be restarted when interrupted by signal *signalnum*, otherwise - system calls will be interrupted. Returns nothing. Availability: Unix (see - the man page :manpage:`siginterrupt(3)` for further information). - - Note that installing a signal handler with :func:`signal` will reset the - restart behaviour to interruptible by implicitly calling - :c:func:`siginterrupt` with a true *flag* value for the given signal. - - -.. function:: signal(signalnum, handler) - - Set the handler for signal *signalnum* to the function *handler*. *handler* can - be a callable Python object taking two arguments (see below), or one of the - special values :const:`signal.SIG_IGN` or :const:`signal.SIG_DFL`. The previous - signal handler will be returned (see the description of :func:`getsignal` - above). (See the Unix man page :manpage:`signal(2)`.) - - When threads are enabled, this function can only be called from the main thread; - attempting to call it from other threads will cause a :exc:`ValueError` - exception to be raised. - - The *handler* is called with two arguments: the signal number and the current - stack frame (``None`` or a frame object; for a description of frame objects, - see the :ref:`description in the type hierarchy ` or see the - attribute descriptions in the :mod:`inspect` module). - - On Windows, :func:`signal` can only be called with :const:`SIGABRT`, - :const:`SIGFPE`, :const:`SIGILL`, :const:`SIGINT`, :const:`SIGSEGV`, - :const:`SIGTERM`, or :const:`SIGBREAK`. - A :exc:`ValueError` will be raised in any other case. - Note that not all systems define the same set of signal names; an - :exc:`AttributeError` will be raised if a signal name is not defined as - ``SIG*`` module level constant. - - -.. function:: sigpending() - - Examine the set of signals that are pending for delivery to the calling - thread (i.e., the signals which have been raised while blocked). Return the - set of the pending signals. - - Availability: Unix (see the man page :manpage:`sigpending(2)` for further - information). - - See also :func:`pause`, :func:`pthread_sigmask` and :func:`sigwait`. - - .. versionadded:: 3.3 - - -.. function:: sigwait(sigset) - - Suspend execution of the calling thread until the delivery of one of the - signals specified in the signal set *sigset*. The function accepts the signal - (removes it from the pending list of signals), and returns the signal number. - - Availability: Unix (see the man page :manpage:`sigwait(3)` for further - information). - - See also :func:`pause`, :func:`pthread_sigmask`, :func:`sigpending`, - :func:`sigwaitinfo` and :func:`sigtimedwait`. - - .. versionadded:: 3.3 - - -.. function:: sigwaitinfo(sigset) - - Suspend execution of the calling thread until the delivery of one of the - signals specified in the signal set *sigset*. The function accepts the - signal and removes it from the pending list of signals. If one of the - signals in *sigset* is already pending for the calling thread, the function - will return immediately with information about that signal. The signal - handler is not called for the delivered signal. The function raises an - :exc:`InterruptedError` if it is interrupted by a signal that is not in - *sigset*. - - The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. - - Availability: Unix (see the man page :manpage:`sigwaitinfo(2)` for further - information). - - See also :func:`pause`, :func:`sigwait` and :func:`sigtimedwait`. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The function is now retried if interrupted by a signal not in *sigset* - and the signal handler does not raise an exception (see :pep:`475` for - the rationale). - - -.. function:: sigtimedwait(sigset, timeout) - - Like :func:`sigwaitinfo`, but takes an additional *timeout* argument - specifying a timeout. If *timeout* is specified as :const:`0`, a poll is - performed. Returns :const:`None` if a timeout occurs. - - Availability: Unix (see the man page :manpage:`sigtimedwait(2)` for further - information). - - See also :func:`pause`, :func:`sigwait` and :func:`sigwaitinfo`. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The function is now retried with the recomputed *timeout* if interrupted - by a signal not in *sigset* and the signal handler does not raise an - exception (see :pep:`475` for the rationale). - - -.. _signal-example: - -Example -------- - -Here is a minimal example program. It uses the :func:`alarm` function to limit -the time spent waiting to open a file; this is useful if the file is for a -serial device that may not be turned on, which would normally cause the -:func:`os.open` to hang indefinitely. The solution is to set a 5-second alarm -before opening the file; if the operation takes too long, the alarm signal will -be sent, and the handler raises an exception. :: - - import signal, os - - def handler(signum, frame): - print('Signal handler called with signal', signum) - raise OSError("Couldn't open device!") - - # Set the signal handler and a 5-second alarm - signal.signal(signal.SIGALRM, handler) - signal.alarm(5) - - # This open() may hang indefinitely - fd = os.open('/dev/ttyS0', os.O_RDWR) - - signal.alarm(0) # Disable the alarm - diff --git a/third_party/python/Doc/library/site.rst b/third_party/python/Doc/library/site.rst deleted file mode 100644 index e1f4a4b22..000000000 --- a/third_party/python/Doc/library/site.rst +++ /dev/null @@ -1,257 +0,0 @@ -:mod:`site` --- Site-specific configuration hook -================================================ - -.. module:: site - :synopsis: Module responsible for site-specific configuration. - -**Source code:** :source:`Lib/site.py` - --------------- - -.. highlightlang:: none - -**This module is automatically imported during initialization.** The automatic -import can be suppressed using the interpreter's :option:`-S` option. - -.. index:: triple: module; search; path - -Importing this module will append site-specific paths to the module search path -and add a few builtins, unless :option:`-S` was used. In that case, this module -can be safely imported with no automatic modifications to the module search path -or additions to the builtins. To explicitly trigger the usual site-specific -additions, call the :func:`site.main` function. - -.. versionchanged:: 3.3 - Importing the module used to trigger paths manipulation even when using - :option:`-S`. - -.. index:: - pair: site-packages; directory - -It starts by constructing up to four directories from a head and a tail part. -For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads -are skipped. For the tail part, it uses the empty string and then -:file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh). For each -of the distinct head-tail combinations, it sees if it refers to an existing -directory, and if so, adds it to ``sys.path`` and also inspects the newly -added path for configuration files. - -.. versionchanged:: 3.5 - Support for the "site-python" directory has been removed. - -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "false" -(case-insensitive), the system-level prefixes will still also be -searched for site-packages; otherwise they won't. - -.. index:: - single: # (hash); comment - statement: import - -A path configuration file is a file whose name has the form :file:`{name}.pth` -and exists in one of the four directories mentioned above; its contents are -additional items (one per line) to be added to ``sys.path``. Non-existing items -are never added to ``sys.path``, and no check is made that the item refers to a -directory rather than a file. No item is added to ``sys.path`` more than -once. Blank lines and lines beginning with ``#`` are skipped. Lines starting -with ``import`` (followed by space or tab) are executed. - -.. index:: - single: package - triple: path; configuration; file - -For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to -:file:`/usr/local`. The Python X.Y library is then installed in -:file:`/usr/local/lib/python{X.Y}`. Suppose this has -a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path -configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume -:file:`foo.pth` contains the following:: - - # foo package configuration - - foo - bar - bletch - -and :file:`bar.pth` contains:: - - # bar package configuration - - bar - -Then the following version-specific directories are added to -``sys.path``, in this order:: - - /usr/local/lib/pythonX.Y/site-packages/bar - /usr/local/lib/pythonX.Y/site-packages/foo - -Note that :file:`bletch` is omitted because it doesn't exist; the :file:`bar` -directory precedes the :file:`foo` directory because :file:`bar.pth` comes -alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is -not mentioned in either path configuration file. - -.. index:: module: sitecustomize - -After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. -It is typically created by a system administrator in the site-packages -directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. If Python is started without output streams available, as -with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any exception other -than :exc:`ImportError` causes a silent and perhaps mysterious failure of the -process. - -.. index:: module: usercustomize - -After this, an attempt is made to import a module named :mod:`usercustomize`, -which can perform arbitrary user-specific customizations, if -:data:`ENABLE_USER_SITE` is true. This file is intended to be created in the -user site-packages directory (see below), which is part of ``sys.path`` unless -disabled by :option:`-s`. An :exc:`ImportError` will be silently ignored. - -Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are -empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. - - -.. _rlcompleter-config: - -Readline configuration ----------------------- - -On systems that support :mod:`readline`, this module will also import and -configure the :mod:`rlcompleter` module, if Python is started in -:ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use -:file:`~/.python_history` as the history save file. To disable it, delete (or -override) the :data:`sys.__interactivehook__` attribute in your -:mod:`sitecustomize` or :mod:`usercustomize` module or your -:envvar:`PYTHONSTARTUP` file. - -.. versionchanged:: 3.4 - Activation of rlcompleter and history was made automatic. - - -Module contents ---------------- - -.. data:: PREFIXES - - A list of prefixes for site-packages directories. - - -.. data:: ENABLE_USER_SITE - - Flag showing the status of the user site-packages directory. ``True`` means - that it is enabled and was added to ``sys.path``. ``False`` means that it - was disabled by user request (with :option:`-s` or - :envvar:`PYTHONNOUSERSITE`). ``None`` means it was disabled for security - reasons (mismatch between user or group id and effective id) or by an - administrator. - - -.. data:: USER_SITE - - Path to the user site-packages for the running Python. Can be ``None`` if - :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework Mac - OS X builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for Mac - framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` - on Windows. This directory is a site directory, which means that - :file:`.pth` files in it will be processed. - - -.. data:: USER_BASE - - Path to the base directory for the user site-packages. Can be ``None`` if - :func:`getuserbase` hasn't been called yet. Default value is - :file:`~/.local` for UNIX and Mac OS X non-framework builds, - :file:`~/Library/Python/{X.Y}` for Mac framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to - compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. - See also :envvar:`PYTHONUSERBASE`. - - -.. function:: main() - - Adds all the standard site-specific directories to the module search - path. This function is called automatically when this module is imported, - unless the Python interpreter was started with the :option:`-S` flag. - - .. versionchanged:: 3.3 - This function used to be called unconditionally. - - -.. function:: addsitedir(sitedir, known_paths=None) - - Add a directory to sys.path and process its :file:`.pth` files. Typically - used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). - - -.. function:: getsitepackages() - - Return a list containing all global site-packages directories. - - .. versionadded:: 3.2 - - -.. function:: getuserbase() - - Return the path of the user base directory, :data:`USER_BASE`. If it is not - initialized yet, this function will also set it, respecting - :envvar:`PYTHONUSERBASE`. - - .. versionadded:: 3.2 - - -.. function:: getusersitepackages() - - Return the path of the user-specific site-packages directory, - :data:`USER_SITE`. If it is not initialized yet, this function will also set - it, respecting :envvar:`PYTHONNOUSERSITE` and :data:`USER_BASE`. - - .. versionadded:: 3.2 - - -The :mod:`site` module also provides a way to get the user directories from the -command line: - -.. code-block:: shell-session - - $ python3 -m site --user-site - /home/user/.local/lib/python3.3/site-packages - -.. program:: site - -If it is called without arguments, it will print the contents of -:data:`sys.path` on the standard output, followed by the value of -:data:`USER_BASE` and whether the directory exists, then the same thing for -:data:`USER_SITE`, and finally the value of :data:`ENABLE_USER_SITE`. - -.. cmdoption:: --user-base - - Print the path to the user base directory. - -.. cmdoption:: --user-site - - Print the path to the user site-packages directory. - -If both options are given, user base and user site will be printed (always in -this order), separated by :data:`os.pathsep`. - -If any option is given, the script will exit with one of these values: ``0`` if -the user site-packages directory is enabled, ``1`` if it was disabled by the -user, ``2`` if it is disabled for security reasons or by an administrator, and a -value greater than 2 if there is an error. - -.. seealso:: - - :pep:`370` -- Per user site-packages directory diff --git a/third_party/python/Doc/library/smtpd.rst b/third_party/python/Doc/library/smtpd.rst deleted file mode 100644 index 85ee8a75c..000000000 --- a/third_party/python/Doc/library/smtpd.rst +++ /dev/null @@ -1,277 +0,0 @@ -:mod:`smtpd` --- SMTP Server -============================ - -.. module:: smtpd - :synopsis: A SMTP server implementation in Python. - -.. moduleauthor:: Barry Warsaw -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/smtpd.py` - --------------- - -This module offers several classes to implement SMTP (email) servers. - -.. seealso:: - - The `aiosmtpd `_ package is a recommended - replacement for this module. It is based on :mod:`asyncio` and provides a - more straightforward API. :mod:`smtpd` should be considered deprecated. - -Several server implementations are present; one is a generic -do-nothing implementation, which can be overridden, while the other two offer -specific mail-sending strategies. - -Additionally the SMTPChannel may be extended to implement very specific -interaction behaviour with SMTP clients. - -The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531` -SMTPUTF8 extensions. - - -SMTPServer Objects ------------------- - - -.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=False) - - Create a new :class:`SMTPServer` object, which binds to local address - *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. Both - *localaddr* and *remoteaddr* should be a :ref:`(host, port) ` - tuple. The object inherits from :class:`asyncore.dispatcher`, and so will - insert itself into :mod:`asyncore`'s event loop on instantiation. - - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. - - *map* is the socket map to use for connections (an initially empty - dictionary is a suitable value). If not specified the :mod:`asyncore` - global socket map is used. - - *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. - When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` - command and when present is passed to :meth:`process_message` in the - ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* - cannot be set to ``True`` at the same time. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. When *decode_data* is ``False`` (the - default), the server advertises the ``8BITMIME`` - extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to - the ``MAIL`` command, and when present passes it to :meth:`process_message` - in the ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* - cannot be set to ``True`` at the same time. - - .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) - - Raise a :exc:`NotImplementedError` exception. Override this in subclasses to - do something useful with this message. Whatever was passed in the - constructor as *remoteaddr* will be available as the :attr:`_remoteaddr` - attribute. *peer* is the remote host's address, *mailfrom* is the envelope - originator, *rcpttos* are the envelope recipients and *data* is a string - containing the contents of the e-mail (which should be in :rfc:`5321` - format). - - If the *decode_data* constructor keyword is set to ``True``, the *data* - argument will be a unicode string. If it is set to ``False``, it - will be a bytes object. - - *kwargs* is a dictionary containing additional information. It is empty - if ``decode_data=True`` was given as an init argument, otherwise - it contains the following keys: - - *mail_options*: - a list of all received parameters to the ``MAIL`` - command (the elements are uppercase strings; example: - ``['BODY=8BITMIME', 'SMTPUTF8']``). - - *rcpt_options*: - same as *mail_options* but for the ``RCPT`` command. - Currently no ``RCPT TO`` options are supported, so for now - this will always be an empty list. - - Implementations of ``process_message`` should use the ``**kwargs`` - signature to accept arbitrary keyword arguments, since future feature - enhancements may add keys to the kwargs dictionary. - - Return ``None`` to request a normal ``250 Ok`` response; otherwise - return the desired response string in :RFC:`5321` format. - - .. attribute:: channel_class - - Override this in subclasses to use a custom :class:`SMTPChannel` for - managing SMTP clients. - - .. versionadded:: 3.4 - The *map* constructor argument. - - .. versionchanged:: 3.5 - *localaddr* and *remoteaddr* may now contain IPv6 addresses. - - .. versionadded:: 3.5 - The *decode_data* and *enable_SMTPUTF8* constructor parameters, and the - *kwargs* parameter to :meth:`process_message` when *decode_data* is - ``False``. - - .. versionchanged:: 3.6 - *decode_data* is now ``False`` by default. - - -DebuggingServer Objects ------------------------ - - -.. class:: DebuggingServer(localaddr, remoteaddr) - - Create a new debugging server. Arguments are as per :class:`SMTPServer`. - Messages will be discarded, and printed on stdout. - - -PureProxy Objects ------------------ - - -.. class:: PureProxy(localaddr, remoteaddr) - - Create a new pure proxy server. Arguments are as per :class:`SMTPServer`. - Everything will be relayed to *remoteaddr*. Note that running this has a good - chance to make you into an open relay, so please be careful. - - -MailmanProxy Objects --------------------- - - -.. class:: MailmanProxy(localaddr, remoteaddr) - - Create a new pure proxy server. Arguments are as per :class:`SMTPServer`. - Everything will be relayed to *remoteaddr*, unless local mailman configurations - knows about an address, in which case it will be handled via mailman. Note that - running this has a good chance to make you into an open relay, so please be - careful. - -SMTPChannel Objects -------------------- - -.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=False) - - Create a new :class:`SMTPChannel` object which manages the communication - between the server and a single SMTP client. - - *conn* and *addr* are as per the instance variables described below. - - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. - - *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. - *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same - time. - - A dictionary can be specified in *map* to avoid using a global socket map. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``False``. - *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same - time. - - To use a custom SMTPChannel implementation you need to override the - :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. - - .. versionchanged:: 3.5 - The *decode_data* and *enable_SMTPUTF8* parameters were added. - - .. versionchanged:: 3.6 - *decode_data* is now ``False`` by default. - - The :class:`SMTPChannel` has the following instance variables: - - .. attribute:: smtp_server - - Holds the :class:`SMTPServer` that spawned this channel. - - .. attribute:: conn - - Holds the socket object connecting to the client. - - .. attribute:: addr - - Holds the address of the client, the second value returned by - :func:`socket.accept ` - - .. attribute:: received_lines - - Holds a list of the line strings (decoded using UTF-8) received from - the client. The lines have their ``"\r\n"`` line ending translated to - ``"\n"``. - - .. attribute:: smtp_state - - Holds the current state of the channel. This will be either - :attr:`COMMAND` initially and then :attr:`DATA` after the client sends - a "DATA" line. - - .. attribute:: seen_greeting - - Holds a string containing the greeting sent by the client in its "HELO". - - .. attribute:: mailfrom - - Holds a string containing the address identified in the "MAIL FROM:" line - from the client. - - .. attribute:: rcpttos - - Holds a list of strings containing the addresses identified in the - "RCPT TO:" lines from the client. - - .. attribute:: received_data - - Holds a string containing all of the data sent by the client during the - DATA state, up to but not including the terminating ``"\r\n.\r\n"``. - - .. attribute:: fqdn - - Holds the fully-qualified domain name of the server as returned by - :func:`socket.getfqdn`. - - .. attribute:: peer - - Holds the name of the client peer as returned by ``conn.getpeername()`` - where ``conn`` is :attr:`conn`. - - The :class:`SMTPChannel` operates by invoking methods named ``smtp_`` - upon reception of a command line from the client. Built into the base - :class:`SMTPChannel` class are methods for handling the following commands - (and responding to them appropriately): - - ======== =================================================================== - Command Action taken - ======== =================================================================== - HELO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to base command mode. - EHLO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to extended command mode. - NOOP Takes no action. - QUIT Closes the connection cleanly. - MAIL Accepts the "MAIL FROM:" syntax and stores the supplied address as - :attr:`mailfrom`. In extended command mode, accepts the - :rfc:`1870` SIZE attribute and responds appropriately based on the - value of *data_size_limit*. - RCPT Accepts the "RCPT TO:" syntax and stores the supplied addresses in - the :attr:`rcpttos` list. - RSET Resets the :attr:`mailfrom`, :attr:`rcpttos`, and - :attr:`received_data`, but not the greeting. - DATA Sets the internal state to :attr:`DATA` and stores remaining lines - from the client in :attr:`received_data` until the terminator - ``"\r\n.\r\n"`` is received. - HELP Returns minimal information on command syntax - VRFY Returns code 252 (the server doesn't know if the address is valid) - EXPN Reports that the command is not implemented. - ======== =================================================================== diff --git a/third_party/python/Doc/library/smtplib.rst b/third_party/python/Doc/library/smtplib.rst deleted file mode 100644 index f59c18409..000000000 --- a/third_party/python/Doc/library/smtplib.rst +++ /dev/null @@ -1,586 +0,0 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= - -.. module:: smtplib - :synopsis: SMTP protocol client (requires sockets). - -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/smtplib.py` - -.. index:: - pair: SMTP; protocol - single: Simple Mail Transfer Protocol - --------------- - -The :mod:`smtplib` module defines an SMTP client session object that can be used -to send mail to any Internet machine with an SMTP or ESMTP listener daemon. For -details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer -Protocol) and :rfc:`1869` (SMTP Service Extensions). - - -.. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) - - An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - host and port parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO - command. Otherwise, the local hostname is found using - :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other - than a success code, an :exc:`SMTPConnectError` is raised. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). If the timeout expires, :exc:`socket.timeout` is - raised. The optional source_address parameter allows binding - to some specific source address in a machine with multiple network - interfaces, and/or to some specific source TCP port. It takes a 2-tuple - (host, port), for the socket to bind to as its source address before - connecting. If omitted (or if host or port are ``''`` and/or 0 respectively) - the OS default behavior will be used. - - For normal use, you should only require the initialization/connect, - :meth:`sendmail`, and :meth:`SMTP.quit` methods. - An example is included below. - - The :class:`SMTP` class supports the :keyword:`with` statement. When used - like this, the SMTP ``QUIT`` command is issued automatically when the - :keyword:`with` statement exits. E.g.:: - - >>> from smtplib import SMTP - >>> with SMTP("domain.org") as smtp: - ... smtp.noop() - ... - (250, b'Ok') - >>> - - .. versionchanged:: 3.3 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - source_address argument was added. - - .. versionadded:: 3.5 - The SMTPUTF8 extension (:rfc:`6531`) is now supported. - - -.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ - certfile=None [, timeout], context=None, \ - source_address=None) - - An :class:`SMTP_SSL` instance behaves exactly the same as instances of - :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same - meaning as they do in the :class:`SMTP` class. *context*, also optional, - can contain a :class:`~ssl.SSLContext` and allows configuring various - aspects of the secure connection. Please read :ref:`ssl-security` for - best practices. - - *keyfile* and *certfile* are a legacy alternative to *context*, and can - point to a PEM formatted private key and certificate chain file for the - SSL connection. - - .. versionchanged:: 3.3 - *context* was added. - - .. versionchanged:: 3.3 - source_address argument was added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - -.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None) - - The LMTP protocol, which is very similar to ESMTP, is heavily based on the - standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port - server. The optional arguments local_hostname and source_address have the - same meaning as they do in the :class:`SMTP` class. To specify a Unix - socket, you must use an absolute path for *host*, starting with a '/'. - - Authentication is supported, using the regular SMTP mechanism. When using a - Unix socket, LMTP generally don't support or require any authentication, but - your mileage might vary. - - -A nice selection of exceptions is defined as well: - - -.. exception:: SMTPException - - Subclass of :exc:`OSError` that is the base exception class for all - the other exceptions provided by this module. - - .. versionchanged:: 3.4 - SMTPException became subclass of :exc:`OSError` - - -.. exception:: SMTPServerDisconnected - - This exception is raised when the server unexpectedly disconnects, or when an - attempt is made to use the :class:`SMTP` instance before connecting it to a - server. - - -.. exception:: SMTPResponseException - - Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. - - -.. exception:: SMTPSenderRefused - - Sender address refused. In addition to the attributes set by on all - :exc:`SMTPResponseException` exceptions, this sets 'sender' to the string that - the SMTP server refused. - - -.. exception:: SMTPRecipientsRefused - - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. - - -.. exception:: SMTPDataError - - The SMTP server refused to accept the message data. - - -.. exception:: SMTPConnectError - - Error occurred during establishment of a connection with the server. - - -.. exception:: SMTPHeloError - - The server refused our ``HELO`` message. - - -.. exception:: SMTPNotSupportedError - - The command or option attempted is not supported by the server. - - .. versionadded:: 3.5 - - -.. exception:: SMTPAuthenticationError - - SMTP authentication went wrong. Most probably the server didn't accept the - username/password combination provided. - - -.. seealso:: - - :rfc:`821` - Simple Mail Transfer Protocol - Protocol definition for SMTP. This document covers the model, operating - procedure, and protocol details for SMTP. - - :rfc:`1869` - SMTP Service Extensions - Definition of the ESMTP extensions for SMTP. This describes a framework for - extending SMTP with new commands, supporting dynamic discovery of the commands - provided by the server, and defines a few additional commands. - - -.. _smtp-objects: - -SMTP Objects ------------- - -An :class:`SMTP` instance has the following methods: - - -.. method:: SMTP.set_debuglevel(level) - - Set the debug output level. A value of 1 or ``True`` for *level* results in - debug messages for connection and for all messages sent to and received from - the server. A value of 2 for *level* results in these messages being - timestamped. - - .. versionchanged:: 3.5 Added debuglevel 2. - - -.. method:: SMTP.docmd(cmd, args='') - - Send a command *cmd* to the server. The optional argument *args* is simply - concatenated to the command, separated by a space. - - This returns a 2-tuple composed of a numeric response code and the actual - response line (multiline responses are joined into one long line.) - - In normal operation it should not be necessary to call this method explicitly. - It is used to implement other methods and may be useful for testing private - extensions. - - If the connection to the server is lost while waiting for the reply, - :exc:`SMTPServerDisconnected` will be raised. - - -.. method:: SMTP.connect(host='localhost', port=0) - - Connect to a host on a given port. The defaults are to connect to the local - host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) - followed by a number, that suffix will be stripped off and the number - interpreted as the port number to use. This method is automatically invoked by - the constructor if a host is specified during instantiation. Returns a - 2-tuple of the response code and message sent by the server in its - connection response. - - -.. method:: SMTP.helo(name='') - - Identify yourself to the SMTP server using ``HELO``. The hostname argument - defaults to the fully qualified domain name of the local host. - The message returned by the server is stored as the :attr:`helo_resp` attribute - of the object. - - In normal operation it should not be necessary to call this method explicitly. - It will be implicitly called by the :meth:`sendmail` when necessary. - - -.. method:: SMTP.ehlo(name='') - - Identify yourself to an ESMTP server using ``EHLO``. The hostname argument - defaults to the fully qualified domain name of the local host. Examine the - response for ESMTP option and store them for use by :meth:`has_extn`. - Also sets several informational attributes: the message returned by - the server is stored as the :attr:`ehlo_resp` attribute, :attr:`does_esmtp` - is set to true or false depending on whether the server supports ESMTP, and - :attr:`esmtp_features` will be a dictionary containing the names of the - SMTP service extensions this server supports, and their parameters (if any). - - Unless you wish to use :meth:`has_extn` before sending mail, it should not be - necessary to call this method explicitly. It will be implicitly called by - :meth:`sendmail` when necessary. - -.. method:: SMTP.ehlo_or_helo_if_needed() - - This method calls :meth:`ehlo` and/or :meth:`helo` if there has been no - previous ``EHLO`` or ``HELO`` command this session. It tries ESMTP ``EHLO`` - first. - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - -.. method:: SMTP.has_extn(name) - - Return :const:`True` if *name* is in the set of SMTP service extensions returned - by the server, :const:`False` otherwise. Case is ignored. - - -.. method:: SMTP.verify(address) - - Check the validity of an address on this server using SMTP ``VRFY``. Returns a - tuple consisting of code 250 and a full :rfc:`822` address (including human - name) if the user address is valid. Otherwise returns an SMTP error code of 400 - or greater and an error string. - - .. note:: - - Many sites disable SMTP ``VRFY`` in order to foil spammers. - - -.. method:: SMTP.login(user, password, *, initial_response_ok=True) - - Log in on an SMTP server that requires authentication. The arguments are the - username and the password to authenticate with. If there has been no previous - ``EHLO`` or ``HELO`` command this session, this method tries ESMTP ``EHLO`` - first. This method will return normally if the authentication was successful, or - may raise the following exceptions: - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPAuthenticationError` - The server didn't accept the username/password combination. - - :exc:`SMTPNotSupportedError` - The ``AUTH`` command is not supported by the server. - - :exc:`SMTPException` - No suitable authentication method was found. - - Each of the authentication methods supported by :mod:`smtplib` are tried in - turn if they are advertised as supported by the server. See :meth:`auth` - for a list of supported authentication methods. *initial_response_ok* is - passed through to :meth:`auth`. - - Optional keyword argument *initial_response_ok* specifies whether, for - authentication methods that support it, an "initial response" as specified - in :rfc:`4954` can be sent along with the ``AUTH`` command, rather than - requiring a challenge/response. - - .. versionchanged:: 3.5 - :exc:`SMTPNotSupportedError` may be raised, and the - *initial_response_ok* parameter was added. - - -.. method:: SMTP.auth(mechanism, authobject, *, initial_response_ok=True) - - Issue an ``SMTP`` ``AUTH`` command for the specified authentication - *mechanism*, and handle the challenge response via *authobject*. - - *mechanism* specifies which authentication mechanism is to - be used as argument to the ``AUTH`` command; the valid values are - those listed in the ``auth`` element of :attr:`esmtp_features`. - - *authobject* must be a callable object taking an optional single argument: - - data = authobject(challenge=None) - - If optional keyword argument *initial_response_ok* is true, - ``authobject()`` will be called first with no argument. It can return the - :rfc:`4954` "initial response" ASCII ``str`` which will be encoded and sent with - the ``AUTH`` command as below. If the ``authobject()`` does not support an - initial response (e.g. because it requires a challenge), it should return - ``None`` when called with ``challenge=None``. If *initial_response_ok* is - false, then ``authobject()`` will not be called first with ``None``. - - If the initial response check returns ``None``, or if *initial_response_ok* is - false, ``authobject()`` will be called to process the server's challenge - response; the *challenge* argument it is passed will be a ``bytes``. It - should return ASCII ``str`` *data* that will be base64 encoded and sent to the - server. - - The ``SMTP`` class provides ``authobjects`` for the ``CRAM-MD5``, ``PLAIN``, - and ``LOGIN`` mechanisms; they are named ``SMTP.auth_cram_md5``, - ``SMTP.auth_plain``, and ``SMTP.auth_login`` respectively. They all require - that the ``user`` and ``password`` properties of the ``SMTP`` instance are - set to appropriate values. - - User code does not normally need to call ``auth`` directly, but can instead - call the :meth:`login` method, which will try each of the above mechanisms - in turn, in the order listed. ``auth`` is exposed to facilitate the - implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. - - .. versionadded:: 3.5 - - -.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None) - - Put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP - commands that follow will be encrypted. You should then call :meth:`ehlo` - again. - - If *keyfile* and *certfile* are provided, they are used to create an - :class:`ssl.SSLContext`. - - Optional *context* parameter is an :class:`ssl.SSLContext` object; This is - an alternative to using a keyfile and a certfile and if specified both - *keyfile* and *certfile* should be ``None``. - - If there has been no previous ``EHLO`` or ``HELO`` command this session, - this method tries ESMTP ``EHLO`` first. - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPNotSupportedError` - The server does not support the STARTTLS extension. - - :exc:`RuntimeError` - SSL/TLS support is not available to your Python interpreter. - - .. versionchanged:: 3.3 - *context* was added. - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). - - .. versionchanged:: 3.5 - The error raised for lack of STARTTLS support is now the - :exc:`SMTPNotSupportedError` subclass instead of the base - :exc:`SMTPException`. - - -.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=(), rcpt_options=()) - - Send mail. The required arguments are an :rfc:`822` from-address string, a list - of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 - address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. - ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP - options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) - - .. note:: - - The *from_addr* and *to_addrs* parameters are used to construct the message - envelope used by the transport agents. ``sendmail`` does not modify the - message headers in any way. - - *msg* may be a string containing characters in the ASCII range, or a byte - string. A string is encoded to bytes using the ascii codec, and lone ``\r`` - and ``\n`` characters are converted to ``\r\n`` characters. A byte string is - not modified. - - If there has been no previous ``EHLO`` or ``HELO`` command this session, this - method tries ESMTP ``EHLO`` first. If the server does ESMTP, message size and - each of the specified options will be passed to it (if the option is in the - feature set the server advertises). If ``EHLO`` fails, ``HELO`` will be tried - and ESMTP options suppressed. - - This method will return normally if the mail is accepted for at least one - recipient. Otherwise it will raise an exception. That is, if this method does - not raise an exception, then someone should get your mail. If this method does - not raise an exception, it returns a dictionary, with one entry for each - recipient that was refused. Each entry contains a tuple of the SMTP error code - and the accompanying error message sent by the server. - - If ``SMTPUTF8`` is included in *mail_options*, and the server supports it, - *from_addr* and *to_addrs* may contain non-ASCII characters. - - This method may raise the following exceptions: - - :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPSenderRefused` - The server didn't accept the *from_addr*. - - :exc:`SMTPDataError` - The server replied with an unexpected error code (other than a refusal of a - recipient). - - :exc:`SMTPNotSupportedError` - ``SMTPUTF8`` was given in the *mail_options* but is not supported by the - server. - - Unless otherwise noted, the connection will be open even after an exception is - raised. - - .. versionchanged:: 3.2 - *msg* may be a byte string. - - .. versionchanged:: 3.5 - ``SMTPUTF8`` support added, and :exc:`SMTPNotSupportedError` may be - raised if ``SMTPUTF8`` is specified but the server does not support it. - - -.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \ - mail_options=(), rcpt_options=()) - - This is a convenience method for calling :meth:`sendmail` with the message - represented by an :class:`email.message.Message` object. The arguments have - the same meaning as for :meth:`sendmail`, except that *msg* is a ``Message`` - object. - - If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills - those arguments with addresses extracted from the headers of *msg* as - specified in :rfc:`5322`\: *from_addr* is set to the :mailheader:`Sender` - field if it is present, and otherwise to the :mailheader:`From` field. - *to_addrs* combines the values (if any) of the :mailheader:`To`, - :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*. If exactly one - set of :mailheader:`Resent-*` headers appear in the message, the regular - headers are ignored and the :mailheader:`Resent-*` headers are used instead. - If the message contains more than one set of :mailheader:`Resent-*` headers, - a :exc:`ValueError` is raised, since there is no way to unambiguously detect - the most recent set of :mailheader:`Resent-` headers. - - ``send_message`` serializes *msg* using - :class:`~email.generator.BytesGenerator` with ``\r\n`` as the *linesep*, and - calls :meth:`sendmail` to transmit the resulting message. Regardless of the - values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any - :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear - in *msg*. If any of the addresses in *from_addr* and *to_addrs* contain - non-ASCII characters and the server does not advertise ``SMTPUTF8`` support, - an :exc:`SMTPNotSupported` error is raised. Otherwise the ``Message`` is - serialized with a clone of its :mod:`~email.policy` with the - :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and - ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*. - - .. versionadded:: 3.2 - - .. versionadded:: 3.5 - Support for internationalized addresses (``SMTPUTF8``). - - -.. method:: SMTP.quit() - - Terminate the SMTP session and close the connection. Return the result of - the SMTP ``QUIT`` command. - - -Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, -``RSET``, ``NOOP``, ``MAIL``, ``RCPT``, and ``DATA`` are also supported. -Normally these do not need to be called directly, so they are not documented -here. For details, consult the module code. - - -.. _smtp-example: - -SMTP Example ------------- - -This example prompts the user for addresses needed in the message envelope ('To' -and 'From' addresses), and the message to be delivered. Note that the headers -to be included with the message must be included in the message as entered; this -example doesn't do any processing of the :rfc:`822` headers. In particular, the -'To' and 'From' addresses must be included in the message headers explicitly. :: - - import smtplib - - def prompt(prompt): - return input(prompt).strip() - - fromaddr = prompt("From: ") - toaddrs = prompt("To: ").split() - print("Enter message, end with ^D (Unix) or ^Z (Windows):") - - # Add the From: and To: headers at the start! - msg = ("From: %s\r\nTo: %s\r\n\r\n" - % (fromaddr, ", ".join(toaddrs))) - while True: - try: - line = input() - except EOFError: - break - if not line: - break - msg = msg + line - - print("Message length is", len(msg)) - - server = smtplib.SMTP('localhost') - server.set_debuglevel(1) - server.sendmail(fromaddr, toaddrs, msg) - server.quit() - -.. note:: - - In general, you will want to use the :mod:`email` package's features to - construct an email message, which you can then send - via :meth:`~smtplib.SMTP.send_message`; see :ref:`email-examples`. diff --git a/third_party/python/Doc/library/sndhdr.rst b/third_party/python/Doc/library/sndhdr.rst deleted file mode 100644 index 6bfa9a9fd..000000000 --- a/third_party/python/Doc/library/sndhdr.rst +++ /dev/null @@ -1,51 +0,0 @@ -:mod:`sndhdr` --- Determine type of sound file -============================================== - -.. module:: sndhdr - :synopsis: Determine type of a sound file. - -.. sectionauthor:: Fred L. Drake, Jr. -.. Based on comments in the module source file. - -**Source code:** :source:`Lib/sndhdr.py` - -.. index:: - single: A-LAW - single: u-LAW - --------------- - -The :mod:`sndhdr` provides utility functions which attempt to determine the type -of sound data which is in a file. When these functions are able to determine -what type of sound data is stored in a file, they return a -:func:`~collections.namedtuple`, containing five attributes: (``filetype``, -``framerate``, ``nchannels``, ``nframes``, ``sampwidth``). The value for *type* -indicates the data type and will be one of the strings ``'aifc'``, ``'aiff'``, -``'au'``, ``'hcom'``, ``'sndr'``, ``'sndt'``, ``'voc'``, ``'wav'``, ``'8svx'``, -``'sb'``, ``'ub'``, or ``'ul'``. The *sampling_rate* will be either the actual -value or ``0`` if unknown or difficult to decode. Similarly, *channels* will be -either the number of channels or ``0`` if it cannot be determined or if the -value is difficult to decode. The value for *frames* will be either the number -of frames or ``-1``. The last item in the tuple, *bits_per_sample*, will either -be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW. - - -.. function:: what(filename) - - Determines the type of sound data stored in the file *filename* using - :func:`whathdr`. If it succeeds, returns a namedtuple as described above, otherwise - ``None`` is returned. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - - -.. function:: whathdr(filename) - - Determines the type of sound data stored in a file based on the file header. - The name of the file is given by *filename*. This function returns a namedtuple as - described above on success, or ``None``. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - diff --git a/third_party/python/Doc/library/socket.rst b/third_party/python/Doc/library/socket.rst deleted file mode 100644 index f4e5af9d5..000000000 --- a/third_party/python/Doc/library/socket.rst +++ /dev/null @@ -1,1763 +0,0 @@ -:mod:`socket` --- Low-level networking interface -================================================ - -.. module:: socket - :synopsis: Low-level networking interface. - -**Source code:** :source:`Lib/socket.py` - --------------- - -This module provides access to the BSD *socket* interface. It is available on -all modern Unix systems, Windows, MacOS, and probably additional platforms. - -.. note:: - - Some behavior may be platform dependent, since calls are made to the operating - system socket APIs. - -.. index:: object: socket - -The Python interface is a straightforward transliteration of the Unix system -call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement -the various socket system calls. Parameter types are somewhat higher-level than -in the C interface: as with :meth:`read` and :meth:`write` operations on Python -files, buffer allocation on receive operations is automatic, and buffer length -is implicit on send operations. - - -.. seealso:: - - Module :mod:`socketserver` - Classes that simplify writing network servers. - - Module :mod:`ssl` - A TLS/SSL wrapper for socket objects. - - -Socket families ---------------- - -Depending on the system and the build options, various socket families -are supported by this module. - -The address format required by a particular socket object is automatically -selected based on the address family specified when the socket object was -created. Socket addresses are represented as follows: - -- The address of an :const:`AF_UNIX` socket bound to a file system node - is represented as a string, using the file system encoding and the - ``'surrogateescape'`` error handler (see :pep:`383`). An address in - Linux's abstract namespace is returned as a :term:`bytes-like object` with - an initial null byte; note that sockets in this namespace can - communicate with normal file system sockets, so programs intended to - run on Linux may need to deal with both types of address. A string or - bytes-like object can be used for either type of address when - passing it as an argument. - - .. versionchanged:: 3.3 - Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 - encoding. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - -.. _host_port: - -- A pair ``(host, port)`` is used for the :const:`AF_INET` address family, - where *host* is a string representing either a hostname in Internet domain - notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integer. - - - For IPv4 addresses, two special forms are accepted instead of a host - address: ``''`` represents :const:`INADDR_ANY`, which is used to bind to all - interfaces, and the string ``''`` represents - :const:`INADDR_BROADCAST`. This behavior is not compatible with IPv6, - therefore, you may want to avoid these if you intend to support IPv6 with your - Python programs. - -- For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, - scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` - and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scopeid* can be omitted just for - backward compatibility. Note, however, omission of *scopeid* can cause problems - in manipulating scoped IPv6 addresses. - -- :const:`AF_NETLINK` sockets are represented as pairs ``(pid, groups)``. - -- Linux-only support for TIPC is available using the :const:`AF_TIPC` - address family. TIPC is an open, non-IP based networked protocol designed - for use in clustered computer environments. Addresses are represented by a - tuple, and the fields depend on the address type. The general tuple form is - ``(addr_type, v1, v2, v3 [, scope])``, where: - - - *addr_type* is one of :const:`TIPC_ADDR_NAMESEQ`, :const:`TIPC_ADDR_NAME`, - or :const:`TIPC_ADDR_ID`. - - *scope* is one of :const:`TIPC_ZONE_SCOPE`, :const:`TIPC_CLUSTER_SCOPE`, and - :const:`TIPC_NODE_SCOPE`. - - If *addr_type* is :const:`TIPC_ADDR_NAME`, then *v1* is the server type, *v2* is - the port identifier, and *v3* should be 0. - - If *addr_type* is :const:`TIPC_ADDR_NAMESEQ`, then *v1* is the server type, *v2* - is the lower port number, and *v3* is the upper port number. - - If *addr_type* is :const:`TIPC_ADDR_ID`, then *v1* is the node, *v2* is the - reference, and *v3* should be set to 0. - -- A tuple ``(interface, )`` is used for the :const:`AF_CAN` address family, - where *interface* is a string representing a network interface name like - ``'can0'``. The network interface name ``''`` can be used to receive packets - from all network interfaces of this family. - -- A string or a tuple ``(id, unit)`` is used for the :const:`SYSPROTO_CONTROL` - protocol of the :const:`PF_SYSTEM` family. The string is the name of a - kernel control using a dynamically-assigned ID. The tuple can be used if ID - and unit number of the kernel control are known or if a registered ID is - used. - - .. versionadded:: 3.3 - -- :const:`AF_BLUETOOTH` supports the following protocols and address - formats: - - - :const:`BTPROTO_L2CAP` accepts ``(bdaddr, psm)`` where ``bdaddr`` is - the Bluetooth address as a string and ``psm`` is an integer. - - - :const:`BTPROTO_RFCOMM` accepts ``(bdaddr, channel)`` where ``bdaddr`` - is the Bluetooth address as a string and ``channel`` is an integer. - - - :const:`BTPROTO_HCI` accepts ``(device_id,)`` where ``device_id`` is - either an integer or a string with the Bluetooth address of the - interface. (This depends on your OS; NetBSD and DragonFlyBSD expect - a Bluetooth address while everything else expects an integer.) - - .. versionchanged:: 3.2 - NetBSD and DragonFlyBSD support added. - - - :const:`BTPROTO_SCO` accepts ``bdaddr`` where ``bdaddr`` is a - :class:`bytes` object containing the Bluetooth address in a - string format. (ex. ``b'12:23:34:45:56:67'``) This protocol is not - supported under FreeBSD. - -- :const:`AF_ALG` is a Linux-only socket based interface to Kernel - cryptography. An algorithm socket is configured with a tuple of two to four - elements ``(type, name [, feat [, mask]])``, where: - - - *type* is the algorithm type as string, e.g. ``aead``, ``hash``, - ``skcipher`` or ``rng``. - - - *name* is the algorithm name and operation mode as string, e.g. - ``sha256``, ``hmac(sha256)``, ``cbc(aes)`` or ``drbg_nopr_ctr_aes256``. - - - *feat* and *mask* are unsigned 32bit integers. - - Availability Linux 2.6.38, some algorithm types require more recent Kernels. - - .. versionadded:: 3.6 - -- :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple - ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - - - *ifname* - String specifying the device name. - - *proto* - An in network-byte-order integer specifying the Ethernet - protocol number. - - *pkttype* - Optional integer specifying the packet type: - - - ``PACKET_HOST`` (the default) - Packet addressed to the local host. - - ``PACKET_BROADCAST`` - Physical-layer broadcast packet. - - ``PACKET_MULTIHOST`` - Packet sent to a physical-layer multicast address. - - ``PACKET_OTHERHOST`` - Packet to some other host that has been caught by - a device driver in promiscuous mode. - - ``PACKET_OUTGOING`` - Packet originating from the local host that is - looped back to a packet socket. - - *hatype* - Optional integer specifying the ARP hardware address type. - - *addr* - Optional bytes-like object specifying the hardware physical - address, whose interpretation depends on the device. - -If you use a hostname in the *host* portion of IPv4/v6 socket address, the -program may show a nondeterministic behavior, as Python uses the first address -returned from the DNS resolution. The socket address will be resolved -differently into an actual IPv4/v6 address, depending on the results from DNS -resolution and/or the host configuration. For deterministic behavior use a -numeric address in *host* portion. - -All errors raise exceptions. The normal exceptions for invalid argument types -and out-of-memory conditions can be raised; starting from Python 3.3, errors -related to socket or address semantics raise :exc:`OSError` or one of its -subclasses (they used to raise :exc:`socket.error`). - -Non-blocking mode is supported through :meth:`~socket.setblocking`. A -generalization of this based on timeouts is supported through -:meth:`~socket.settimeout`. - - -Module contents ---------------- - -The module :mod:`socket` exports the following elements. - - -Exceptions -^^^^^^^^^^ - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -.. exception:: herror - - A subclass of :exc:`OSError`, this exception is raised for - address-related errors, i.e. for functions that use *h_errno* in the POSIX - C API, including :func:`gethostbyname_ex` and :func:`gethostbyaddr`. - The accompanying value is a pair ``(h_errno, string)`` representing an - error returned by a library call. *h_errno* is a numeric value, while - *string* represents the description of *h_errno*, as returned by the - :c:func:`hstrerror` C function. - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - -.. exception:: gaierror - - A subclass of :exc:`OSError`, this exception is raised for - address-related errors by :func:`getaddrinfo` and :func:`getnameinfo`. - The accompanying value is a pair ``(error, string)`` representing an error - returned by a library call. *string* represents the description of - *error*, as returned by the :c:func:`gai_strerror` C function. The - numeric *error* value will match one of the :const:`EAI_\*` constants - defined in this module. - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - -.. exception:: timeout - - A subclass of :exc:`OSError`, this exception is raised when a timeout - occurs on a socket which has had timeouts enabled via a prior call to - :meth:`~socket.settimeout` (or implicitly through - :func:`~socket.setdefaulttimeout`). The accompanying value is a string - whose value is currently always "timed out". - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - - -Constants -^^^^^^^^^ - - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. - - .. versionadded:: 3.4 - -.. data:: AF_UNIX - AF_INET - AF_INET6 - - These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not - defined then this protocol is unsupported. More constants may be available - depending on the system. - - -.. data:: SOCK_STREAM - SOCK_DGRAM - SOCK_RAW - SOCK_RDM - SOCK_SEQPACKET - - These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. - (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally - useful.) - -.. data:: SOCK_CLOEXEC - SOCK_NONBLOCK - - These two constants, if defined, can be combined with the socket types and - allow you to set some flags atomically (thus avoiding possible race - conditions and the need for separate calls). - - .. seealso:: - - `Secure File Descriptor Handling `_ - for a more thorough explanation. - - Availability: Linux >= 2.6.27. - - .. versionadded:: 3.2 - -.. data:: SO_* - SOMAXCONN - MSG_* - SOL_* - SCM_* - IPPROTO_* - IPPORT_* - INADDR_* - IP_* - IPV6_* - EAI_* - AI_* - NI_* - TCP_* - - Many constants of these forms, documented in the Unix documentation on sockets - and/or the IP protocol, are also defined in the socket module. They are - generally used in arguments to the :meth:`setsockopt` and :meth:`getsockopt` - methods of socket objects. In most cases, only those symbols that are defined - in the Unix header files are defined; for a few symbols, default values are - provided. - - .. versionchanged:: 3.6 - ``SO_DOMAIN``, ``SO_PROTOCOL``, ``SO_PEERSEC``, ``SO_PASSSEC``, - ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. - - .. versionchanged:: 3.6.5 - On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows - supports. - -.. data:: AF_CAN - PF_CAN - SOL_CAN_* - CAN_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - Availability: Linux >= 2.6.25. - - .. versionadded:: 3.3 - -.. data:: CAN_BCM - CAN_BCM_* - - CAN_BCM, in the CAN protocol family, is the broadcast manager (BCM) protocol. - Broadcast manager constants, documented in the Linux documentation, are also - defined in the socket module. - - Availability: Linux >= 2.6.25. - - .. versionadded:: 3.4 - -.. data:: CAN_RAW_FD_FRAMES - - Enables CAN FD support in a CAN_RAW socket. This is disabled by default. - This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. - - This constant is documented in the Linux documentation. - - Availability: Linux >= 3.6. - - .. versionadded:: 3.5 - - -.. data:: AF_PACKET - PF_PACKET - PACKET_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - Availability: Linux >= 2.2. - - -.. data:: AF_RDS - PF_RDS - SOL_RDS - RDS_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - Availability: Linux >= 2.6.30. - - .. versionadded:: 3.3 - - -.. data:: SIO_RCVALL - SIO_KEEPALIVE_VALS - SIO_LOOPBACK_FAST_PATH - RCVALL_* - - Constants for Windows' WSAIoctl(). The constants are used as arguments to the - :meth:`~socket.socket.ioctl` method of socket objects. - - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. - - -.. data:: TIPC_* - - TIPC related constants, matching the ones exported by the C socket API. See - the TIPC documentation for more information. - -.. data:: AF_ALG - SOL_ALG - ALG_* - - Constants for Linux Kernel cryptography. - - Availability: Linux >= 2.6.38. - - .. versionadded:: 3.6 - -.. data:: AF_LINK - - Availability: BSD, OSX. - - .. versionadded:: 3.4 - -.. data:: has_ipv6 - - This constant contains a boolean value which indicates if IPv6 is supported on - this platform. - -.. data:: BDADDR_ANY - BDADDR_LOCAL - - These are string constants containing Bluetooth addresses with special - meanings. For example, :const:`BDADDR_ANY` can be used to indicate - any address when specifying the binding socket with - :const:`BTPROTO_RFCOMM`. - -.. data:: HCI_FILTER - HCI_TIME_STAMP - HCI_DATA_DIR - - For use with :const:`BTPROTO_HCI`. :const:`HCI_FILTER` is not - available for NetBSD or DragonFlyBSD. :const:`HCI_TIME_STAMP` and - :const:`HCI_DATA_DIR` are not available for FreeBSD, NetBSD, or - DragonFlyBSD. - -Functions -^^^^^^^^^ - -Creating sockets -'''''''''''''''' - -The following functions all create :ref:`socket objects `. - - -.. function:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, or - :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW` or :const:`CAN_BCM`. If *fileno* is - specified, the other arguments are ignored, causing the socket with the - specified file descriptor to return. Unlike :func:`socket.fromfd`, *fileno* - will return the same socket and not a duplicate. This may help close a - detached socket using :meth:`socket.close()`. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - -.. function:: socketpair([family[, type[, proto]]]) - - Build a pair of connected socket objects using the given address family, socket - type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` - if defined on the platform; otherwise, the default is :const:`AF_INET`. - - The newly created sockets are :ref:`non-inheritable `. - - .. versionchanged:: 3.2 - The returned socket objects now support the whole socket API, rather - than a subset. - - .. versionchanged:: 3.4 - The returned sockets are now non-inheritable. - - .. versionchanged:: 3.5 - Windows support added. - - -.. function:: create_connection(address[, timeout[, source_address]]) - - Connect to a TCP service listening on the Internet *address* (a 2-tuple - ``(host, port)``), and return the socket object. This is a higher-level - function than :meth:`socket.connect`: if *host* is a non-numeric hostname, - it will try to resolve it for both :data:`AF_INET` and :data:`AF_INET6`, - and then try to connect to all possible addresses in turn until a - connection succeeds. This makes it easy to write clients that are - compatible to both IPv4 and IPv6. - - Passing the optional *timeout* parameter will set the timeout on the - socket instance before attempting to connect. If no *timeout* is - supplied, the global default timeout setting returned by - :func:`getdefaulttimeout` is used. - - If supplied, *source_address* must be a 2-tuple ``(host, port)`` for the - socket to bind to as its source address before connecting. If host or port - are '' or 0 respectively the OS default behavior will be used. - - .. versionchanged:: 3.2 - *source_address* was added. - - -.. function:: fromfd(fd, family, type, proto=0) - - Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- - subsequent operations on the object may fail if the file descriptor is invalid. - This function is rarely needed, but can be used to get or set socket options on - a socket passed to a program as standard input or output (such as a server - started by the Unix inet daemon). The socket is assumed to be in blocking mode. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - -.. function:: fromshare(data) - - Instantiate a socket from data obtained from the :meth:`socket.share` - method. The socket is assumed to be in blocking mode. - - Availability: Windows. - - .. versionadded:: 3.3 - - -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - -Other functions -''''''''''''''' - -The :mod:`socket` module also offers various network-related services: - - -.. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) - - Translate the *host*/*port* argument into a sequence of 5-tuples that contain - all the necessary arguments for creating a socket connected to that service. - *host* is a domain name, a string representation of an IPv4/v6 address - or ``None``. *port* is a string service name such as ``'http'``, a numeric - port number or ``None``. By passing ``None`` as the value of *host* - and *port*, you can pass ``NULL`` to the underlying C API. - - The *family*, *type* and *proto* arguments can be optionally specified - in order to narrow the list of addresses returned. Passing zero as a - value for each of these arguments selects the full range of results. - The *flags* argument can be one or several of the ``AI_*`` constants, - and will influence how results are computed and returned. - For example, :const:`AI_NUMERICHOST` will disable domain name resolution - and will raise an error if *host* is a domain name. - - The function returns a list of 5-tuples with the following structure: - - ``(family, type, proto, canonname, sockaddr)`` - - In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be - a string representing the canonical name of the *host* if - :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* - will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flow info, scope id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. - - The following example fetches address information for a hypothetical TCP - connection to ``example.org`` on port 80 (results may differ on your - system if IPv6 isn't enabled):: - - >>> socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP) - [(, , - 6, '', ('2606:2800:220:1:248:1893:25c8:1946', 80, 0, 0)), - (, , - 6, '', ('93.184.216.34', 80))] - - .. versionchanged:: 3.2 - parameters can now be passed using keyword arguments. - -.. function:: getfqdn([name]) - - Return a fully qualified domain name for *name*. If *name* is omitted or empty, - it is interpreted as the local host. To find the fully qualified name, the - hostname returned by :func:`gethostbyaddr` is checked, followed by aliases for the - host, if available. The first name which includes a period is selected. In - case no fully qualified domain name is available, the hostname as returned by - :func:`gethostname` is returned. - - -.. function:: gethostbyname(hostname) - - Translate a host name to IPv4 address format. The IPv4 address is returned as a - string, such as ``'100.50.200.5'``. If the host name is an IPv4 address itself - it is returned unchanged. See :func:`gethostbyname_ex` for a more complete - interface. :func:`gethostbyname` does not support IPv6 name resolution, and - :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - - -.. function:: gethostbyname_ex(hostname) - - Translate a host name to IPv4 address format, extended interface. Return a - triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the primary - host name responding to the given *ip_address*, *aliaslist* is a (possibly - empty) list of alternative host names for the same address, and *ipaddrlist* is - a list of IPv4 addresses for the same interface on the same host (often but not - always a single address). :func:`gethostbyname_ex` does not support IPv6 name - resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual - stack support. - - -.. function:: gethostname() - - Return a string containing the hostname of the machine where the Python - interpreter is currently executing. - - Note: :func:`gethostname` doesn't always return the fully qualified domain - name; use :func:`getfqdn` for that. - - -.. function:: gethostbyaddr(ip_address) - - Return a triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the - primary host name responding to the given *ip_address*, *aliaslist* is a - (possibly empty) list of alternative host names for the same address, and - *ipaddrlist* is a list of IPv4/v6 addresses for the same interface on the same - host (most likely containing only a single address). To find the fully qualified - domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports - both IPv4 and IPv6. - - -.. function:: getnameinfo(sockaddr, flags) - - Translate a socket address *sockaddr* into a 2-tuple ``(host, port)``. Depending - on the settings of *flags*, the result can contain a fully-qualified domain name - or numeric address representation in *host*. Similarly, *port* can contain a - string port name or a numeric port number. - - -.. function:: getprotobyname(protocolname) - - Translate an Internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` - function. This is usually only needed for sockets opened in "raw" mode - (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen - automatically if the protocol is omitted or zero. - - -.. function:: getservbyname(servicename[, protocolname]) - - Translate an Internet service name and protocol name to a port number for that - service. The optional protocol name, if given, should be ``'tcp'`` or - ``'udp'``, otherwise any protocol will match. - - -.. function:: getservbyport(port[, protocolname]) - - Translate an Internet port number and protocol name to a service name for that - service. The optional protocol name, if given, should be ``'tcp'`` or - ``'udp'``, otherwise any protocol will match. - - -.. function:: ntohl(x) - - Convert 32-bit positive integers from network to host byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 4-byte swap operation. - - -.. function:: ntohs(x) - - Convert 16-bit positive integers from network to host byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 2-byte swap operation. - - -.. function:: htonl(x) - - Convert 32-bit positive integers from host to network byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 4-byte swap operation. - - -.. function:: htons(x) - - Convert 16-bit positive integers from host to network byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 2-byte swap operation. - - -.. function:: inet_aton(ip_string) - - Convert an IPv4 address from dotted-quad string format (for example, - '123.45.67.89') to 32-bit packed binary format, as a bytes object four characters in - length. This is useful when conversing with a program that uses the standard C - library and needs objects of type :c:type:`struct in_addr`, which is the C type - for the 32-bit packed binary this function returns. - - :func:`inet_aton` also accepts strings with less than three dots; see the - Unix manual page :manpage:`inet(3)` for details. - - If the IPv4 address string passed to this function is invalid, - :exc:`OSError` will be raised. Note that exactly what is valid depends on - the underlying C implementation of :c:func:`inet_aton`. - - :func:`inet_aton` does not support IPv6, and :func:`inet_pton` should be used - instead for IPv4/v6 dual stack support. - - -.. function:: inet_ntoa(packed_ip) - - Convert a 32-bit packed IPv4 address (a :term:`bytes-like object` four - bytes in length) to its standard dotted-quad string representation (for example, - '123.45.67.89'). This is useful when conversing with a program that uses the - standard C library and needs objects of type :c:type:`struct in_addr`, which - is the C type for the 32-bit packed binary data this function takes as an - argument. - - If the byte sequence passed to this function is not exactly 4 bytes in - length, :exc:`OSError` will be raised. :func:`inet_ntoa` does not - support IPv6, and :func:`inet_ntop` should be used instead for IPv4/v6 dual - stack support. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. function:: inet_pton(address_family, ip_string) - - Convert an IP address from its family-specific string format to a packed, - binary format. :func:`inet_pton` is useful when a library or network protocol - calls for an object of type :c:type:`struct in_addr` (similar to - :func:`inet_aton`) or :c:type:`struct in6_addr`. - - Supported values for *address_family* are currently :const:`AF_INET` and - :const:`AF_INET6`. If the IP address string *ip_string* is invalid, - :exc:`OSError` will be raised. Note that exactly what is valid depends on - both the value of *address_family* and the underlying implementation of - :c:func:`inet_pton`. - - Availability: Unix (maybe not all platforms), Windows. - - .. versionchanged:: 3.4 - Windows support added - - -.. function:: inet_ntop(address_family, packed_ip) - - Convert a packed IP address (a :term:`bytes-like object` of some number of - bytes) to its standard, family-specific string representation (for - example, ``'7.10.0.5'`` or ``'5aef:2b::8'``). - :func:`inet_ntop` is useful when a library or network protocol returns an - object of type :c:type:`struct in_addr` (similar to :func:`inet_ntoa`) or - :c:type:`struct in6_addr`. - - Supported values for *address_family* are currently :const:`AF_INET` and - :const:`AF_INET6`. If the bytes object *packed_ip* is not the correct - length for the specified address family, :exc:`ValueError` will be raised. - :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`. - - Availability: Unix (maybe not all platforms), Windows. - - .. versionchanged:: 3.4 - Windows support added - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. - XXX: Are sendmsg(), recvmsg() and CMSG_*() available on any - non-Unix platforms? The old (obsolete?) 4.2BSD form of the - interface, in which struct msghdr has no msg_control or - msg_controllen members, is not currently supported. - -.. function:: CMSG_LEN(length) - - Return the total length, without trailing padding, of an ancillary - data item with associated data of the given *length*. This value - can often be used as the buffer size for :meth:`~socket.recvmsg` to - receive a single item of ancillary data, but :rfc:`3542` requires - portable applications to use :func:`CMSG_SPACE` and thus include - space for padding, even when the item will be the last in the - buffer. Raises :exc:`OverflowError` if *length* is outside the - permissible range of values. - - Availability: most Unix platforms, possibly others. - - .. versionadded:: 3.3 - - -.. function:: CMSG_SPACE(length) - - Return the buffer size needed for :meth:`~socket.recvmsg` to - receive an ancillary data item with associated data of the given - *length*, along with any trailing padding. The buffer space needed - to receive multiple items is the sum of the :func:`CMSG_SPACE` - values for their associated data lengths. Raises - :exc:`OverflowError` if *length* is outside the permissible range - of values. - - Note that some systems might support ancillary data without - providing this function. Also note that setting the buffer size - using the results of this function may not precisely limit the - amount of ancillary data that can be received, since additional - data may be able to fit into the padding area. - - Availability: most Unix platforms, possibly others. - - .. versionadded:: 3.3 - - -.. function:: getdefaulttimeout() - - Return the default timeout in seconds (float) for new socket objects. A value - of ``None`` indicates that new socket objects have no timeout. When the socket - module is first imported, the default is ``None``. - - -.. function:: setdefaulttimeout(timeout) - - Set the default timeout in seconds (float) for new socket objects. When - the socket module is first imported, the default is ``None``. See - :meth:`~socket.settimeout` for possible values and their respective - meanings. - - -.. function:: sethostname(name) - - Set the machine's hostname to *name*. This will raise an - :exc:`OSError` if you don't have enough rights. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: if_nameindex() - - Return a list of network interface information - (index int, name string) tuples. - :exc:`OSError` if the system call fails. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: if_nametoindex(if_name) - - Return a network interface index number corresponding to an - interface name. - :exc:`OSError` if no interface with the given name exists. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: if_indextoname(if_index) - - Return a network interface name corresponding to an - interface index number. - :exc:`OSError` if no interface with the given index exists. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. _socket-objects: - -Socket Objects --------------- - -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. - -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. - - -.. method:: socket.accept() - - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.bind(address) - - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) - - -.. method:: socket.close() - - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile()` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). - - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. - - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - - .. note:: - - :meth:`close()` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown()` - before :meth:`close()`. - - -.. method:: socket.connect(address) - - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) - - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`socket.timeout` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). - - -.. method:: socket.connect_ex(address) - - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - - -.. method:: socket.detach() - - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. - - .. versionadded:: 3.2 - - -.. method:: socket.dup() - - Duplicate the socket. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - -.. method:: socket.fileno() - - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. - -.. method:: socket.get_inheritable() - - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. - - .. versionadded:: 3.4 - - -.. method:: socket.getpeername() - - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. - - -.. method:: socket.getsockname() - - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) - - -.. method:: socket.getsockopt(level, optname[, buflen]) - - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:const:`SO_\*` etc.) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - - -.. method:: socket.gettimeout() - - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. - - -.. method:: socket.ioctl(control, option) - - :platform: Windows - - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. - - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. - - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. - -.. method:: socket.listen([backlog]) - - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. - -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) - - .. index:: single: I/O control; buffering - - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. - - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. - - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. - - .. note:: - - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. - - -.. method:: socket.recv(bufsize[, flags]) - - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. - - .. note:: - - For best match with hardware and network realities, the value of *bufsize* - should be a relatively small power of 2, for example, 4096. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvfrom(bufsize[, flags]) - - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:type:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS): - # Append data, ignoring any truncated integers at the end. - fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - Availability: most Unix platforms, possibly others. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - Availability: most Unix platforms, possibly others. - - .. versionadded:: 3.3 - - -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) - - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) - - -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) - - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. - - -.. method:: socket.send(bytes[, flags]) - - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendall(bytes[, flags]) - - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) - - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - - Availability: most Unix platforms, possibly others. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. - - Availability: Linux >= 2.6.38 - - .. versionadded:: 3.6 - -.. method:: socket.sendfile(file, offset=0, count=None) - - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. - - .. versionadded:: 3.5 - -.. method:: socket.set_inheritable(inheritable) - - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - - .. versionadded:: 3.4 - - -.. method:: socket.setblocking(flag) - - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. - - This method is a shorthand for certain :meth:`~socket.settimeout` calls: - - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - - -.. method:: socket.settimeout(value) - - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating point number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. - - For further information, please consult the :ref:`notes on socket timeouts `. - - -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) -.. method:: socket.setsockopt(level, optname, None, optlen: int) - - .. index:: module: struct - - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When value is set to ``None``, - optlen argument is required. It's equivalent to call setsockopt C - function with optval=NULL and optlen=optlen. - - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - - -.. method:: socket.shutdown(how) - - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - - -.. method:: socket.share(process_id) - - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. - - Availability: Windows. - - .. versionadded:: 3.3 - - -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. - -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. - - -.. attribute:: socket.family - - The socket family. - - -.. attribute:: socket.type - - The socket type. - - -.. attribute:: socket.proto - - The socket protocol. - - - -.. _socket-timeouts: - -Notes on socket timeouts ------------------------- - -A socket object can be in one of three modes: blocking, non-blocking, or -timeout. Sockets are by default always created in blocking mode, but this -can be changed by calling :func:`setdefaulttimeout`. - -* In *blocking mode*, operations block until complete or the system returns - an error (such as connection timed out). - -* In *non-blocking mode*, operations fail (with an error that is unfortunately - system-dependent) if they cannot be completed immediately: functions from the - :mod:`select` can be used to know when and whether a socket is available for - reading or writing. - -* In *timeout mode*, operations fail if they cannot be completed within the - timeout specified for the socket (they raise a :exc:`timeout` exception) - or if the system returns an error. - -.. note:: - At the operating system level, sockets in *timeout mode* are internally set - in non-blocking mode. Also, the blocking and timeout modes are shared between - file descriptors and socket objects that refer to the same network endpoint. - This implementation detail can have visible consequences if e.g. you decide - to use the :meth:`~socket.fileno()` of a socket. - -Timeouts and the ``connect`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~socket.connect` operation is also subject to the timeout -setting, and in general it is recommended to call :meth:`~socket.settimeout` -before calling :meth:`~socket.connect` or pass a timeout parameter to -:meth:`create_connection`. However, the system network stack may also -return a connection timeout error of its own regardless of any Python socket -timeout setting. - -Timeouts and the ``accept`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If :func:`getdefaulttimeout` is not :const:`None`, sockets returned by -the :meth:`~socket.accept` method inherit that timeout. Otherwise, the -behaviour depends on settings of the listening socket: - -* if the listening socket is in *blocking mode* or in *timeout mode*, - the socket returned by :meth:`~socket.accept` is in *blocking mode*; - -* if the listening socket is in *non-blocking mode*, whether the socket - returned by :meth:`~socket.accept` is in blocking or non-blocking mode - is operating system-dependent. If you want to ensure cross-platform - behaviour, it is recommended you manually override this setting. - - -.. _socket-example: - -Example -------- - -Here are four minimal example programs using the TCP/IP protocol: a server that -echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, -:meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly -repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also -note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on -the socket it is listening on but on the new socket returned by -:meth:`~socket.accept`. - -The first two examples support IPv4 only. :: - - # Echo server program - import socket - - HOST = '' # Symbolic name meaning all available interfaces - PORT = 50007 # Arbitrary non-privileged port - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind((HOST, PORT)) - s.listen(1) - conn, addr = s.accept() - with conn: - print('Connected by', addr) - while True: - data = conn.recv(1024) - if not data: break - conn.sendall(data) - -:: - - # Echo client program - import socket - - HOST = 'daring.cwi.nl' # The remote host - PORT = 50007 # The same port as used by the server - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((HOST, PORT)) - s.sendall(b'Hello, world') - data = s.recv(1024) - print('Received', repr(data)) - -The next two examples are identical to the above two, but support both IPv4 and -IPv6. The server side will listen to the first address family available (it -should listen to both instead). On most of IPv6-ready systems, IPv6 will take -precedence and the server may not accept IPv4 traffic. The client side will try -to connect to the all addresses returned as a result of the name resolution, and -sends traffic to the first one connected successfully. :: - - # Echo server program - import socket - import sys - - HOST = None # Symbolic name meaning all available interfaces - PORT = 50007 # Arbitrary non-privileged port - s = None - for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE): - af, socktype, proto, canonname, sa = res - try: - s = socket.socket(af, socktype, proto) - except OSError as msg: - s = None - continue - try: - s.bind(sa) - s.listen(1) - except OSError as msg: - s.close() - s = None - continue - break - if s is None: - print('could not open socket') - sys.exit(1) - conn, addr = s.accept() - with conn: - print('Connected by', addr) - while True: - data = conn.recv(1024) - if not data: break - conn.send(data) - -:: - - # Echo client program - import socket - import sys - - HOST = 'daring.cwi.nl' # The remote host - PORT = 50007 # The same port as used by the server - s = None - for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - try: - s = socket.socket(af, socktype, proto) - except OSError as msg: - s = None - continue - try: - s.connect(sa) - except OSError as msg: - s.close() - s = None - continue - break - if s is None: - print('could not open socket') - sys.exit(1) - with s: - s.sendall(b'Hello, world') - data = s.recv(1024) - print('Received', repr(data)) - - -The next example shows how to write a very simple network sniffer with raw -sockets on Windows. The example requires administrator privileges to modify -the interface:: - - import socket - - # the public network interface - HOST = socket.gethostbyname(socket.gethostname()) - - # create a raw socket and bind it to the public interface - s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_IP) - s.bind((HOST, 0)) - - # Include IP headers - s.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) - - # receive all packages - s.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON) - - # receive a package - print(s.recvfrom(65565)) - - # disabled promiscuous mode - s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF) - -The last example shows how to use the socket interface to communicate to a CAN -network using the raw socket protocol. To use CAN with the broadcast -manager protocol instead, open a socket with:: - - socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) - -After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you -can use the :meth:`socket.send`, and the :meth:`socket.recv` operations (and -their counterparts) on the socket object as usual. - -This example might require special privileges:: - - import socket - import struct - - - # CAN frame packing/unpacking (see 'struct can_frame' in ) - - can_frame_fmt = "=IB3x8s" - can_frame_size = struct.calcsize(can_frame_fmt) - - def build_can_frame(can_id, data): - can_dlc = len(data) - data = data.ljust(8, b'\x00') - return struct.pack(can_frame_fmt, can_id, can_dlc, data) - - def dissect_can_frame(frame): - can_id, can_dlc, data = struct.unpack(can_frame_fmt, frame) - return (can_id, can_dlc, data[:can_dlc]) - - - # create a raw socket and bind it to the 'vcan0' interface - s = socket.socket(socket.AF_CAN, socket.SOCK_RAW, socket.CAN_RAW) - s.bind(('vcan0',)) - - while True: - cf, addr = s.recvfrom(can_frame_size) - - print('Received: can_id=%x, can_dlc=%x, data=%s' % dissect_can_frame(cf)) - - try: - s.send(cf) - except OSError: - print('Error sending CAN frame') - - try: - s.send(build_can_frame(0x01, b'\x01\x02\x03')) - except OSError: - print('Error sending CAN frame') - -Running an example several times with too small delay between executions, could -lead to this error:: - - OSError: [Errno 98] Address already in use - -This is because the previous execution has left the socket in a ``TIME_WAIT`` -state, and can't be immediately reused. - -There is a :mod:`socket` flag to set, in order to prevent this, -:data:`socket.SO_REUSEADDR`:: - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind((HOST, PORT)) - -the :data:`SO_REUSEADDR` flag tells the kernel to reuse a local socket in -``TIME_WAIT`` state, without waiting for its natural timeout to expire. - - -.. seealso:: - - For an introduction to socket programming (in C), see the following papers: - - - *An Introductory 4.3BSD Interprocess Communication Tutorial*, by Stuart Sechrest - - - *An Advanced 4.3BSD Interprocess Communication Tutorial*, by Samuel J. Leffler et - al, - - both in the UNIX Programmer's Manual, Supplementary Documents 1 (sections - PS1:7 and PS1:8). The platform-specific reference material for the various - socket-related system calls are also a valuable source of information on the - details of socket semantics. For Unix, refer to the manual pages; for Windows, - see the WinSock (or Winsock 2) specification. For IPv6-ready APIs, readers may - want to refer to :rfc:`3493` titled Basic Socket Interface Extensions for IPv6. diff --git a/third_party/python/Doc/library/socketserver.rst b/third_party/python/Doc/library/socketserver.rst deleted file mode 100644 index e12c8c978..000000000 --- a/third_party/python/Doc/library/socketserver.rst +++ /dev/null @@ -1,640 +0,0 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= - -.. module:: socketserver - :synopsis: A framework for network servers. - -**Source code:** :source:`Lib/socketserver.py` - --------------- - -The :mod:`socketserver` module simplifies the task of writing network servers. - -There are four basic concrete server classes: - - -.. class:: TCPServer(server_address, RequestHandlerClass, bind_and_activate=True) - - This uses the Internet TCP protocol, which provides for - continuous streams of data between the client and server. - If *bind_and_activate* is true, the constructor automatically attempts to - invoke :meth:`~BaseServer.server_bind` and - :meth:`~BaseServer.server_activate`. The other parameters are passed to - the :class:`BaseServer` base class. - - -.. class:: UDPServer(server_address, RequestHandlerClass, bind_and_activate=True) - - This uses datagrams, which are discrete packets of information that may - arrive out of order or be lost while in transit. The parameters are - the same as for :class:`TCPServer`. - - -.. class:: UnixStreamServer(server_address, RequestHandlerClass, bind_and_activate=True) - UnixDatagramServer(server_address, RequestHandlerClass, bind_and_activate=True) - - These more infrequently used classes are similar to the TCP and - UDP classes, but use Unix domain sockets; they're not available on - non-Unix platforms. The parameters are the same as for - :class:`TCPServer`. - - -These four classes process requests :dfn:`synchronously`; each request must be -completed before the next request can be started. This isn't suitable if each -request takes a long time to complete, because it requires a lot of computation, -or because it returns a lot of data which the client is slow to process. The -solution is to create a separate process or thread to handle each request; the -:class:`ForkingMixIn` and :class:`ThreadingMixIn` mix-in classes can be used to -support asynchronous behaviour. - -Creating a server requires several steps. First, you must create a request -handler class by subclassing the :class:`BaseRequestHandler` class and -overriding its :meth:`~BaseRequestHandler.handle` method; -this method will process incoming -requests. Second, you must instantiate one of the server classes, passing it -the server's address and the request handler class. It is recommended to use -the server in a :keyword:`with` statement. Then call the -:meth:`~BaseServer.handle_request` or -:meth:`~BaseServer.serve_forever` method of the server object to -process one or many requests. Finally, call :meth:`~BaseServer.server_close` -to close the socket (unless you used a :keyword:`with` statement). - -When inheriting from :class:`ThreadingMixIn` for threaded connection behavior, -you should explicitly declare how you want your threads to behave on an abrupt -shutdown. The :class:`ThreadingMixIn` class defines an attribute -*daemon_threads*, which indicates whether or not the server should wait for -thread termination. You should set the flag explicitly if you would like -threads to behave autonomously; the default is :const:`False`, meaning that -Python will not exit until all threads created by :class:`ThreadingMixIn` have -exited. - -Server classes have the same external methods and attributes, no matter what -network protocol they use. - - -Server Creation Notes ---------------------- - -There are five classes in an inheritance diagram, four of which represent -synchronous servers of four types:: - - +------------+ - | BaseServer | - +------------+ - | - v - +-----------+ +------------------+ - | TCPServer |------->| UnixStreamServer | - +-----------+ +------------------+ - | - v - +-----------+ +--------------------+ - | UDPServer |------->| UnixDatagramServer | - +-----------+ +--------------------+ - -Note that :class:`UnixDatagramServer` derives from :class:`UDPServer`, not from -:class:`UnixStreamServer` --- the only difference between an IP and a Unix -stream server is the address family, which is simply repeated in both Unix -server classes. - - -.. class:: ForkingMixIn - ThreadingMixIn - - Forking and threading versions of each type of server can be created - using these mix-in classes. For instance, :class:`ThreadingUDPServer` - is created as follows:: - - class ThreadingUDPServer(ThreadingMixIn, UDPServer): - pass - - The mix-in class comes first, since it overrides a method defined in - :class:`UDPServer`. Setting the various attributes also changes the - behavior of the underlying server mechanism. - - :class:`ForkingMixIn` and the Forking classes mentioned below are - only available on POSIX platforms that support :func:`~os.fork`. - -.. class:: ForkingTCPServer - ForkingUDPServer - ThreadingTCPServer - ThreadingUDPServer - - These classes are pre-defined using the mix-in classes. - - -To implement a service, you must derive a class from :class:`BaseRequestHandler` -and redefine its :meth:`~BaseRequestHandler.handle` method. -You can then run various versions of -the service by combining one of the server classes with your request handler -class. The request handler class must be different for datagram or stream -services. This can be hidden by using the handler subclasses -:class:`StreamRequestHandler` or :class:`DatagramRequestHandler`. - -Of course, you still have to use your head! For instance, it makes no sense to -use a forking server if the service contains state in memory that can be -modified by different requests, since the modifications in the child process -would never reach the initial state kept in the parent process and passed to -each child. In this case, you can use a threading server, but you will probably -have to use locks to protect the integrity of the shared data. - -On the other hand, if you are building an HTTP server where all data is stored -externally (for instance, in the file system), a synchronous class will -essentially render the service "deaf" while one request is being handled -- -which may be for a very long time if a client is slow to receive all the data it -has requested. Here a threading or forking server is appropriate. - -In some cases, it may be appropriate to process part of a request synchronously, -but to finish processing in a forked child depending on the request data. This -can be implemented by using a synchronous server and doing an explicit fork in -the request handler class :meth:`~BaseRequestHandler.handle` method. - -Another approach to handling multiple simultaneous requests in an environment -that supports neither threads nor :func:`~os.fork` (or where these are too -expensive or inappropriate for the service) is to maintain an explicit table of -partially finished requests and to use :mod:`selectors` to decide which -request to work on next (or whether to handle a new incoming request). This is -particularly important for stream services where each client can potentially be -connected for a long time (if threads or subprocesses cannot be used). See -:mod:`asyncore` for another way to manage this. - -.. XXX should data and methods be intermingled, or separate? - how should the distinction between class and instance variables be drawn? - - -Server Objects --------------- - -.. class:: BaseServer(server_address, RequestHandlerClass) - - This is the superclass of all Server objects in the module. It defines the - interface, given below, but does not implement most of the methods, which is - done in subclasses. The two parameters are stored in the respective - :attr:`server_address` and :attr:`RequestHandlerClass` attributes. - - - .. method:: fileno() - - Return an integer file descriptor for the socket on which the server is - listening. This function is most commonly passed to :mod:`selectors`, to - allow monitoring multiple servers in the same process. - - - .. method:: handle_request() - - Process a single request. This function calls the following methods in - order: :meth:`get_request`, :meth:`verify_request`, and - :meth:`process_request`. If the user-provided - :meth:`~BaseRequestHandler.handle` method of the - handler class raises an exception, the server's :meth:`handle_error` method - will be called. If no request is received within :attr:`timeout` - seconds, :meth:`handle_timeout` will be called and :meth:`handle_request` - will return. - - - .. method:: serve_forever(poll_interval=0.5) - - Handle requests until an explicit :meth:`shutdown` request. Poll for - shutdown every *poll_interval* seconds. - Ignores the :attr:`timeout` attribute. It - also calls :meth:`service_actions`, which may be used by a subclass or mixin - to provide actions specific to a given service. For example, the - :class:`ForkingMixIn` class uses :meth:`service_actions` to clean up zombie - child processes. - - .. versionchanged:: 3.3 - Added ``service_actions`` call to the ``serve_forever`` method. - - - .. method:: service_actions() - - This is called in the :meth:`serve_forever` loop. This method can be - overridden by subclasses or mixin classes to perform actions specific to - a given service, such as cleanup actions. - - .. versionadded:: 3.3 - - .. method:: shutdown() - - Tell the :meth:`serve_forever` loop to stop and wait until it does. - - - .. method:: server_close() - - Clean up the server. May be overridden. - - - .. attribute:: address_family - - The family of protocols to which the server's socket belongs. - Common examples are :const:`socket.AF_INET` and :const:`socket.AF_UNIX`. - - - .. attribute:: RequestHandlerClass - - The user-provided request handler class; an instance of this class is created - for each request. - - - .. attribute:: server_address - - The address on which the server is listening. The format of addresses varies - depending on the protocol family; - see the documentation for the :mod:`socket` module - for details. For Internet protocols, this is a tuple containing a string giving - the address, and an integer port number: ``('127.0.0.1', 80)``, for example. - - - .. attribute:: socket - - The socket object on which the server will listen for incoming requests. - - - The server classes support the following class variables: - - .. XXX should class variables be covered before instance variables, or vice versa? - - .. attribute:: allow_reuse_address - - Whether the server will allow the reuse of an address. This defaults to - :const:`False`, and can be set in subclasses to change the policy. - - - .. attribute:: request_queue_size - - The size of the request queue. If it takes a long time to process a single - request, any requests that arrive while the server is busy are placed into a - queue, up to :attr:`request_queue_size` requests. Once the queue is full, - further requests from clients will get a "Connection denied" error. The default - value is usually 5, but this can be overridden by subclasses. - - - .. attribute:: socket_type - - The type of socket used by the server; :const:`socket.SOCK_STREAM` and - :const:`socket.SOCK_DGRAM` are two common values. - - - .. attribute:: timeout - - Timeout duration, measured in seconds, or :const:`None` if no timeout is - desired. If :meth:`handle_request` receives no incoming requests within the - timeout period, the :meth:`handle_timeout` method is called. - - - There are various server methods that can be overridden by subclasses of base - server classes like :class:`TCPServer`; these methods aren't useful to external - users of the server object. - - .. XXX should the default implementations of these be documented, or should - it be assumed that the user will look at socketserver.py? - - .. method:: finish_request(request, client_address) - - Actually processes the request by instantiating :attr:`RequestHandlerClass` and - calling its :meth:`~BaseRequestHandler.handle` method. - - - .. method:: get_request() - - Must accept a request from the socket, and return a 2-tuple containing the *new* - socket object to be used to communicate with the client, and the client's - address. - - - .. method:: handle_error(request, client_address) - - This function is called if the :meth:`~BaseRequestHandler.handle` - method of a :attr:`RequestHandlerClass` instance raises - an exception. The default action is to print the traceback to - standard error and continue handling further requests. - - .. versionchanged:: 3.6 - Now only called for exceptions derived from the :exc:`Exception` - class. - - - .. method:: handle_timeout() - - This function is called when the :attr:`timeout` attribute has been set to a - value other than :const:`None` and the timeout period has passed with no - requests being received. The default action for forking servers is - to collect the status of any child processes that have exited, while - in threading servers this method does nothing. - - - .. method:: process_request(request, client_address) - - Calls :meth:`finish_request` to create an instance of the - :attr:`RequestHandlerClass`. If desired, this function can create a new process - or thread to handle the request; the :class:`ForkingMixIn` and - :class:`ThreadingMixIn` classes do this. - - - .. Is there any point in documenting the following two functions? - What would the purpose of overriding them be: initializing server - instance variables, adding new network families? - - .. method:: server_activate() - - Called by the server's constructor to activate the server. The default behavior - for a TCP server just invokes :meth:`~socket.socket.listen` - on the server's socket. May be overridden. - - - .. method:: server_bind() - - Called by the server's constructor to bind the socket to the desired address. - May be overridden. - - - .. method:: verify_request(request, client_address) - - Must return a Boolean value; if the value is :const:`True`, the request will - be processed, and if it's :const:`False`, the request will be denied. This - function can be overridden to implement access controls for a server. The - default implementation always returns :const:`True`. - - - .. versionchanged:: 3.6 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`server_close`. - - -Request Handler Objects ------------------------ - -.. class:: BaseRequestHandler - - This is the superclass of all request handler objects. It defines - the interface, given below. A concrete request handler subclass must - define a new :meth:`handle` method, and can override any of - the other methods. A new instance of the subclass is created for each - request. - - - .. method:: setup() - - Called before the :meth:`handle` method to perform any initialization actions - required. The default implementation does nothing. - - - .. method:: handle() - - This function must do all the work required to service a request. The - default implementation does nothing. Several instance attributes are - available to it; the request is available as :attr:`self.request`; the client - address as :attr:`self.client_address`; and the server instance as - :attr:`self.server`, in case it needs access to per-server information. - - The type of :attr:`self.request` is different for datagram or stream - services. For stream services, :attr:`self.request` is a socket object; for - datagram services, :attr:`self.request` is a pair of string and socket. - - - .. method:: finish() - - Called after the :meth:`handle` method to perform any clean-up actions - required. The default implementation does nothing. If :meth:`setup` - raises an exception, this function will not be called. - - -.. class:: StreamRequestHandler - DatagramRequestHandler - - These :class:`BaseRequestHandler` subclasses override the - :meth:`~BaseRequestHandler.setup` and :meth:`~BaseRequestHandler.finish` - methods, and provide :attr:`self.rfile` and :attr:`self.wfile` attributes. - The :attr:`self.rfile` and :attr:`self.wfile` attributes can be - read or written, respectively, to get the request data or return data - to the client. - - The :attr:`rfile` attributes of both classes support the - :class:`io.BufferedIOBase` readable interface, and - :attr:`DatagramRequestHandler.wfile` supports the - :class:`io.BufferedIOBase` writable interface. - - .. versionchanged:: 3.6 - :attr:`StreamRequestHandler.wfile` also supports the - :class:`io.BufferedIOBase` writable interface. - - -Examples --------- - -:class:`socketserver.TCPServer` Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the server side:: - - import socketserver - - class MyTCPHandler(socketserver.BaseRequestHandler): - """ - The request handler class for our server. - - It is instantiated once per connection to the server, and must - override the handle() method to implement communication to the - client. - """ - - def handle(self): - # self.request is the TCP socket connected to the client - self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) - print(self.data) - # just send back the same data, but upper-cased - self.request.sendall(self.data.upper()) - - if __name__ == "__main__": - HOST, PORT = "localhost", 9999 - - # Create the server, binding to localhost on port 9999 - with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: - # Activate the server; this will keep running until you - # interrupt the program with Ctrl-C - server.serve_forever() - -An alternative request handler class that makes use of streams (file-like -objects that simplify communication by providing the standard file interface):: - - class MyTCPHandler(socketserver.StreamRequestHandler): - - def handle(self): - # self.rfile is a file-like object created by the handler; - # we can now use e.g. readline() instead of raw recv() calls - self.data = self.rfile.readline().strip() - print("{} wrote:".format(self.client_address[0])) - print(self.data) - # Likewise, self.wfile is a file-like object used to write back - # to the client - self.wfile.write(self.data.upper()) - -The difference is that the ``readline()`` call in the second handler will call -``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. - - -This is the client side:: - - import socket - import sys - - HOST, PORT = "localhost", 9999 - data = " ".join(sys.argv[1:]) - - # Create a socket (SOCK_STREAM means a TCP socket) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - # Connect to server and send data - sock.connect((HOST, PORT)) - sock.sendall(bytes(data + "\n", "utf-8")) - - # Receive data from the server and shut down - received = str(sock.recv(1024), "utf-8") - - print("Sent: {}".format(data)) - print("Received: {}".format(received)) - - -The output of the example should look something like this: - -Server: - -.. code-block:: shell-session - - $ python TCPServer.py - 127.0.0.1 wrote: - b'hello world with TCP' - 127.0.0.1 wrote: - b'python is nice' - -Client: - -.. code-block:: shell-session - - $ python TCPClient.py hello world with TCP - Sent: hello world with TCP - Received: HELLO WORLD WITH TCP - $ python TCPClient.py python is nice - Sent: python is nice - Received: PYTHON IS NICE - - -:class:`socketserver.UDPServer` Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the server side:: - - import socketserver - - class MyUDPHandler(socketserver.BaseRequestHandler): - """ - This class works similar to the TCP handler class, except that - self.request consists of a pair of data and client socket, and since - there is no connection the client address must be given explicitly - when sending data back via sendto(). - """ - - def handle(self): - data = self.request[0].strip() - socket = self.request[1] - print("{} wrote:".format(self.client_address[0])) - print(data) - socket.sendto(data.upper(), self.client_address) - - if __name__ == "__main__": - HOST, PORT = "localhost", 9999 - with socketserver.UDPServer((HOST, PORT), MyUDPHandler) as server: - server.serve_forever() - -This is the client side:: - - import socket - import sys - - HOST, PORT = "localhost", 9999 - data = " ".join(sys.argv[1:]) - - # SOCK_DGRAM is the socket type to use for UDP sockets - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - - # As you can see, there is no connect() call; UDP has no connections. - # Instead, data is directly sent to the recipient via sendto(). - sock.sendto(bytes(data + "\n", "utf-8"), (HOST, PORT)) - received = str(sock.recv(1024), "utf-8") - - print("Sent: {}".format(data)) - print("Received: {}".format(received)) - -The output of the example should look exactly like for the TCP server example. - - -Asynchronous Mixins -~~~~~~~~~~~~~~~~~~~ - -To build asynchronous handlers, use the :class:`ThreadingMixIn` and -:class:`ForkingMixIn` classes. - -An example for the :class:`ThreadingMixIn` class:: - - import socket - import threading - import socketserver - - class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler): - - def handle(self): - data = str(self.request.recv(1024), 'ascii') - cur_thread = threading.current_thread() - response = bytes("{}: {}".format(cur_thread.name, data), 'ascii') - self.request.sendall(response) - - class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - pass - - def client(ip, port, message): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.connect((ip, port)) - sock.sendall(bytes(message, 'ascii')) - response = str(sock.recv(1024), 'ascii') - print("Received: {}".format(response)) - - if __name__ == "__main__": - # Port 0 means to select an arbitrary unused port - HOST, PORT = "localhost", 0 - - server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler) - with server: - ip, port = server.server_address - - # Start a thread with the server -- that thread will then start one - # more thread for each request - server_thread = threading.Thread(target=server.serve_forever) - # Exit the server thread when the main thread terminates - server_thread.daemon = True - server_thread.start() - print("Server loop running in thread:", server_thread.name) - - client(ip, port, "Hello World 1") - client(ip, port, "Hello World 2") - client(ip, port, "Hello World 3") - - server.shutdown() - - -The output of the example should look something like this: - -.. code-block:: shell-session - - $ python ThreadedTCPServer.py - Server loop running in thread: Thread-1 - Received: Thread-2: Hello World 1 - Received: Thread-3: Hello World 2 - Received: Thread-4: Hello World 3 - - -The :class:`ForkingMixIn` class is used in the same way, except that the server -will spawn a new process for each request. -Available only on POSIX platforms that support :func:`~os.fork`. - diff --git a/third_party/python/Doc/library/spwd.rst b/third_party/python/Doc/library/spwd.rst deleted file mode 100644 index c6cad2a3c..000000000 --- a/third_party/python/Doc/library/spwd.rst +++ /dev/null @@ -1,75 +0,0 @@ -:mod:`spwd` --- The shadow password database -============================================ - -.. module:: spwd - :platform: Unix - :synopsis: The shadow password database (getspnam() and friends). - --------------- - -This module provides access to the Unix shadow password database. It is -available on various Unix versions. - -You must have enough privileges to access the shadow password database (this -usually means you have to be root). - -Shadow password database entries are reported as a tuple-like object, whose -attributes correspond to the members of the ``spwd`` structure (Attribute field -below, see ````): - -+-------+---------------+---------------------------------+ -| Index | Attribute | Meaning | -+=======+===============+=================================+ -| 0 | ``sp_namp`` | Login name | -+-------+---------------+---------------------------------+ -| 1 | ``sp_pwdp`` | Encrypted password | -+-------+---------------+---------------------------------+ -| 2 | ``sp_lstchg`` | Date of last change | -+-------+---------------+---------------------------------+ -| 3 | ``sp_min`` | Minimal number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 4 | ``sp_max`` | Maximum number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 5 | ``sp_warn`` | Number of days before password | -| | | expires to warn user about it | -+-------+---------------+---------------------------------+ -| 6 | ``sp_inact`` | Number of days after password | -| | | expires until account is | -| | | disabled | -+-------+---------------+---------------------------------+ -| 7 | ``sp_expire`` | Number of days since 1970-01-01 | -| | | when account expires | -+-------+---------------+---------------------------------+ -| 8 | ``sp_flag`` | Reserved | -+-------+---------------+---------------------------------+ - -The sp_namp and sp_pwdp items are strings, all others are integers. -:exc:`KeyError` is raised if the entry asked for cannot be found. - -The following functions are defined: - - -.. function:: getspnam(name) - - Return the shadow password database entry for the given user name. - - .. versionchanged:: 3.6 - Raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user - doesn't have privileges. - -.. function:: getspall() - - Return a list of all available shadow password database entries, in arbitrary - order. - - -.. seealso:: - - Module :mod:`grp` - An interface to the group database, similar to this. - - Module :mod:`pwd` - An interface to the normal password database, similar to this. - diff --git a/third_party/python/Doc/library/sqlite3.rst b/third_party/python/Doc/library/sqlite3.rst deleted file mode 100644 index f426a9c43..000000000 --- a/third_party/python/Doc/library/sqlite3.rst +++ /dev/null @@ -1,1042 +0,0 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ - -.. module:: sqlite3 - :synopsis: A DB-API 2.0 implementation using SQLite 3.x. - -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/sqlite3/` - --------------- - -SQLite is a C library that provides a lightweight disk-based database that -doesn't require a separate server process and allows accessing the database -using a nonstandard variant of the SQL query language. Some applications can use -SQLite for internal data storage. It's also possible to prototype an -application using SQLite and then port the code to a larger database such as -PostgreSQL or Oracle. - -The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`. - -To use the module, you must first create a :class:`Connection` object that -represents the database. Here the data will be stored in the -:file:`example.db` file:: - - import sqlite3 - conn = sqlite3.connect('example.db') - -You can also supply the special name ``:memory:`` to create a database in RAM. - -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`~Cursor.execute` method to perform SQL commands:: - - c = conn.cursor() - - # Create table - c.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') - - # Insert a row of data - c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") - - # Save (commit) the changes - conn.commit() - - # We can also close the connection if we are done with it. - # Just be sure any changes have been committed or they will be lost. - conn.close() - -The data you've saved is persistent and is available in subsequent sessions:: - - import sqlite3 - conn = sqlite3.connect('example.db') - c = conn.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - c.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(c.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) - -To retrieve data after executing a SELECT statement, you can either treat the -cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to -retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the -matching rows. - -This example uses the iterator form:: - - >>> for row in c.execute('SELECT * FROM stocks ORDER BY price'): - print(row) - - ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) - ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) - ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) - - -.. seealso:: - - https://github.com/ghaering/pysqlite - The pysqlite web page -- sqlite3 is developed externally under the name - "pysqlite". - - https://www.sqlite.org - The SQLite web page; the documentation describes the syntax and the - available data types for the supported SQL dialect. - - http://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - - :pep:`249` - Database API Specification 2.0 - PEP written by Marc-André Lemburg. - - -.. _sqlite3-module-contents: - -Module functions and constants ------------------------------- - - -.. data:: version - - The version number of this module, as a string. This is not the version of - the SQLite library. - - -.. data:: version_info - - The version number of this module, as a tuple of integers. This is not the - version of the SQLite library. - - -.. data:: sqlite_version - - The version number of the run-time SQLite library, as a string. - - -.. data:: sqlite_version_info - - The version number of the run-time SQLite library, as a tuple of integers. - - -.. data:: PARSE_DECLTYPES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting it makes the :mod:`sqlite3` module parse the declared type for each - column it returns. It will parse out the first word of the declared type, - i. e. for "integer primary key", it will parse out "integer", or for - "number(10)" it will parse out "number". Then for that column, it will look - into the converters dictionary and use the converter function registered for - that type there. - - -.. data:: PARSE_COLNAMES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting this makes the SQLite interface parse the column name for each column it - returns. It will look for a string formed [mytype] in there, and then decide - that 'mytype' is the type of the column. It will try to find an entry of - 'mytype' in the converters dictionary and then use the converter function found - there to return the value. The column name found in :attr:`Cursor.description` - is only the first word of the column name, i. e. if you use something like - ``'as "x [datetime]"'`` in your SQL, then we will parse out everything until the - first blank for the column name: the column name would simply be "x". - - -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) - - Opens a connection to the SQLite database file *database*. You can use - ``":memory:"`` to open a database connection to a database that resides in RAM - instead of on disk. - - When a database is accessed by multiple connections, and one of the processes - modifies the database, the SQLite database is locked until that transaction is - committed. The *timeout* parameter specifies how long the connection should wait - for the lock to go away until raising an exception. The default for the timeout - parameter is 5.0 (five seconds). - - For the *isolation_level* parameter, please see the - :attr:`~Connection.isolation_level` property of :class:`Connection` objects. - - SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If - you want to use other types you must add support for them yourself. The - *detect_types* parameter and the using custom **converters** registered with the - module-level :func:`register_converter` function allow you to easily do that. - - *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to - any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. - - By default, *check_same_thread* is :const:`True` and only the creating thread may - use the connection. If set :const:`False`, the returned connection may be shared - across multiple threads. When using multiple threads with the same connection - writing operations should be serialized by the user to avoid data corruption. - - By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the - connect call. You can, however, subclass the :class:`Connection` class and make - :func:`connect` use your class instead by providing your class for the *factory* - parameter. - - Consult the section :ref:`sqlite3-types` of this manual for details. - - The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing - overhead. If you want to explicitly set the number of statements that are cached - for the connection, you can set the *cached_statements* parameter. The currently - implemented default is to cache 100 statements. - - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. - - .. versionchanged:: 3.4 - Added the *uri* parameter. - - -.. function:: register_converter(typename, callable) - - Registers a callable to convert a bytestring from the database into a custom - Python type. The callable will be invoked for all database values that are of - the type *typename*. Confer the parameter *detect_types* of the :func:`connect` - function for how the type detection works. Note that *typename* and the name of - the type in your query are matched in case-insensitive manner. - - -.. function:: register_adapter(type, callable) - - Registers a callable to convert the custom Python type *type* into one of - SQLite's supported types. The callable *callable* accepts as single parameter - the Python value, and must return a value of the following types: int, - float, str or bytes. - - -.. function:: complete_statement(sql) - - Returns :const:`True` if the string *sql* contains one or more complete SQL - statements terminated by semicolons. It does not verify that the SQL is - syntactically correct, only that there are no unclosed string literals and the - statement is terminated by a semicolon. - - This can be used to build a shell for SQLite, as in the following example: - - - .. literalinclude:: ../includes/sqlite3/complete_statement.py - - -.. function:: enable_callback_tracebacks(flag) - - By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. - - -.. _sqlite3-connection-objects: - -Connection Objects ------------------- - -.. class:: Connection - - A SQLite database connection has the following attributes and methods: - - .. attribute:: isolation_level - - Get or set the current default isolation level. :const:`None` for autocommit mode or - one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section - :ref:`sqlite3-controlling-transactions` for a more detailed explanation. - - .. attribute:: in_transaction - - :const:`True` if a transaction is active (there are uncommitted changes), - :const:`False` otherwise. Read-only attribute. - - .. versionadded:: 3.2 - - .. method:: cursor(factory=Cursor) - - The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. - - .. method:: commit() - - This method commits the current transaction. If you don't call this method, - anything you did since the last call to ``commit()`` is not visible from - other database connections. If you wonder why you don't see the data you've - written to the database, please check you didn't forget to call this method. - - .. method:: rollback() - - This method rolls back any changes to the database since the last call to - :meth:`commit`. - - .. method:: close() - - This closes the database connection. Note that this does not automatically - call :meth:`commit`. If you just close your database connection without - calling :meth:`commit` first, your changes will be lost! - - .. method:: execute(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by calling - the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.execute` method with the *parameters* given, and returns - the cursor. - - .. method:: executemany(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executemany` method with the *parameters* given, and - returns the cursor. - - .. method:: executescript(sql_script) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executescript` method with the given *sql_script*, and - returns the cursor. - - .. method:: create_function(name, num_params, func) - - Creates a user-defined function that you can later use from within SQL - statements under the function name *name*. *num_params* is the number of - parameters the function accepts (if *num_params* is -1, the function may - take any number of arguments), and *func* is a Python callable that is - called as the SQL function. - - The function can return any of the types supported by SQLite: bytes, str, int, - float and ``None``. - - Example: - - .. literalinclude:: ../includes/sqlite3/md5func.py - - - .. method:: create_aggregate(name, num_params, aggregate_class) - - Creates a user-defined aggregate function. - - The aggregate class must implement a ``step`` method, which accepts the number - of parameters *num_params* (if *num_params* is -1, the function may take - any number of arguments), and a ``finalize`` method which will return the - final result of the aggregate. - - The ``finalize`` method can return any of the types supported by SQLite: - bytes, str, int, float and ``None``. - - Example: - - .. literalinclude:: ../includes/sqlite3/mysumaggr.py - - - .. method:: create_collation(name, callable) - - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. - - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. - - The following example shows a custom collation that sorts "the wrong way": - - .. literalinclude:: ../includes/sqlite3/collation_reverse.py - - To remove a collation, call ``create_collation`` with ``None`` as callable:: - - con.create_collation("reverse", None) - - - .. method:: interrupt() - - You can call this method from a different thread to abort any queries that might - be executing on the connection. The query will then abort and the caller will - get an exception. - - - .. method:: set_authorizer(authorizer_callback) - - This routine registers a callback. The callback is invoked for each attempt to - access a column of a table in the database. The callback should return - :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL - statement should be aborted with an error and :const:`SQLITE_IGNORE` if the - column should be treated as a NULL value. These constants are available in the - :mod:`sqlite3` module. - - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or :const:`None` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - :const:`None` if this access attempt is directly from input SQL code. - - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`sqlite3` module. - - - .. method:: set_progress_handler(handler, n) - - This routine registers a callback. The callback is invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. - - If you want to clear any previously installed progress handler, call the - method with :const:`None` for *handler*. - - Returning a non-zero value from the handler function will terminate the - currently executing query and cause it to raise an :exc:`OperationalError` - exception. - - - .. method:: set_trace_callback(trace_callback) - - Registers *trace_callback* to be called for each SQL statement that is - actually executed by the SQLite backend. - - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. - - Passing :const:`None` as *trace_callback* will disable the trace callback. - - .. versionadded:: 3.3 - - - .. method:: enable_load_extension(enabled) - - This routine allows/disallows the SQLite engine to load SQLite extensions - from shared libraries. SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. literalinclude:: ../includes/sqlite3/load_extension.py - - .. method:: load_extension(path) - - This routine loads a SQLite extension from a shared library. You have to - enable extension loading with :meth:`enable_load_extension` before you can - use this routine. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. attribute:: row_factory - - You can change this attribute to a callable that accepts the cursor and the - original row as a tuple and will return the real result row. This way, you can - implement more advanced ways of returning results, such as returning an object - that can also access columns by name. - - Example: - - .. literalinclude:: ../includes/sqlite3/row_factory.py - - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. - - .. XXX what's a db_row-based solution? - - - .. attribute:: text_factory - - Using this attribute you can control what objects are returned for the ``TEXT`` - data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. - - You can also set it to any other callable that accepts a single bytestring - parameter and returns the resulting object. - - See the following example code for illustration: - - .. literalinclude:: ../includes/sqlite3/text_factory.py - - - .. attribute:: total_changes - - Returns the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. - - - .. method:: iterdump - - Returns an iterator to dump the database in an SQL text format. Useful when - saving an in-memory database for later restoration. This function provides - the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` - shell. - - Example:: - - # Convert file existing_db.db to SQL dump file dump.sql - import sqlite3 - - con = sqlite3.connect('existing_db.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) - - -.. _sqlite3-cursor-objects: - -Cursor Objects --------------- - -.. class:: Cursor - - A :class:`Cursor` instance has the following attributes and methods. - - .. index:: single: ? (question mark); in SQL statements - .. index:: single: : (colon); in SQL statements - - .. method:: execute(sql[, parameters]) - - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py - - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a :exc:`.Warning`. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. - - - .. method:: executemany(sql, seq_of_parameters) - - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. - - .. literalinclude:: ../includes/sqlite3/executemany_1.py - - Here's a shorter example using a :term:`generator`: - - .. literalinclude:: ../includes/sqlite3/executemany_2.py - - - .. method:: executescript(sql_script) - - This is a nonstandard convenience method for executing multiple SQL statements - at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. - - *sql_script* can be an instance of :class:`str`. - - Example: - - .. literalinclude:: ../includes/sqlite3/executescript.py - - - .. method:: fetchone() - - Fetches the next row of a query result set, returning a single sequence, - or :const:`None` when no more data is available. - - - .. method:: fetchmany(size=cursor.arraysize) - - Fetches the next set of rows of a query result, returning a list. An empty - list is returned when no more rows are available. - - The number of rows to fetch per call is specified by the *size* parameter. - If it is not given, the cursor's arraysize determines the number of rows - to be fetched. The method should try to fetch as many rows as indicated by - the size parameter. If this is not possible due to the specified number of - rows not being available, fewer rows may be returned. - - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. - - .. method:: fetchall() - - Fetches all (remaining) rows of a query result, returning a list. Note that - the cursor's arraysize attribute can affect the performance of this operation. - An empty list is returned when no rows are available. - - .. method:: close() - - Close the cursor now (rather than whenever ``__del__`` is called). - - The cursor will be unusable from this point forward; a :exc:`ProgrammingError` - exception will be raised if any operation is attempted with the cursor. - - .. attribute:: rowcount - - Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this - attribute, the database engine's own support for the determination of "rows - affected"/"rows selected" is quirky. - - For :meth:`executemany` statements, the number of modifications are summed up - into :attr:`rowcount`. - - As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in - case no ``executeXX()`` has been performed on the cursor or the rowcount of the - last operation is not determinable by the interface". This includes ``SELECT`` - statements because we cannot determine the number of rows a query produced - until all rows were fetched. - - With SQLite versions before 3.6.5, :attr:`rowcount` is set to 0 if - you make a ``DELETE FROM table`` without any condition. - - .. attribute:: lastrowid - - This read-only attribute provides the rowid of the last modified row. It is - only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the - :meth:`execute` method. For operations other than ``INSERT`` or - ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is - set to :const:`None`. - - If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous - successful rowid is returned. - - .. versionchanged:: 3.6 - Added support for the ``REPLACE`` statement. - - .. attribute:: arraysize - - Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. - The default value is 1 which means a single row would be fetched per call. - - .. attribute:: description - - This read-only attribute provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are :const:`None`. - - It is set for ``SELECT`` statements without any matching rows as well. - - .. attribute:: connection - - This read-only attribute provides the SQLite database :class:`Connection` - used by the :class:`Cursor` object. A :class:`Cursor` object created by - calling :meth:`con.cursor() ` will have a - :attr:`connection` attribute that refers to *con*:: - - >>> con = sqlite3.connect(":memory:") - >>> cur = con.cursor() - >>> cur.connection == con - True - -.. _sqlite3-row-objects: - -Row Objects ------------ - -.. class:: Row - - A :class:`Row` instance serves as a highly optimized - :attr:`~Connection.row_factory` for :class:`Connection` objects. - It tries to mimic a tuple in most of its features. - - It supports mapping access by column name and index, iteration, - representation, equality testing and :func:`len`. - - If two :class:`Row` objects have exactly the same columns and their - members are equal, they compare equal. - - .. method:: keys - - This method returns a list of column names. Immediately after a query, - it is the first member of each tuple in :attr:`Cursor.description`. - - .. versionchanged:: 3.5 - Added support of slicing. - -Let's assume we initialize a table as in the example given above:: - - conn = sqlite3.connect(":memory:") - c = conn.cursor() - c.execute('''create table stocks - (date text, trans text, symbol text, - qty real, price real)''') - c.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - conn.commit() - c.close() - -Now we plug :class:`Row` in:: - - >>> conn.row_factory = sqlite3.Row - >>> c = conn.cursor() - >>> c.execute('select * from stocks') - - >>> r = c.fetchone() - >>> type(r) - - >>> tuple(r) - ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) - >>> len(r) - 5 - >>> r[2] - 'RHAT' - >>> r.keys() - ['date', 'trans', 'symbol', 'qty', 'price'] - >>> r['qty'] - 100.0 - >>> for member in r: - ... print(member) - ... - 2006-01-05 - BUY - RHAT - 100.0 - 35.14 - - -.. _sqlite3-exceptions: - -Exceptions ----------- - -.. exception:: Warning - - A subclass of :exc:`Exception`. - -.. exception:: Error - - The base class of the other exceptions in this module. It is a subclass - of :exc:`Exception`. - -.. exception:: DatabaseError - - Exception raised for errors that are related to the database. - -.. exception:: IntegrityError - - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. - -.. exception:: ProgrammingError - - Exception raised for programming errors, e.g. table not found or already - exists, syntax error in the SQL statement, wrong number of parameters - specified, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: OperationalError - - Exception raised for errors that are related to the database's operation - and not necessarily under the control of the programmer, e.g. an unexpected - disconnect occurs, the data source name is not found, a transaction could - not be processed, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: NotSupportedError - - Exception raised in case a method or database API was used which is not - supported by the database, e.g. calling the :meth:`~Connection.rollback` - method on a connection that does not support transaction or has - transactions turned off. It is a subclass of :exc:`DatabaseError`. - - -.. _sqlite3-types: - -SQLite and Python types ------------------------ - - -Introduction -^^^^^^^^^^^^ - -SQLite natively supports the following types: ``NULL``, ``INTEGER``, -``REAL``, ``TEXT``, ``BLOB``. - -The following Python types can thus be sent to SQLite without any problem: - -+-------------------------------+-------------+ -| Python type | SQLite type | -+===============================+=============+ -| :const:`None` | ``NULL`` | -+-------------------------------+-------------+ -| :class:`int` | ``INTEGER`` | -+-------------------------------+-------------+ -| :class:`float` | ``REAL`` | -+-------------------------------+-------------+ -| :class:`str` | ``TEXT`` | -+-------------------------------+-------------+ -| :class:`bytes` | ``BLOB`` | -+-------------------------------+-------------+ - - -This is how SQLite types are converted to Python types by default: - -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | :const:`None` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ - -The type system of the :mod:`sqlite3` module is extensible in two ways: you can -store additional Python types in a SQLite database via object adaptation, and -you can let the :mod:`sqlite3` module convert SQLite types to different Python -types via converters. - - -Using adapters to store additional Python types in SQLite databases -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As described before, SQLite supports only a limited set of types natively. To -use other Python types with SQLite, you must **adapt** them to one of the -sqlite3 module's supported types for SQLite: one of NoneType, int, float, -str, bytes. - -There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python -type to one of the supported ones. - - -Letting your object adapt itself -"""""""""""""""""""""""""""""""" - -This is a good approach if you write the class yourself. Let's suppose you have -a class like this:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - -Now you want to store the point in a single SQLite column. First you'll have to -choose one of the supported types first to be used for representing the point. -Let's just use str and separate the coordinates using a semicolon. Then you need -to give your class a method ``__conform__(self, protocol)`` which must return -the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_1.py - - -Registering an adapter callable -""""""""""""""""""""""""""""""" - -The other possibility is to create a function that converts the type to the -string representation and register the function with :meth:`register_adapter`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_2.py - -The :mod:`sqlite3` module has two default adapters for Python's built-in -:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose -we want to store :class:`datetime.datetime` objects not in ISO representation, -but as a Unix timestamp. - -.. literalinclude:: ../includes/sqlite3/adapter_datetime.py - - -Converting SQLite values to custom Python types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing an adapter lets you send custom Python types to SQLite. But to make it -really useful we need to make the Python to SQLite to Python roundtrip work. - -Enter converters. - -Let's go back to the :class:`Point` class. We stored the x and y coordinates -separated via semicolons as strings in SQLite. - -First, we'll define a converter function that accepts the string as a parameter -and constructs a :class:`Point` object from it. - -.. note:: - - Converter functions **always** get called with a :class:`bytes` object, no - matter under which data type you sent the value to SQLite. - -:: - - def convert_point(s): - x, y = map(float, s.split(b";")) - return Point(x, y) - -Now you need to make the :mod:`sqlite3` module know that what you select from -the database is actually a point. There are two ways of doing this: - -* Implicitly via the declared type - -* Explicitly via the column name - -Both ways are described in section :ref:`sqlite3-module-contents`, in the entries -for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. - -The following example illustrates both approaches. - -.. literalinclude:: ../includes/sqlite3/converter_point.py - - -Default adapters and converters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are default adapters for the date and datetime types in the datetime -module. They will be sent as ISO dates/ISO timestamps to SQLite. - -The default converters are registered under the name "date" for -:class:`datetime.date` and under the name "timestamp" for -:class:`datetime.datetime`. - -This way, you can use date/timestamps from Python without any additional -fiddling in most cases. The format of the adapters is also compatible with the -experimental SQLite date/time functions. - -The following example demonstrates this. - -.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py - -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - - -.. _sqlite3-controlling-transactions: - -Controlling Transactions ------------------------- - -The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, -but the Python :mod:`sqlite3` module by default does not. - -``autocommit`` mode means that statements that modify the database take effect -immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` -mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the -outermost transaction, turns ``autocommit`` mode back on. - -The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement -implicitly before a Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). - -You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly -executes via the *isolation_level* parameter to the :func:`connect` -call, or via the :attr:`isolation_level` property of connections. -If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is -equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` -and ``EXCLUSIVE``. - -You can disable the :mod:`sqlite3` module's implicit transaction management by -setting :attr:`isolation_level` to ``None``. This will leave the underlying -``sqlite3`` library operating in ``autocommit`` mode. You can then completely -control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, -``SAVEPOINT``, and ``RELEASE`` statements in your code. - -.. versionchanged:: 3.6 - :mod:`sqlite3` used to implicitly commit an open transaction before DDL - statements. This is no longer the case. - - -Using :mod:`sqlite3` efficiently --------------------------------- - - -Using shortcut methods -^^^^^^^^^^^^^^^^^^^^^^ - -Using the nonstandard :meth:`execute`, :meth:`executemany` and -:meth:`executescript` methods of the :class:`Connection` object, your code can -be written more concisely because you don't have to create the (often -superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` -objects are created implicitly and these shortcut methods return the cursor -objects. This way, you can execute a ``SELECT`` statement and iterate over it -directly using only a single call on the :class:`Connection` object. - -.. literalinclude:: ../includes/sqlite3/shortcut_methods.py - - -Accessing columns by name instead of by index -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -One useful feature of the :mod:`sqlite3` module is the built-in -:class:`sqlite3.Row` class designed to be used as a row factory. - -Rows wrapped with this class can be accessed both by index (like tuples) and -case-insensitively by name: - -.. literalinclude:: ../includes/sqlite3/rowclass.py - - -Using the connection as a context manager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Connection objects can be used as context managers -that automatically commit or rollback transactions. In the event of an -exception, the transaction is rolled back; otherwise, the transaction is -committed: - -.. literalinclude:: ../includes/sqlite3/ctx_manager.py - - -Common issues -------------- - -Multithreading -^^^^^^^^^^^^^^ - -Older SQLite versions had issues with sharing connections between threads. -That's why the Python module disallows sharing connections and cursors between -threads. If you still try to do so, you will get an exception at runtime. - -The only exception is calling the :meth:`~Connection.interrupt` method, which -only makes sense to call from a different thread. - -.. rubric:: Footnotes - -.. [#f1] The sqlite3 module is not built with loadable extension support by - default, because some platforms (notably Mac OS X) have SQLite - libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to - configure. diff --git a/third_party/python/Doc/library/ssl.rst b/third_party/python/Doc/library/ssl.rst deleted file mode 100644 index a85be1a74..000000000 --- a/third_party/python/Doc/library/ssl.rst +++ /dev/null @@ -1,2434 +0,0 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= - -.. module:: ssl - :synopsis: TLS/SSL wrapper for socket objects - -.. moduleauthor:: Bill Janssen -.. sectionauthor:: Bill Janssen - -**Source code:** :source:`Lib/ssl.py` - -.. index:: single: OpenSSL; (use in module ssl) - -.. index:: TLS, SSL, Transport Layer Security, Secure Sockets Layer - --------------- - -This module provides access to Transport Layer Security (often known as "Secure -Sockets Layer") encryption and peer authentication facilities for network -sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, Mac OS X, and -probably additional platforms, as long as OpenSSL is installed on that platform. - -.. note:: - - Some behavior may be platform dependent, since calls are made to the - operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.1 and TLSv1.2 come with - openssl version 1.0.1. - -.. warning:: - Don't use this module without reading the :ref:`ssl-security`. Doing so - may lead to a false sense of security, as the default settings of the - ssl module are not necessarily appropriate for your application. - - -This section documents the objects and functions in the ``ssl`` module; for more -general information about TLS, SSL, and certificates, the reader is referred to -the documents in the "See Also" section at the bottom. - -This module provides a class, :class:`ssl.SSLSocket`, which is derived from the -:class:`socket.socket` type, and provides a socket-like wrapper that also -encrypts and decrypts the data going over the socket with SSL. It supports -additional methods such as :meth:`getpeercert`, which retrieves the -certificate of the other side of the connection, and :meth:`cipher`,which -retrieves the cipher being used for the secure connection. - -For more sophisticated applications, the :class:`ssl.SSLContext` class -helps manage settings and certificates, which can then be inherited -by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. - -.. versionchanged:: 3.5.3 - Updated to support linking with OpenSSL 1.1.0 - -.. versionchanged:: 3.6 - - OpenSSL 0.9.8, 1.0.0 and 1.0.1 are deprecated and no longer supported. - In the future the ssl module will require at least OpenSSL 1.0.2 or - 1.1.0. - - -Functions, Constants, and Exceptions ------------------------------------- - -.. exception:: SSLError - - Raised to signal an error from the underlying SSL implementation - (currently provided by the OpenSSL library). This signifies some - problem in the higher-level encryption and authentication layer that's - superimposed on the underlying network connection. This error - is a subtype of :exc:`OSError`. The error code and message of - :exc:`SSLError` instances are provided by the OpenSSL library. - - .. versionchanged:: 3.3 - :exc:`SSLError` used to be a subtype of :exc:`socket.error`. - - .. attribute:: library - - A string mnemonic designating the OpenSSL submodule in which the error - occurred, such as ``SSL``, ``PEM`` or ``X509``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - - .. attribute:: reason - - A string mnemonic designating the reason this error occurred, for - example ``CERTIFICATE_VERIFY_FAILED``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - -.. exception:: SSLZeroReturnError - - A subclass of :exc:`SSLError` raised when trying to read or write and - the SSL connection has been closed cleanly. Note that this doesn't - mean that the underlying transport (read TCP) has been closed. - - .. versionadded:: 3.3 - -.. exception:: SSLWantReadError - - A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket - ` when trying to read or write data, but more data needs - to be received on the underlying TCP transport before the request can be - fulfilled. - - .. versionadded:: 3.3 - -.. exception:: SSLWantWriteError - - A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket - ` when trying to read or write data, but more data needs - to be sent on the underlying TCP transport before the request can be - fulfilled. - - .. versionadded:: 3.3 - -.. exception:: SSLSyscallError - - A subclass of :exc:`SSLError` raised when a system error was encountered - while trying to fulfill an operation on a SSL socket. Unfortunately, - there is no easy way to inspect the original errno number. - - .. versionadded:: 3.3 - -.. exception:: SSLEOFError - - A subclass of :exc:`SSLError` raised when the SSL connection has been - terminated abruptly. Generally, you shouldn't try to reuse the underlying - transport when this error is encountered. - - .. versionadded:: 3.3 - -.. exception:: CertificateError - - Raised to signal an error with a certificate (such as mismatching - hostname). Certificate errors detected by OpenSSL, though, raise - an :exc:`SSLError`. - - -Socket creation -^^^^^^^^^^^^^^^ - -The following function allows for standalone socket creation. Starting from -Python 3.2, it can be more flexible to use :meth:`SSLContext.wrap_socket` -instead. - -.. function:: wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, ssl_version={see docs}, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None) - - Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance - of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps - the underlying socket in an SSL context. ``sock`` must be a - :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported. - - For client-side sockets, the context construction is lazy; if the - underlying socket isn't connected yet, the context construction will be - performed after :meth:`connect` is called on the socket. For - server-side sockets, if the socket has no remote peer, it is assumed - to be a listening socket, and the server-side SSL wrapping is - automatically performed on client connections accepted via the - :meth:`accept` method. :func:`wrap_socket` may raise :exc:`SSLError`. - - The ``keyfile`` and ``certfile`` parameters specify optional files which - contain a certificate to be used to identify the local side of the - connection. See the discussion of :ref:`ssl-certificates` for more - information on how the certificate is stored in the ``certfile``. - - The parameter ``server_side`` is a boolean which identifies whether - server-side or client-side behavior is desired from this socket. - - The parameter ``cert_reqs`` specifies whether a certificate is required from - the other side of the connection, and whether it will be validated if - provided. It must be one of the three values :const:`CERT_NONE` - (certificates ignored), :const:`CERT_OPTIONAL` (not required, but validated - if provided), or :const:`CERT_REQUIRED` (required and validated). If the - value of this parameter is not :const:`CERT_NONE`, then the ``ca_certs`` - parameter must point to a file of CA certificates. - - The ``ca_certs`` file contains a set of concatenated "certification - authority" certificates, which are used to validate certificates passed from - the other end of the connection. See the discussion of - :ref:`ssl-certificates` for more information about how to arrange the - certificates in this file. - - The parameter ``ssl_version`` specifies which version of the SSL protocol to - use. Typically, the server chooses a particular protocol version, and the - client must adapt to the server's choice. Most of the versions are not - interoperable with the other versions. If not specified, the default is - :data:`PROTOCOL_TLS`; it provides the most compatibility with other - versions. - - Here's a table showing which versions in a client (down the side) can connect - to which versions in a server (along the top): - - .. table:: - - ======================== ============ ============ ============= ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** - ------------------------ ------------ ------------ ------------- --------- ----------- ----------- - *SSLv2* yes no no [1]_ no no no - *SSLv3* no yes no [2]_ no no no - *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes - *TLSv1* no no yes yes no no - *TLSv1.1* no no yes no yes no - *TLSv1.2* no no yes no no yes - ======================== ============ ============ ============= ========= =========== =========== - - .. rubric:: Footnotes - .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. - .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default. - .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in - OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just - TLS 1.3. - - .. note:: - - Which connections succeed will vary depending on the version of - OpenSSL. For example, before OpenSSL 1.0.0, an SSLv23 client - would always attempt SSLv2 connections. - - The *ciphers* parameter sets the available ciphers for this SSL object. - It should be a string in the `OpenSSL cipher list format - `_. - - The parameter ``do_handshake_on_connect`` specifies whether to do the SSL - handshake automatically after doing a :meth:`socket.connect`, or whether the - application program will call it explicitly, by invoking the - :meth:`SSLSocket.do_handshake` method. Calling - :meth:`SSLSocket.do_handshake` explicitly gives the program control over the - blocking behavior of the socket I/O involved in the handshake. - - The parameter ``suppress_ragged_eofs`` specifies how the - :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end - of the connection. If specified as :const:`True` (the default), it returns a - normal EOF (an empty bytes object) in response to unexpected EOF errors - raised from the underlying socket; if :const:`False`, it will raise the - exceptions back to the caller. - - .. versionchanged:: 3.2 - New optional argument *ciphers*. - -Context creation -^^^^^^^^^^^^^^^^ - -A convenience function helps create :class:`SSLContext` objects for common -purposes. - -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) - - Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, - and usually represent a higher security level than when calling the - :class:`SSLContext` constructor directly. - - *cafile*, *capath*, *cadata* represent optional CA certificates to - trust for certificate verification, as in - :meth:`SSLContext.load_verify_locations`. If all three are - :const:`None`, this function can choose to trust the system's default - CA certificates instead. - - The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and - :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and - without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` - as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` - and either loads CA certificates (when at least one of *cafile*, *capath* or - *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load - default CA certificates. - - .. note:: - The protocol, options, cipher and other settings may change to more - restrictive values anytime without prior deprecation. The values - represent a fair balance between compatibility and security. - - If your application needs specific settings, you should create a - :class:`SSLContext` and apply the settings yourself. - - .. note:: - If you find that when certain older clients or servers attempt to connect - with a :class:`SSLContext` created by this function that they get an error - stating "Protocol or cipher suite mismatch", it may be that they only - support SSL3.0 which this function excludes using the - :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken - `_. If you still wish to continue to - use this function but still allow SSL 3.0 connections you can re-enable - them using:: - - ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) - ctx.options &= ~ssl.OP_NO_SSLv3 - - .. versionadded:: 3.4 - - .. versionchanged:: 3.4.4 - - RC4 was dropped from the default cipher string. - - .. versionchanged:: 3.6 - - ChaCha20/Poly1305 was added to the default cipher string. - - 3DES was dropped from the default cipher string. - - -Random generation -^^^^^^^^^^^^^^^^^ - -.. function:: RAND_bytes(num) - - Return *num* cryptographically strong pseudo-random bytes. Raises an - :class:`SSLError` if the PRNG has not been seeded with enough data or if the - operation is not supported by the current RAND method. :func:`RAND_status` - can be used to check the status of the PRNG and :func:`RAND_add` can be used - to seed the PRNG. - - For almost all applications :func:`os.urandom` is preferable. - - Read the Wikipedia article, `Cryptographically secure pseudorandom number - generator (CSPRNG) - `_, - to get the requirements of a cryptographically generator. - - .. versionadded:: 3.3 - -.. function:: RAND_pseudo_bytes(num) - - Return (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, - is_cryptographic is ``True`` if the bytes generated are cryptographically - strong. Raises an :class:`SSLError` if the operation is not supported by the - current RAND method. - - Generated pseudo-random byte sequences will be unique if they are of - sufficient length, but are not necessarily unpredictable. They can be used - for non-cryptographic purposes and for certain purposes in cryptographic - protocols, but usually not for key generation etc. - - For almost all applications :func:`os.urandom` is preferable. - - .. versionadded:: 3.3 - - .. deprecated:: 3.6 - - OpenSSL has deprecated :func:`ssl.RAND_pseudo_bytes`, use - :func:`ssl.RAND_bytes` instead. - -.. function:: RAND_status() - - Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. - -.. function:: RAND_egd(path) - - If you are running an entropy-gathering daemon (EGD) somewhere, and *path* - is the pathname of a socket connection open to it, this will read 256 bytes - of randomness from the socket, and add it to the SSL pseudo-random number - generator to increase the security of generated secret keys. This is - typically only necessary on systems without better sources of randomness. - - See http://egd.sourceforge.net/ or http://prngd.sourceforge.net/ for sources - of entropy-gathering daemons. - - Availability: not available with LibreSSL and OpenSSL > 1.1.0 - -.. function:: RAND_add(bytes, entropy) - - Mix the given *bytes* into the SSL pseudo-random number generator. The - parameter *entropy* (a float) is a lower bound on the entropy contained in - string (so you can always use :const:`0.0`). See :rfc:`1750` for more - information on sources of entropy. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - -Certificate handling -^^^^^^^^^^^^^^^^^^^^ - -.. function:: match_hostname(cert, hostname) - - Verify that *cert* (in decoded format as returned by - :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules - applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818`, :rfc:`5280` and :rfc:`6125`. In addition to HTTPS, this - function should be suitable for checking the identity of servers in - various SSL-based protocols such as FTPS, IMAPS, POPS and others. - - :exc:`CertificateError` is raised on failure. On success, the function - returns nothing:: - - >>> cert = {'subject': ((('commonName', 'example.com'),),)} - >>> ssl.match_hostname(cert, "example.com") - >>> ssl.match_hostname(cert, "example.org") - Traceback (most recent call last): - File "", line 1, in - File "/home/py3k/Lib/ssl.py", line 130, in match_hostname - ssl.CertificateError: hostname 'example.org' doesn't match 'example.com' - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3.3 - The function now follows :rfc:`6125`, section 6.4.3 and does neither - match multiple wildcards (e.g. ``*.*.com`` or ``*a*.example.org``) nor - a wildcard inside an internationalized domain names (IDN) fragment. - IDN A-labels such as ``www*.xn--pthon-kva.org`` are still supported, - but ``x*.python.org`` no longer matches ``xn--tda.python.org``. - - .. versionchanged:: 3.5 - Matching of IP addresses, when present in the subjectAltName field - of the certificate, is now supported. - -.. function:: cert_time_to_seconds(cert_time) - - Return the time in seconds since the Epoch, given the ``cert_time`` - string representing the "notBefore" or "notAfter" date from a - certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C - locale). - - Here's an example: - - .. doctest:: newcontext - - >>> import ssl - >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") - >>> timestamp - 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) - 2018-01-05 09:34:43 - - "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). - - .. versionchanged:: 3.5 - Interpret the input time as a time in UTC as specified by 'GMT' - timezone in the input string. Local timezone was used - previously. Return an integer (no fractions of a second in the - input format) - -.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None) - - Given the address ``addr`` of an SSL-protected server, as a (*hostname*, - *port-number*) pair, fetches the server's certificate, and returns it as a - PEM-encoded string. If ``ssl_version`` is specified, uses that version of - the SSL protocol to attempt to connect to the server. If ``ca_certs`` is - specified, it should be a file containing a list of root certificates, the - same format as used for the same parameter in :func:`wrap_socket`. The call - will attempt to validate the server certificate against that set of root - certificates, and will fail if the validation attempt fails. - - .. versionchanged:: 3.3 - This function is now IPv6-compatible. - - .. versionchanged:: 3.5 - The default *ssl_version* is changed from :data:`PROTOCOL_SSLv3` to - :data:`PROTOCOL_TLS` for maximum compatibility with modern servers. - -.. function:: DER_cert_to_PEM_cert(DER_cert_bytes) - - Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded - string version of the same certificate. - -.. function:: PEM_cert_to_DER_cert(PEM_cert_string) - - Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of - bytes for that same certificate. - -.. function:: get_default_verify_paths() - - Returns a named tuple with paths to OpenSSL's default cafile and capath. - The paths are the same as used by - :meth:`SSLContext.set_default_verify_paths`. The return value is a - :term:`named tuple` ``DefaultVerifyPaths``: - - * :attr:`cafile` - resolved path to cafile or ``None`` if the file doesn't exist, - * :attr:`capath` - resolved path to capath or ``None`` if the directory doesn't exist, - * :attr:`openssl_cafile_env` - OpenSSL's environment key that points to a cafile, - * :attr:`openssl_cafile` - hard coded path to a cafile, - * :attr:`openssl_capath_env` - OpenSSL's environment key that points to a capath, - * :attr:`openssl_capath` - hard coded path to a capath directory - - Availability: LibreSSL ignores the environment vars - :attr:`openssl_cafile_env` and :attr:`openssl_capath_env` - - .. versionadded:: 3.4 - -.. function:: enum_certificates(store_name) - - Retrieve certificates from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. Trust specifies the purpose of the certificate as a set - of OIDS or exactly ``True`` if the certificate is trustworthy for all - purposes. - - Example:: - - >>> ssl.enum_certificates("CA") - [(b'data...', 'x509_asn', {'1.3.6.1.5.5.7.3.1', '1.3.6.1.5.5.7.3.2'}), - (b'data...', 'x509_asn', True)] - - Availability: Windows. - - .. versionadded:: 3.4 - -.. function:: enum_crls(store_name) - - Retrieve CRLs from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. - - Availability: Windows. - - .. versionadded:: 3.4 - - -Constants -^^^^^^^^^ - - All constants are now :class:`enum.IntEnum` or :class:`enum.IntFlag` collections. - - .. versionadded:: 3.6 - -.. data:: CERT_NONE - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. Except for :const:`PROTOCOL_TLS_CLIENT`, - it is the default mode. With client-side sockets, just about any - cert is accepted. Validation errors, such as untrusted or expired cert, - are ignored and do not abort the TLS/SSL handshake. - - In server mode, no certificate is requested from the client, so the client - does not send any for client cert authentication. - - See the discussion of :ref:`ssl-security` below. - -.. data:: CERT_OPTIONAL - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. In client mode, :const:`CERT_OPTIONAL` - has the same meaning as :const:`CERT_REQUIRED`. It is recommended to - use :const:`CERT_REQUIRED` for client-side sockets instead. - - In server mode, a client certificate request is sent to the client. The - client may either ignore the request or send a certificate in order - perform TLS client cert authentication. If the client chooses to send - a certificate, it is verified. Any verification error immediately aborts - the TLS handshake. - - Use of this setting requires a valid set of CA certificates to - be passed, either to :meth:`SSLContext.load_verify_locations` or as a - value of the ``ca_certs`` parameter to :func:`wrap_socket`. - -.. data:: CERT_REQUIRED - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. In this mode, certificates are - required from the other side of the socket connection; an :class:`SSLError` - will be raised if no certificate is provided, or if its validation fails. - This mode is **not** sufficient to verify a certificate in client mode as - it does not match hostnames. :attr:`~SSLContext.check_hostname` must be - enabled as well to verify the authenticity of a cert. - :const:`PROTOCOL_TLS_CLIENT` uses :const:`CERT_REQUIRED` and - enables :attr:`~SSLContext.check_hostname` by default. - - With server socket, this mode provides mandatory TLS client cert - authentication. A client certificate request is sent to the client and - the client must provide a valid and trusted certificate. - - Use of this setting requires a valid set of CA certificates to - be passed, either to :meth:`SSLContext.load_verify_locations` or as a - value of the ``ca_certs`` parameter to :func:`wrap_socket`. - -.. class:: VerifyMode - - :class:`enum.IntEnum` collection of CERT_* constants. - - .. versionadded:: 3.6 - -.. data:: VERIFY_DEFAULT - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, certificate - revocation lists (CRLs) are not checked. By default OpenSSL does neither - require nor verify CRLs. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_LEAF - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the - peer cert is check but non of the intermediate CA certificates. The mode - requires a valid CRL that is signed by the peer cert's issuer (its direct - ancestor CA). If no proper has been loaded - :attr:`SSLContext.load_verify_locations`, validation will fail. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_CHAIN - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, CRLs of - all certificates in the peer cert chain are checked. - - .. versionadded:: 3.4 - -.. data:: VERIFY_X509_STRICT - - Possible value for :attr:`SSLContext.verify_flags` to disable workarounds - for broken X.509 certificates. - - .. versionadded:: 3.4 - -.. data:: VERIFY_X509_TRUSTED_FIRST - - Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to - prefer trusted certificates when building the trust chain to validate a - certificate. This flag is enabled by default. - - .. versionadded:: 3.4.4 - -.. class:: VerifyFlags - - :class:`enum.IntFlag` collection of VERIFY_* constants. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_TLS - - Selects the highest protocol version that both the client and server support. - Despite the name, this option can select both "SSL" and "TLS" protocols. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_TLS_CLIENT - - Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, - but only support client-side :class:`SSLSocket` connections. The protocol - enables :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` by - default. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_TLS_SERVER - - Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, - but only support server-side :class:`SSLSocket` connections. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_SSLv23 - - Alias for data:`PROTOCOL_TLS`. - - .. deprecated:: 3.6 - - Use :data:`PROTOCOL_TLS` instead. - -.. data:: PROTOCOL_SSLv2 - - Selects SSL version 2 as the channel encryption protocol. - - This protocol is not available if OpenSSL is compiled with the - ``OPENSSL_NO_SSL2`` flag. - - .. warning:: - - SSL version 2 is insecure. Its use is highly discouraged. - - .. deprecated:: 3.6 - - OpenSSL has removed support for SSLv2. - -.. data:: PROTOCOL_SSLv3 - - Selects SSL version 3 as the channel encryption protocol. - - This protocol is not be available if OpenSSL is compiled with the - ``OPENSSL_NO_SSLv3`` flag. - - .. warning:: - - SSL version 3 is insecure. Its use is highly discouraged. - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. - -.. data:: PROTOCOL_TLSv1 - - Selects TLS version 1.0 as the channel encryption protocol. - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. - -.. data:: PROTOCOL_TLSv1_1 - - Selects TLS version 1.1 as the channel encryption protocol. - Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. - -.. data:: PROTOCOL_TLSv1_2 - - Selects TLS version 1.2 as the channel encryption protocol. This is the - most modern version, and probably the best choice for maximum protection, - if both sides can speak it. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. - -.. data:: OP_ALL - - Enables workarounds for various bugs present in other SSL implementations. - This option is set by default. It does not necessarily set the same - flags as OpenSSL's ``SSL_OP_ALL`` constant. - - .. versionadded:: 3.2 - -.. data:: OP_NO_SSLv2 - - Prevents an SSLv2 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing SSLv2 as the protocol version. - - .. versionadded:: 3.2 - - .. deprecated:: 3.6 - - SSLv2 is deprecated - - -.. data:: OP_NO_SSLv3 - - Prevents an SSLv3 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing SSLv3 as the protocol version. - - .. versionadded:: 3.2 - - .. deprecated:: 3.6 - - SSLv3 is deprecated - -.. data:: OP_NO_TLSv1 - - Prevents a TLSv1 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing TLSv1 as the protocol version. - - .. versionadded:: 3.2 - -.. data:: OP_NO_TLSv1_1 - - Prevents a TLSv1.1 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.1 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - -.. data:: OP_NO_TLSv1_2 - - Prevents a TLSv1.2 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.2 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - -.. data:: OP_NO_TLSv1_3 - - Prevents a TLSv1.3 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.3 as - the protocol version. TLS 1.3 is available with OpenSSL 1.1.1 or later. - When Python has been compiled against an older version of OpenSSL, the - flag defaults to *0*. - - .. versionadded:: 3.6.3 - -.. data:: OP_CIPHER_SERVER_PREFERENCE - - Use the server's cipher ordering preference, rather than the client's. - This option has no effect on client sockets and SSLv2 server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_SINGLE_DH_USE - - Prevents re-use of the same DH key for distinct SSL sessions. This - improves forward secrecy but requires more computational resources. - This option only applies to server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_SINGLE_ECDH_USE - - Prevents re-use of the same ECDH key for distinct SSL sessions. This - improves forward secrecy but requires more computational resources. - This option only applies to server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_ENABLE_MIDDLEBOX_COMPAT - - Send dummy Change Cipher Spec (CCS) messages in TLS 1.3 handshake to make - a TLS 1.3 connection look more like a TLS 1.2 connection. - - This option is only available with OpenSSL 1.1.1 and later. - - .. versionadded:: 3.6.7 - -.. data:: OP_NO_COMPRESSION - - Disable compression on the SSL channel. This is useful if the application - protocol supports its own compression scheme. - - This option is only available with OpenSSL 1.0.0 and later. - - .. versionadded:: 3.3 - -.. class:: Options - - :class:`enum.IntFlag` collection of OP_* constants. - -.. data:: OP_NO_TICKET - - Prevent client side from requesting a session ticket. - - .. versionadded:: 3.6 - -.. data:: HAS_ALPN - - Whether the OpenSSL library has built-in support for the *Application-Layer - Protocol Negotiation* TLS extension as described in :rfc:`7301`. - - .. versionadded:: 3.5 - -.. data:: HAS_ECDH - - Whether the OpenSSL library has built-in support for Elliptic Curve-based - Diffie-Hellman key exchange. This should be true unless the feature was - explicitly disabled by the distributor. - - .. versionadded:: 3.3 - -.. data:: HAS_SNI - - Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`6066`). - - .. versionadded:: 3.2 - -.. data:: HAS_NPN - - Whether the OpenSSL library has built-in support for *Next Protocol - Negotiation* as described in the `NPN draft specification - `_. When true, - you can use the :meth:`SSLContext.set_npn_protocols` method to advertise - which protocols you want to support. - - .. versionadded:: 3.3 - -.. data:: HAS_TLSv1_3 - - Whether the OpenSSL library has built-in support for the TLS 1.3 protocol. - - .. versionadded:: 3.6.3 - -.. data:: CHANNEL_BINDING_TYPES - - List of supported TLS channel binding types. Strings in this list - can be used as arguments to :meth:`SSLSocket.get_channel_binding`. - - .. versionadded:: 3.3 - -.. data:: OPENSSL_VERSION - - The version string of the OpenSSL library loaded by the interpreter:: - - >>> ssl.OPENSSL_VERSION - 'OpenSSL 1.0.2k 26 Jan 2017' - - .. versionadded:: 3.2 - -.. data:: OPENSSL_VERSION_INFO - - A tuple of five integers representing version information about the - OpenSSL library:: - - >>> ssl.OPENSSL_VERSION_INFO - (1, 0, 2, 11, 15) - - .. versionadded:: 3.2 - -.. data:: OPENSSL_VERSION_NUMBER - - The raw version number of the OpenSSL library, as a single integer:: - - >>> ssl.OPENSSL_VERSION_NUMBER - 268443839 - >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x100020bf' - - .. versionadded:: 3.2 - -.. data:: ALERT_DESCRIPTION_HANDSHAKE_FAILURE - ALERT_DESCRIPTION_INTERNAL_ERROR - ALERT_DESCRIPTION_* - - Alert Descriptions from :rfc:`5246` and others. The `IANA TLS Alert Registry - `_ - contains this list and references to the RFCs where their meaning is defined. - - Used as the return value of the callback function in - :meth:`SSLContext.set_servername_callback`. - - .. versionadded:: 3.4 - -.. class:: AlertDescription - - :class:`enum.IntEnum` collection of ALERT_DESCRIPTION_* constants. - - .. versionadded:: 3.6 - -.. data:: Purpose.SERVER_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate Web servers (therefore, it will - be used to create client-side sockets). - - .. versionadded:: 3.4 - -.. data:: Purpose.CLIENT_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate Web clients (therefore, it will - be used to create server-side sockets). - - .. versionadded:: 3.4 - -.. class:: SSLErrorNumber - - :class:`enum.IntEnum` collection of SSL_ERROR_* constants. - - .. versionadded:: 3.6 - - -SSL Sockets ------------ - -.. class:: SSLSocket(socket.socket) - - SSL sockets provide the following methods of :ref:`socket-objects`: - - - :meth:`~socket.socket.accept()` - - :meth:`~socket.socket.bind()` - - :meth:`~socket.socket.close()` - - :meth:`~socket.socket.connect()` - - :meth:`~socket.socket.detach()` - - :meth:`~socket.socket.fileno()` - - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` - - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` - - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` - - :meth:`~socket.socket.listen()` - - :meth:`~socket.socket.makefile()` - - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` - (but passing a non-zero ``flags`` argument is not allowed) - - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with - the same limitation) - - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send()` will be used) - - :meth:`~socket.socket.shutdown()` - - However, since the SSL (and TLS) protocol has its own framing atop - of TCP, the SSL sockets abstraction can, in certain respects, diverge from - the specification of normal, OS-level sockets. See especially the - :ref:`notes on non-blocking sockets `. - - Usually, :class:`SSLSocket` are not created directly, but using the - :meth:`SSLContext.wrap_socket` method. - - .. versionchanged:: 3.5 - The :meth:`sendfile` method was added. - - .. versionchanged:: 3.5 - The :meth:`shutdown` does not reset the socket timeout each time bytes - are received or sent. The socket timeout is now to maximum total duration - of the shutdown. - - .. deprecated:: 3.6 - It is deprecated to create a :class:`SSLSocket` instance directly, use - :meth:`SSLContext.wrap_socket` to wrap a socket. - - -SSL sockets also have the following additional methods and attributes: - -.. method:: SSLSocket.read(len=1024, buffer=None) - - Read up to *len* bytes of data from the SSL socket and return the result as - a ``bytes`` instance. If *buffer* is specified, then read into the buffer - instead, and return the number of bytes read. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the read would block. - - As at any time a re-negotiation is possible, a call to :meth:`read` can also - cause write operations. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration to read up to *len* - bytes. - - .. deprecated:: 3.6 - Use :meth:`~SSLSocket.recv` instead of :meth:`~SSLSocket.read`. - -.. method:: SSLSocket.write(buf) - - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the write would block. - - As at any time a re-negotiation is possible, a call to :meth:`write` can - also cause read operations. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration to write *buf*. - - .. deprecated:: 3.6 - Use :meth:`~SSLSocket.send` instead of :meth:`~SSLSocket.write`. - -.. note:: - - The :meth:`~SSLSocket.read` and :meth:`~SSLSocket.write` methods are the - low-level methods that read and write unencrypted, application-level data - and decrypt/encrypt it to encrypted, wire-level data. These methods - require an active SSL connection, i.e. the handshake was completed and - :meth:`SSLSocket.unwrap` was not called. - - Normally you should use the socket API methods like - :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these - methods. - -.. method:: SSLSocket.do_handshake() - - Perform the SSL setup handshake. - - .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the - :attr:`~SSLContext.check_hostname` attribute of the socket's - :attr:`~SSLSocket.context` is true. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration of the handshake. - -.. method:: SSLSocket.getpeercert(binary_form=False) - - If there is no certificate for the peer on the other end of the connection, - return ``None``. If the SSL handshake hasn't been done yet, raise - :exc:`ValueError`. - - If the ``binary_form`` parameter is :const:`False`, and a certificate was - received from the peer, this method returns a :class:`dict` instance. If the - certificate was not validated, the dict is empty. If the certificate was - validated, it returns a dict with several keys, amongst them ``subject`` - (the principal for which the certificate was issued) and ``issuer`` - (the principal issuing the certificate). If a certificate contains an - instance of the *Subject Alternative Name* extension (see :rfc:`3280`), - there will also be a ``subjectAltName`` key in the dictionary. - - The ``subject`` and ``issuer`` fields are tuples containing the sequence - of relative distinguished names (RDNs) given in the certificate's data - structure for the respective fields, and each RDN is a sequence of - name-value pairs. Here is a real-world example:: - - {'issuer': ((('countryName', 'IL'),), - (('organizationName', 'StartCom Ltd.'),), - (('organizationalUnitName', - 'Secure Digital Certificate Signing'),), - (('commonName', - 'StartCom Class 2 Primary Intermediate Server CA'),)), - 'notAfter': 'Nov 22 08:15:19 2013 GMT', - 'notBefore': 'Nov 21 03:09:52 2011 GMT', - 'serialNumber': '95F0', - 'subject': ((('description', '571208-SLe257oHY9fVQ07Z'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'California'),), - (('localityName', 'San Francisco'),), - (('organizationName', 'Electronic Frontier Foundation, Inc.'),), - (('commonName', '*.eff.org'),), - (('emailAddress', 'hostmaster@eff.org'),)), - 'subjectAltName': (('DNS', '*.eff.org'), ('DNS', 'eff.org')), - 'version': 3} - - .. note:: - - To validate a certificate for a particular service, you can use the - :func:`match_hostname` function. - - If the ``binary_form`` parameter is :const:`True`, and a certificate was - provided, this method returns the DER-encoded form of the entire certificate - as a sequence of bytes, or :const:`None` if the peer did not provide a - certificate. Whether the peer provides a certificate depends on the SSL - socket's role: - - * for a client SSL socket, the server will always provide a certificate, - regardless of whether validation was required; - - * for a server SSL socket, the client will only provide a certificate - when requested by the server; therefore :meth:`getpeercert` will return - :const:`None` if you used :const:`CERT_NONE` (rather than - :const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`). - - .. versionchanged:: 3.2 - The returned dictionary includes additional items such as ``issuer`` - and ``notBefore``. - - .. versionchanged:: 3.4 - :exc:`ValueError` is raised when the handshake isn't done. - The returned dictionary includes additional X509v3 extension items - such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs. - -.. method:: SSLSocket.cipher() - - Returns a three-value tuple containing the name of the cipher being used, the - version of the SSL protocol that defines its use, and the number of secret - bits being used. If no connection has been established, returns ``None``. - -.. method:: SSLSocket.shared_ciphers() - - Return the list of ciphers shared by the client during the handshake. Each - entry of the returned list is a three-value tuple containing the name of the - cipher, the version of the SSL protocol that defines its use, and the number - of secret bits the cipher uses. :meth:`~SSLSocket.shared_ciphers` returns - ``None`` if no connection has been established or the socket is a client - socket. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.compression() - - Return the compression algorithm being used as a string, or ``None`` - if the connection isn't compressed. - - If the higher-level protocol supports its own compression mechanism, - you can use :data:`OP_NO_COMPRESSION` to disable SSL-level compression. - - .. versionadded:: 3.3 - -.. method:: SSLSocket.get_channel_binding(cb_type="tls-unique") - - Get channel binding data for current connection, as a bytes object. Returns - ``None`` if not connected or the handshake has not been completed. - - The *cb_type* parameter allow selection of the desired channel binding - type. Valid channel binding types are listed in the - :data:`CHANNEL_BINDING_TYPES` list. Currently only the 'tls-unique' channel - binding, defined by :rfc:`5929`, is supported. :exc:`ValueError` will be - raised if an unsupported channel binding type is requested. - - .. versionadded:: 3.3 - -.. method:: SSLSocket.selected_alpn_protocol() - - Return the protocol that was selected during the TLS handshake. If - :meth:`SSLContext.set_alpn_protocols` was not called, if the other party does - not support ALPN, if this socket does not support any of the client's - proposed protocols, or if the handshake has not happened yet, ``None`` is - returned. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.selected_npn_protocol() - - Return the higher-level protocol that was selected during the TLS/SSL - handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or - if the other party does not support NPN, or if the handshake has not yet - happened, this will return ``None``. - - .. versionadded:: 3.3 - -.. method:: SSLSocket.unwrap() - - Performs the SSL shutdown handshake, which removes the TLS layer from the - underlying socket, and returns the underlying socket object. This can be - used to go from encrypted operation over a connection to unencrypted. The - returned socket should always be used for further communication with the - other side of the connection, rather than the original socket. - -.. method:: SSLSocket.verify_client_post_handshake() - - Requests post-handshake authentication (PHA) from a TLS 1.3 client. PHA - can only be initiated for a TLS 1.3 connection from a server-side socket, - after the initial TLS handshake and with PHA enabled on both sides, see - :attr:`SSLContext.post_handshake_auth`. - - The method does not perform a cert exchange immediately. The server-side - sends a CertificateRequest during the next write event and expects the - client to respond with a certificate on the next read event. - - If any precondition isn't met (e.g. not TLS 1.3, PHA not enabled), an - :exc:`SSLError` is raised. - - .. versionadded:: 3.6.7 - - .. note:: - Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 - support, the method raises :exc:`NotImplementedError`. - -.. method:: SSLSocket.version() - - Return the actual SSL protocol version negotiated by the connection - as a string, or ``None`` is no secure connection is established. - As of this writing, possible return values include ``"SSLv2"``, - ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. - Recent OpenSSL versions may define more return values. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.pending() - - Returns the number of already decrypted bytes available for read, pending on - the connection. - -.. attribute:: SSLSocket.context - - The :class:`SSLContext` object this SSL socket is tied to. If the SSL - socket was created using the top-level :func:`wrap_socket` function - (rather than :meth:`SSLContext.wrap_socket`), this is a custom context - object created for this SSL socket. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.server_side - - A boolean which is ``True`` for server-side sockets and ``False`` for - client-side sockets. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.server_hostname - - Hostname of the server: :class:`str` type, or ``None`` for server-side - socket or if the hostname was not specified in the constructor. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.session - - The :class:`SSLSession` for this SSL connection. The session is available - for client and server side sockets after the TLS handshake has been - performed. For client sockets the session can be set before - :meth:`~SSLSocket.do_handshake` has been called to reuse a session. - - .. versionadded:: 3.6 - -.. attribute:: SSLSocket.session_reused - - .. versionadded:: 3.6 - - -SSL Contexts ------------- - -.. versionadded:: 3.2 - -An SSL context holds various data longer-lived than single SSL connections, -such as SSL configuration options, certificate(s) and private key(s). -It also manages a cache of SSL sessions for server-side sockets, in order -to speed up repeated connections from the same clients. - -.. class:: SSLContext(protocol=PROTOCOL_TLS) - - Create a new SSL context. You may pass *protocol* which must be one - of the ``PROTOCOL_*`` constants defined in this module. - :data:`PROTOCOL_TLS` is currently recommended for maximum - interoperability and default value. - - .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose - security settings for a given purpose. - - .. versionchanged:: 3.6 - - The context is created with secure default values. The options - :data:`OP_NO_COMPRESSION`, :data:`OP_CIPHER_SERVER_PREFERENCE`, - :data:`OP_SINGLE_DH_USE`, :data:`OP_SINGLE_ECDH_USE`, - :data:`OP_NO_SSLv2` (except for :data:`PROTOCOL_SSLv2`), - and :data:`OP_NO_SSLv3` (except for :data:`PROTOCOL_SSLv3`) are - set by default. The initial cipher suite list contains only ``HIGH`` - ciphers, no ``NULL`` ciphers and no ``MD5`` ciphers (except for - :data:`PROTOCOL_SSLv2`). - - -:class:`SSLContext` objects have the following methods and attributes: - -.. method:: SSLContext.cert_store_stats() - - Get statistics about quantities of loaded X.509 certificates, count of - X.509 certificates flagged as CA certificates and certificate revocation - lists as dictionary. - - Example for a context with one CA cert and one other cert:: - - >>> context.cert_store_stats() - {'crl': 0, 'x509_ca': 1, 'x509': 2} - - .. versionadded:: 3.4 - - -.. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None) - - Load a private key and the corresponding certificate. The *certfile* - string must be the path to a single file in PEM format containing the - certificate as well as any number of CA certificates needed to establish - the certificate's authenticity. The *keyfile* string, if present, must - point to a file containing the private key in. Otherwise the private - key will be taken from *certfile* as well. See the discussion of - :ref:`ssl-certificates` for more information on how the certificate - is stored in the *certfile*. - - The *password* argument may be a function to call to get the password for - decrypting the private key. It will only be called if the private key is - encrypted and a password is necessary. It will be called with no arguments, - and it should return a string, bytes, or bytearray. If the return value is - a string it will be encoded as UTF-8 before using it to decrypt the key. - Alternatively a string, bytes, or bytearray value may be supplied directly - as the *password* argument. It will be ignored if the private key is not - encrypted and no password is needed. - - If the *password* argument is not specified and a password is required, - OpenSSL's built-in password prompting mechanism will be used to - interactively prompt the user for a password. - - An :class:`SSLError` is raised if the private key doesn't - match with the certificate. - - .. versionchanged:: 3.3 - New optional argument *password*. - -.. method:: SSLContext.load_default_certs(purpose=Purpose.SERVER_AUTH) - - Load a set of default "certification authority" (CA) certificates from - default locations. On Windows it loads CA certs from the ``CA`` and - ``ROOT`` system stores. On other systems it calls - :meth:`SSLContext.set_default_verify_paths`. In the future the method may - load CA certificates from other locations, too. - - The *purpose* flag specifies what kind of CA certificates are loaded. The - default settings :data:`Purpose.SERVER_AUTH` loads certificates, that are - flagged and trusted for TLS web server authentication (client side - sockets). :data:`Purpose.CLIENT_AUTH` loads CA certificates for client - certificate verification on the server side. - - .. versionadded:: 3.4 - -.. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None) - - Load a set of "certification authority" (CA) certificates used to validate - other peers' certificates when :data:`verify_mode` is other than - :data:`CERT_NONE`. At least one of *cafile* or *capath* must be specified. - - This method can also load certification revocation lists (CRLs) in PEM or - DER format. In order to make use of CRLs, :attr:`SSLContext.verify_flags` - must be configured properly. - - The *cafile* string, if present, is the path to a file of concatenated - CA certificates in PEM format. See the discussion of - :ref:`ssl-certificates` for more information about how to arrange the - certificates in this file. - - The *capath* string, if present, is - the path to a directory containing several CA certificates in PEM format, - following an `OpenSSL specific layout - `_. - - The *cadata* object, if present, is either an ASCII string of one or more - PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded - certificates. Like with *capath* extra lines around PEM-encoded - certificates are ignored but at least one certificate must be present. - - .. versionchanged:: 3.4 - New optional argument *cadata* - -.. method:: SSLContext.get_ca_certs(binary_form=False) - - Get a list of loaded "certification authority" (CA) certificates. If the - ``binary_form`` parameter is :const:`False` each list - entry is a dict like the output of :meth:`SSLSocket.getpeercert`. Otherwise - the method returns a list of DER-encoded certificates. The returned list - does not contain certificates from *capath* unless a certificate was - requested and loaded by a SSL connection. - - .. note:: - Certificates in a capath directory aren't loaded unless they have - been used at least once. - - .. versionadded:: 3.4 - -.. method:: SSLContext.get_ciphers() - - Get a list of enabled ciphers. The list is in order of cipher priority. - See :meth:`SSLContext.set_ciphers`. - - Example:: - - >>> ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - >>> ctx.set_ciphers('ECDHE+AESGCM:!ECDSA') - >>> ctx.get_ciphers() # OpenSSL 1.0.x - [{'alg_bits': 256, - 'description': 'ECDHE-RSA-AES256-GCM-SHA384 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(256) Mac=AEAD', - 'id': 50380848, - 'name': 'ECDHE-RSA-AES256-GCM-SHA384', - 'protocol': 'TLSv1/SSLv3', - 'strength_bits': 256}, - {'alg_bits': 128, - 'description': 'ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(128) Mac=AEAD', - 'id': 50380847, - 'name': 'ECDHE-RSA-AES128-GCM-SHA256', - 'protocol': 'TLSv1/SSLv3', - 'strength_bits': 128}] - - On OpenSSL 1.1 and newer the cipher dict contains additional fields:: - >>> ctx.get_ciphers() # OpenSSL 1.1+ - [{'aead': True, - 'alg_bits': 256, - 'auth': 'auth-rsa', - 'description': 'ECDHE-RSA-AES256-GCM-SHA384 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(256) Mac=AEAD', - 'digest': None, - 'id': 50380848, - 'kea': 'kx-ecdhe', - 'name': 'ECDHE-RSA-AES256-GCM-SHA384', - 'protocol': 'TLSv1.2', - 'strength_bits': 256, - 'symmetric': 'aes-256-gcm'}, - {'aead': True, - 'alg_bits': 128, - 'auth': 'auth-rsa', - 'description': 'ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(128) Mac=AEAD', - 'digest': None, - 'id': 50380847, - 'kea': 'kx-ecdhe', - 'name': 'ECDHE-RSA-AES128-GCM-SHA256', - 'protocol': 'TLSv1.2', - 'strength_bits': 128, - 'symmetric': 'aes-128-gcm'}] - - Availability: OpenSSL 1.0.2+ - - .. versionadded:: 3.6 - -.. method:: SSLContext.set_default_verify_paths() - - Load a set of default "certification authority" (CA) certificates from - a filesystem path defined when building the OpenSSL library. Unfortunately, - there's no easy way to know whether this method succeeds: no error is - returned if no certificates are to be found. When the OpenSSL library is - provided as part of the operating system, though, it is likely to be - configured properly. - -.. method:: SSLContext.set_ciphers(ciphers) - - Set the available ciphers for sockets created with this context. - It should be a string in the `OpenSSL cipher list format - `_. - If no cipher can be selected (because compile-time options or other - configuration forbids use of all the specified ciphers), an - :class:`SSLError` will be raised. - - .. note:: - when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will - give the currently selected cipher. - - OpenSSL 1.1.1 has TLS 1.3 cipher suites enabled by default. The suites - cannot be disabled with :meth:`~SSLContext.set_ciphers`. - -.. method:: SSLContext.set_alpn_protocols(protocols) - - Specify which protocols the socket should advertise during the SSL/TLS - handshake. It should be a list of ASCII strings, like ``['http/1.1', - 'spdy/2']``, ordered by preference. The selection of a protocol will happen - during the handshake, and will play out according to :rfc:`7301`. After a - successful handshake, the :meth:`SSLSocket.selected_alpn_protocol` method will - return the agreed-upon protocol. - - This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is - False. - - OpenSSL 1.1.0 to 1.1.0e will abort the handshake and raise :exc:`SSLError` - when both sides support ALPN but cannot agree on a protocol. 1.1.0f+ - behaves like 1.0.2, :meth:`SSLSocket.selected_alpn_protocol` returns None. - - .. versionadded:: 3.5 - -.. method:: SSLContext.set_npn_protocols(protocols) - - Specify which protocols the socket should advertise during the SSL/TLS - handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, - ordered by preference. The selection of a protocol will happen during the - handshake, and will play out according to the `NPN draft specification - `_. After a - successful handshake, the :meth:`SSLSocket.selected_npn_protocol` method will - return the agreed-upon protocol. - - This method will raise :exc:`NotImplementedError` if :data:`HAS_NPN` is - False. - - .. versionadded:: 3.3 - -.. method:: SSLContext.set_servername_callback(server_name_callback) - - Register a callback function that will be called after the TLS Client Hello - handshake message has been received by the SSL/TLS server when the TLS client - specifies a server name indication. The server name indication mechanism - is specified in :rfc:`6066` section 3 - Server Name Indication. - - Only one callback can be set per ``SSLContext``. If *server_name_callback* - is ``None`` then the callback is disabled. Calling this function a - subsequent time will disable the previously registered callback. - - The callback function, *server_name_callback*, will be called with three - arguments; the first being the :class:`ssl.SSLSocket`, the second is a string - that represents the server name that the client is intending to communicate - (or :const:`None` if the TLS Client Hello does not contain a server name) - and the third argument is the original :class:`SSLContext`. The server name - argument is the IDNA decoded server name. - - A typical use of this callback is to change the :class:`ssl.SSLSocket`'s - :attr:`SSLSocket.context` attribute to a new object of type - :class:`SSLContext` representing a certificate chain that matches the server - name. - - Due to the early negotiation phase of the TLS connection, only limited - methods and attributes are usable like - :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`. - :meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.getpeercert`, - :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compress` methods require that - the TLS connection has progressed beyond the TLS Client Hello and therefore - will not contain return meaningful values nor can they be called safely. - - The *server_name_callback* function must return ``None`` to allow the - TLS negotiation to continue. If a TLS failure is required, a constant - :const:`ALERT_DESCRIPTION_* ` can be - returned. Other return values will result in a TLS fatal error with - :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`. - - If there is an IDNA decoding error on the server name, the TLS connection - will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS - alert message to the client. - - If an exception is raised from the *server_name_callback* function the TLS - connection will terminate with a fatal TLS alert message - :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`. - - This method will raise :exc:`NotImplementedError` if the OpenSSL library - had OPENSSL_NO_TLSEXT defined when it was built. - - .. versionadded:: 3.4 - -.. method:: SSLContext.load_dh_params(dhfile) - - Load the key generation parameters for Diffie-Hellman (DH) key exchange. - Using DH key exchange improves forward secrecy at the expense of - computational resources (both on the server and on the client). - The *dhfile* parameter should be the path to a file containing DH - parameters in PEM format. - - This setting doesn't apply to client sockets. You can also use the - :data:`OP_SINGLE_DH_USE` option to further improve security. - - .. versionadded:: 3.3 - -.. method:: SSLContext.set_ecdh_curve(curve_name) - - Set the curve name for Elliptic Curve-based Diffie-Hellman (ECDH) key - exchange. ECDH is significantly faster than regular DH while arguably - as secure. The *curve_name* parameter should be a string describing - a well-known elliptic curve, for example ``prime256v1`` for a widely - supported curve. - - This setting doesn't apply to client sockets. You can also use the - :data:`OP_SINGLE_ECDH_USE` option to further improve security. - - This method is not available if :data:`HAS_ECDH` is ``False``. - - .. versionadded:: 3.3 - - .. seealso:: - `SSL/TLS & Perfect Forward Secrecy `_ - Vincent Bernat. - -.. method:: SSLContext.wrap_socket(sock, server_side=False, \ - do_handshake_on_connect=True, suppress_ragged_eofs=True, \ - server_hostname=None, session=None) - - Wrap an existing Python socket *sock* and return an :class:`SSLSocket` - object. *sock* must be a :data:`~socket.SOCK_STREAM` socket; other socket - types are unsupported. - - The returned SSL socket is tied to the context, its settings and - certificates. The parameters *server_side*, *do_handshake_on_connect* - and *suppress_ragged_eofs* have the same meaning as in the top-level - :func:`wrap_socket` function. - - On client connections, the optional parameter *server_hostname* specifies - the hostname of the service which we are connecting to. This allows a - single server to host multiple SSL-based services with distinct certificates, - quite similarly to HTTP virtual hosts. Specifying *server_hostname* will - raise a :exc:`ValueError` if *server_side* is true. - - *session*, see :attr:`~SSLSocket.session`. - - .. versionchanged:: 3.5 - Always allow a server_hostname to be passed, even if OpenSSL does not - have SNI. - - .. versionchanged:: 3.6 - *session* argument was added. - -.. method:: SSLContext.wrap_bio(incoming, outgoing, server_side=False, \ - server_hostname=None, session=None) - - Create a new :class:`SSLObject` instance by wrapping the BIO objects - *incoming* and *outgoing*. The SSL routines will read input data from the - incoming BIO and write data to the outgoing BIO. - - The *server_side*, *server_hostname* and *session* parameters have the - same meaning as in :meth:`SSLContext.wrap_socket`. - - .. versionchanged:: 3.6 - *session* argument was added. - -.. method:: SSLContext.session_stats() - - Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information - `_ to their - numeric values. For example, here is the total number of hits and misses - in the session cache since the context was created:: - - >>> stats = context.session_stats() - >>> stats['hits'], stats['misses'] - (0, 0) - -.. attribute:: SSLContext.check_hostname - - Whether to match the peer cert's hostname with :func:`match_hostname` in - :meth:`SSLSocket.do_handshake`. The context's - :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or - :data:`CERT_REQUIRED`, and you must pass *server_hostname* to - :meth:`~SSLContext.wrap_socket` in order to match the hostname. - - Example:: - - import socket, ssl - - context = ssl.SSLContext() - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True - context.load_default_certs() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ssl_sock = context.wrap_socket(s, server_hostname='www.verisign.com') - ssl_sock.connect(('www.verisign.com', 443)) - - .. versionadded:: 3.4 - - .. note:: - - This features requires OpenSSL 0.9.8f or newer. - -.. attribute:: SSLContext.options - - An integer representing the set of SSL options enabled on this context. - The default value is :data:`OP_ALL`, but you can specify other options - such as :data:`OP_NO_SSLv2` by ORing them together. - - .. note:: - With versions of OpenSSL older than 0.9.8m, it is only possible - to set options, not to clear them. Attempting to clear an option - (by resetting the corresponding bits) will raise a ``ValueError``. - - .. versionchanged:: 3.6 - :attr:`SSLContext.options` returns :class:`Options` flags: - - >>> ssl.create_default_context().options - - -.. attribute:: SSLContext.post_handshake_auth - - Enable TLS 1.3 post-handshake client authentication. Post-handshake auth - is disabled by default and a server can only request a TLS client - certificate during the initial handshake. When enabled, a server may - request a TLS client certificate at any time after the handshake. - - When enabled on client-side sockets, the client signals the server that - it supports post-handshake authentication. - - When enabled on server-side sockets, :attr:`SSLContext.verify_mode` must - be set to :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`, too. The - actual client cert exchange is delayed until - :meth:`SSLSocket.verify_client_post_handshake` is called and some I/O is - performed. - - .. versionadded:: 3.6.7 - - .. note:: - Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 - support, the property value is None and can't be modified - -.. attribute:: SSLContext.protocol - - The protocol version chosen when constructing the context. This attribute - is read-only. - -.. attribute:: SSLContext.verify_flags - - The flags for certificate verification operations. You can set flags like - :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL - does neither require nor verify certificate revocation lists (CRLs). - Available only with openssl version 0.9.8+. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - :attr:`SSLContext.verify_flags` returns :class:`VerifyFlags` flags: - - >>> ssl.create_default_context().verify_flags - - -.. attribute:: SSLContext.verify_mode - - Whether to try to verify other peers' certificates and how to behave - if verification fails. This attribute must be one of - :data:`CERT_NONE`, :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`. - - .. versionchanged:: 3.6 - :attr:`SSLContext.verify_mode` returns :class:`VerifyMode` enum: - - >>> ssl.create_default_context().verify_mode - - -.. index:: single: certificates - -.. index:: single: X509 certificate - -.. _ssl-certificates: - -Certificates ------------- - -Certificates in general are part of a public-key / private-key system. In this -system, each *principal*, (which may be a machine, or a person, or an -organization) is assigned a unique two-part encryption key. One part of the key -is public, and is called the *public key*; the other part is kept secret, and is -called the *private key*. The two parts are related, in that if you encrypt a -message with one of the parts, you can decrypt it with the other part, and -**only** with the other part. - -A certificate contains information about two principals. It contains the name -of a *subject*, and the subject's public key. It also contains a statement by a -second principal, the *issuer*, that the subject is who they claim to be, and -that this is indeed the subject's public key. The issuer's statement is signed -with the issuer's private key, which only the issuer knows. However, anyone can -verify the issuer's statement by finding the issuer's public key, decrypting the -statement with it, and comparing it to the other information in the certificate. -The certificate also contains information about the time period over which it is -valid. This is expressed as two fields, called "notBefore" and "notAfter". - -In the Python use of certificates, a client or server can use a certificate to -prove who they are. The other side of a network connection can also be required -to produce a certificate, and that certificate can be validated to the -satisfaction of the client or server that requires such validation. The -connection attempt can be set to raise an exception if the validation fails. -Validation is done automatically, by the underlying OpenSSL framework; the -application need not concern itself with its mechanics. But the application -does usually need to provide sets of certificates to allow this process to take -place. - -Python uses files to contain certificates. They should be formatted as "PEM" -(see :rfc:`1422`), which is a base-64 encoded form wrapped with a header line -and a footer line:: - - -----BEGIN CERTIFICATE----- - ... (certificate in base64 PEM encoding) ... - -----END CERTIFICATE----- - -Certificate chains -^^^^^^^^^^^^^^^^^^ - -The Python files which contain certificates can contain a sequence of -certificates, sometimes called a *certificate chain*. This chain should start -with the specific certificate for the principal who "is" the client or server, -and then the certificate for the issuer of that certificate, and then the -certificate for the issuer of *that* certificate, and so on up the chain till -you get to a certificate which is *self-signed*, that is, a certificate which -has the same subject and issuer, sometimes called a *root certificate*. The -certificates should just be concatenated together in the certificate file. For -example, suppose we had a three certificate chain, from our server certificate -to the certificate of the certification authority that signed our server -certificate, to the root certificate of the agency which issued the -certification authority's certificate:: - - -----BEGIN CERTIFICATE----- - ... (certificate for your server)... - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - ... (the certificate for the CA)... - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - ... (the root certificate for the CA's issuer)... - -----END CERTIFICATE----- - -CA certificates -^^^^^^^^^^^^^^^ - -If you are going to require validation of the other side of the connection's -certificate, you need to provide a "CA certs" file, filled with the certificate -chains for each issuer you are willing to trust. Again, this file just contains -these chains concatenated together. For validation, Python will use the first -chain it finds in the file which matches. The platform's certificates file can -be used by calling :meth:`SSLContext.load_default_certs`, this is done -automatically with :func:`.create_default_context`. - -Combined key and certificate -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Often the private key is stored in the same file as the certificate; in this -case, only the ``certfile`` parameter to :meth:`SSLContext.load_cert_chain` -and :func:`wrap_socket` needs to be passed. If the private key is stored -with the certificate, it should come before the first certificate in -the certificate chain:: - - -----BEGIN RSA PRIVATE KEY----- - ... (private key in base64 encoding) ... - -----END RSA PRIVATE KEY----- - -----BEGIN CERTIFICATE----- - ... (certificate in base64 PEM encoding) ... - -----END CERTIFICATE----- - -Self-signed certificates -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are going to create a server that provides SSL-encrypted connection -services, you will need to acquire a certificate for that service. There are -many ways of acquiring appropriate certificates, such as buying one from a -certification authority. Another common practice is to generate a self-signed -certificate. The simplest way to do this is with the OpenSSL package, using -something like the following:: - - % openssl req -new -x509 -days 365 -nodes -out cert.pem -keyout cert.pem - Generating a 1024 bit RSA private key - .......++++++ - .............................++++++ - writing new private key to 'cert.pem' - ----- - You are about to be asked to enter information that will be incorporated - into your certificate request. - What you are about to enter is what is called a Distinguished Name or a DN. - There are quite a few fields but you can leave some blank - For some fields there will be a default value, - If you enter '.', the field will be left blank. - ----- - Country Name (2 letter code) [AU]:US - State or Province Name (full name) [Some-State]:MyState - Locality Name (eg, city) []:Some City - Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc. - Organizational Unit Name (eg, section) []:My Group - Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com - Email Address []:ops@myserver.mygroup.myorganization.com - % - -The disadvantage of a self-signed certificate is that it is its own root -certificate, and no one else will have it in their cache of known (and trusted) -root certificates. - - -Examples --------- - -Testing for SSL support -^^^^^^^^^^^^^^^^^^^^^^^ - -To test for the presence of SSL support in a Python installation, user code -should use the following idiom:: - - try: - import ssl - except ImportError: - pass - else: - ... # do something that requires SSL support - -Client-side operation -^^^^^^^^^^^^^^^^^^^^^ - -This example creates a SSL context with the recommended security settings -for client sockets, including automatic certificate verification:: - - >>> context = ssl.create_default_context() - -If you prefer to tune security settings yourself, you might create -a context from scratch (but beware that you might not get the settings -right):: - - >>> context = ssl.SSLContext() - >>> context.verify_mode = ssl.CERT_REQUIRED - >>> context.check_hostname = True - >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt") - -(this snippet assumes your operating system places a bundle of all CA -certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an -error and have to adjust the location) - -When you use the context to connect to a server, :const:`CERT_REQUIRED` -validates the server certificate: it ensures that the server certificate -was signed with one of the CA certificates, and checks the signature for -correctness:: - - >>> conn = context.wrap_socket(socket.socket(socket.AF_INET), - ... server_hostname="www.python.org") - >>> conn.connect(("www.python.org", 443)) - -You may then fetch the certificate:: - - >>> cert = conn.getpeercert() - -Visual inspection shows that the certificate does identify the desired service -(that is, the HTTPS host ``www.python.org``):: - - >>> pprint.pprint(cert) - {'OCSP': ('http://ocsp.digicert.com',), - 'caIssuers': ('http://cacerts.digicert.com/DigiCertSHA2ExtendedValidationServerCA.crt',), - 'crlDistributionPoints': ('http://crl3.digicert.com/sha2-ev-server-g1.crl', - 'http://crl4.digicert.com/sha2-ev-server-g1.crl'), - 'issuer': ((('countryName', 'US'),), - (('organizationName', 'DigiCert Inc'),), - (('organizationalUnitName', 'www.digicert.com'),), - (('commonName', 'DigiCert SHA2 Extended Validation Server CA'),)), - 'notAfter': 'Sep 9 12:00:00 2016 GMT', - 'notBefore': 'Sep 5 00:00:00 2014 GMT', - 'serialNumber': '01BB6F00122B177F36CAB49CEA8B6B26', - 'subject': ((('businessCategory', 'Private Organization'),), - (('1.3.6.1.4.1.311.60.2.1.3', 'US'),), - (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), - (('serialNumber', '3359300'),), - (('streetAddress', '16 Allen Rd'),), - (('postalCode', '03894-4801'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'NH'),), - (('localityName', 'Wolfeboro,'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'www.python.org'),)), - 'subjectAltName': (('DNS', 'www.python.org'), - ('DNS', 'python.org'), - ('DNS', 'pypi.org'), - ('DNS', 'docs.python.org'), - ('DNS', 'testpypi.org'), - ('DNS', 'bugs.python.org'), - ('DNS', 'wiki.python.org'), - ('DNS', 'hg.python.org'), - ('DNS', 'mail.python.org'), - ('DNS', 'packaging.python.org'), - ('DNS', 'pythonhosted.org'), - ('DNS', 'www.pythonhosted.org'), - ('DNS', 'test.pythonhosted.org'), - ('DNS', 'us.pycon.org'), - ('DNS', 'id.python.org')), - 'version': 3} - -Now the SSL channel is established and the certificate verified, you can -proceed to talk with the server:: - - >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: linuxfr.org\r\n\r\n") - >>> pprint.pprint(conn.recv(1024).split(b"\r\n")) - [b'HTTP/1.1 200 OK', - b'Date: Sat, 18 Oct 2014 18:27:20 GMT', - b'Server: nginx', - b'Content-Type: text/html; charset=utf-8', - b'X-Frame-Options: SAMEORIGIN', - b'Content-Length: 45679', - b'Accept-Ranges: bytes', - b'Via: 1.1 varnish', - b'Age: 2188', - b'X-Served-By: cache-lcy1134-LCY', - b'X-Cache: HIT', - b'X-Cache-Hits: 11', - b'Vary: Cookie', - b'Strict-Transport-Security: max-age=63072000; includeSubDomains', - b'Connection: close', - b'', - b''] - -See the discussion of :ref:`ssl-security` below. - - -Server-side operation -^^^^^^^^^^^^^^^^^^^^^ - -For server operation, typically you'll need to have a server certificate, and -private key, each in a file. You'll first create a context holding the key -and the certificate, so that clients can check your authenticity. Then -you'll open a socket, bind it to a port, call :meth:`listen` on it, and start -waiting for clients to connect:: - - import socket, ssl - - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - context.load_cert_chain(certfile="mycertfile", keyfile="mykeyfile") - - bindsocket = socket.socket() - bindsocket.bind(('myaddr.mydomain.com', 10023)) - bindsocket.listen(5) - -When a client connects, you'll call :meth:`accept` on the socket to get the -new socket from the other end, and use the context's :meth:`SSLContext.wrap_socket` -method to create a server-side SSL socket for the connection:: - - while True: - newsocket, fromaddr = bindsocket.accept() - connstream = context.wrap_socket(newsocket, server_side=True) - try: - deal_with_client(connstream) - finally: - connstream.shutdown(socket.SHUT_RDWR) - connstream.close() - -Then you'll read data from the ``connstream`` and do something with it till you -are finished with the client (or the client is finished with you):: - - def deal_with_client(connstream): - data = connstream.recv(1024) - # empty data means the client is finished with us - while data: - if not do_something(connstream, data): - # we'll assume do_something returns False - # when we're finished with client - break - data = connstream.recv(1024) - # finished with client - -And go back to listening for new client connections (of course, a real server -would probably handle each client connection in a separate thread, or put -the sockets in :ref:`non-blocking mode ` and use an event loop). - - -.. _ssl-nonblocking: - -Notes on non-blocking sockets ------------------------------ - -SSL sockets behave slightly different than regular sockets in -non-blocking mode. When working with non-blocking sockets, there are -thus several things you need to be aware of: - -- Most :class:`SSLSocket` methods will raise either - :exc:`SSLWantWriteError` or :exc:`SSLWantReadError` instead of - :exc:`BlockingIOError` if an I/O operation would - block. :exc:`SSLWantReadError` will be raised if a read operation on - the underlying socket is necessary, and :exc:`SSLWantWriteError` for - a write operation on the underlying socket. Note that attempts to - *write* to an SSL socket may require *reading* from the underlying - socket first, and attempts to *read* from the SSL socket may require - a prior *write* to the underlying socket. - - .. versionchanged:: 3.5 - - In earlier Python versions, the :meth:`!SSLSocket.send` method - returned zero instead of raising :exc:`SSLWantWriteError` or - :exc:`SSLWantReadError`. - -- Calling :func:`~select.select` tells you that the OS-level socket can be - read from (or written to), but it does not imply that there is sufficient - data at the upper SSL layer. For example, only part of an SSL frame might - have arrived. Therefore, you must be ready to handle :meth:`SSLSocket.recv` - and :meth:`SSLSocket.send` failures, and retry after another call to - :func:`~select.select`. - -- Conversely, since the SSL layer has its own framing, a SSL socket may - still have data available for reading without :func:`~select.select` - being aware of it. Therefore, you should first call - :meth:`SSLSocket.recv` to drain any potentially available data, and then - only block on a :func:`~select.select` call if still necessary. - - (of course, similar provisions apply when using other primitives such as - :func:`~select.poll`, or those in the :mod:`selectors` module) - -- The SSL handshake itself will be non-blocking: the - :meth:`SSLSocket.do_handshake` method has to be retried until it returns - successfully. Here is a synopsis using :func:`~select.select` to wait for - the socket's readiness:: - - while True: - try: - sock.do_handshake() - break - except ssl.SSLWantReadError: - select.select([sock], [], []) - except ssl.SSLWantWriteError: - select.select([], [sock], []) - -.. seealso:: - - The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and - handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and - :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously - as well. - - -Memory BIO Support ------------------- - -.. versionadded:: 3.5 - -Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket` -class has provided two related but distinct areas of functionality: - -- SSL protocol handling -- Network IO - -The network IO API is identical to that provided by :class:`socket.socket`, -from which :class:`SSLSocket` also inherits. This allows an SSL socket to be -used as a drop-in replacement for a regular socket, making it very easy to add -SSL support to an existing application. - -Combining SSL protocol handling and network IO usually works well, but there -are some cases where it doesn't. An example is async IO frameworks that want to -use a different IO multiplexing model than the "select/poll on a file -descriptor" (readiness based) model that is assumed by :class:`socket.socket` -and by the internal OpenSSL socket IO routines. This is mostly relevant for -platforms like Windows where this model is not efficient. For this purpose, a -reduced scope variant of :class:`SSLSocket` called :class:`SSLObject` is -provided. - -.. class:: SSLObject - - A reduced-scope variant of :class:`SSLSocket` representing an SSL protocol - instance that does not contain any network IO methods. This class is - typically used by framework authors that want to implement asynchronous IO - for SSL through memory buffers. - - This class implements an interface on top of a low-level SSL object as - implemented by OpenSSL. This object captures the state of an SSL connection - but does not provide any network IO itself. IO needs to be performed through - separate "BIO" objects which are OpenSSL's IO abstraction layer. - - An :class:`SSLObject` instance can be created using the - :meth:`~SSLContext.wrap_bio` method. This method will create the - :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming* - BIO is used to pass data from Python to the SSL protocol instance, while the - *outgoing* BIO is used to pass data the other way around. - - The following methods are available: - - - :attr:`~SSLSocket.context` - - :attr:`~SSLSocket.server_side` - - :attr:`~SSLSocket.server_hostname` - - :attr:`~SSLSocket.session` - - :attr:`~SSLSocket.session_reused` - - :meth:`~SSLSocket.read` - - :meth:`~SSLSocket.write` - - :meth:`~SSLSocket.getpeercert` - - :meth:`~SSLSocket.selected_npn_protocol` - - :meth:`~SSLSocket.cipher` - - :meth:`~SSLSocket.shared_ciphers` - - :meth:`~SSLSocket.compression` - - :meth:`~SSLSocket.pending` - - :meth:`~SSLSocket.do_handshake` - - :meth:`~SSLSocket.unwrap` - - :meth:`~SSLSocket.get_channel_binding` - - When compared to :class:`SSLSocket`, this object lacks the following - features: - - - Any form of network IO; ``recv()`` and ``send()`` read and write only to - the underlying :class:`MemoryBIO` buffers. - - - There is no *do_handshake_on_connect* machinery. You must always manually - call :meth:`~SSLSocket.do_handshake` to start the handshake. - - - There is no handling of *suppress_ragged_eofs*. All end-of-file conditions - that are in violation of the protocol are reported via the - :exc:`SSLEOFError` exception. - - - The method :meth:`~SSLSocket.unwrap` call does not return anything, - unlike for an SSL socket where it returns the underlying socket. - - - The *server_name_callback* callback passed to - :meth:`SSLContext.set_servername_callback` will get an :class:`SSLObject` - instance instead of a :class:`SSLSocket` instance as its first parameter. - - Some notes related to the use of :class:`SSLObject`: - - - All IO on an :class:`SSLObject` is :ref:`non-blocking `. - This means that for example :meth:`~SSLSocket.read` will raise an - :exc:`SSLWantReadError` if it needs more data than the incoming BIO has - available. - - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - -An SSLObject communicates with the outside world using memory buffers. The -class :class:`MemoryBIO` provides a memory buffer that can be used for this -purpose. It wraps an OpenSSL memory BIO (Basic IO) object: - -.. class:: MemoryBIO - - A memory buffer that can be used to pass data between Python and an SSL - protocol instance. - - .. attribute:: MemoryBIO.pending - - Return the number of bytes currently in the memory buffer. - - .. attribute:: MemoryBIO.eof - - A boolean indicating whether the memory BIO is current at the end-of-file - position. - - .. method:: MemoryBIO.read(n=-1) - - Read up to *n* bytes from the memory buffer. If *n* is not specified or - negative, all bytes are returned. - - .. method:: MemoryBIO.write(buf) - - Write the bytes from *buf* to the memory BIO. The *buf* argument must be an - object supporting the buffer protocol. - - The return value is the number of bytes written, which is always equal to - the length of *buf*. - - .. method:: MemoryBIO.write_eof() - - Write an EOF marker to the memory BIO. After this method has been called, it - is illegal to call :meth:`~MemoryBIO.write`. The attribute :attr:`eof` will - become true after all data currently in the buffer has been read. - - -SSL session ------------ - -.. versionadded:: 3.6 - -.. class:: SSLSession - - Session object used by :attr:`~SSLSocket.session`. - - .. attribute:: id - .. attribute:: time - .. attribute:: timeout - .. attribute:: ticket_lifetime_hint - .. attribute:: has_ticket - - -.. _ssl-security: - -Security considerations ------------------------ - -Best defaults -^^^^^^^^^^^^^ - -For **client use**, if you don't have any special requirements for your -security policy, it is highly recommended that you use the -:func:`create_default_context` function to create your SSL context. -It will load the system's trusted CA certificates, enable certificate -validation and hostname checking, and try to choose reasonably secure -protocol and cipher settings. - -For example, here is how you would use the :class:`smtplib.SMTP` class to -create a trusted, secure connection to a SMTP server:: - - >>> import ssl, smtplib - >>> smtp = smtplib.SMTP("mail.python.org", port=587) - >>> context = ssl.create_default_context() - >>> smtp.starttls(context=context) - (220, b'2.0.0 Ready to start TLS') - -If a client certificate is needed for the connection, it can be added with -:meth:`SSLContext.load_cert_chain`. - -By contrast, if you create the SSL context by calling the :class:`SSLContext` -constructor yourself, it will not have certificate validation nor hostname -checking enabled by default. If you do so, please read the paragraphs below -to achieve a good security level. - -Manual settings -^^^^^^^^^^^^^^^ - -Verifying certificates -'''''''''''''''''''''' - -When calling the :class:`SSLContext` constructor directly, -:const:`CERT_NONE` is the default. Since it does not authenticate the other -peer, it can be insecure, especially in client mode where most of time you -would like to ensure the authenticity of the server you're talking to. -Therefore, when in client mode, it is highly recommended to use -:const:`CERT_REQUIRED`. However, it is in itself not sufficient; you also -have to check that the server certificate, which can be obtained by calling -:meth:`SSLSocket.getpeercert`, matches the desired service. For many -protocols and applications, the service can be identified by the hostname; -in this case, the :func:`match_hostname` function can be used. This common -check is automatically performed when :attr:`SSLContext.check_hostname` is -enabled. - -In server mode, if you want to authenticate your clients using the SSL layer -(rather than using a higher-level authentication mechanism), you'll also have -to specify :const:`CERT_REQUIRED` and similarly check the client certificate. - - -Protocol versions -''''''''''''''''' - -SSL versions 2 and 3 are considered insecure and are therefore dangerous to -use. If you want maximum compatibility between clients and servers, it is -recommended to use :const:`PROTOCOL_TLS_CLIENT` or -:const:`PROTOCOL_TLS_SERVER` as the protocol version. SSLv2 and SSLv3 are -disabled by default. - - >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.options |= ssl.OP_NO_TLSv1 - >>> client_context.options |= ssl.OP_NO_TLSv1_1 - - -The SSL context created above will only allow TLSv1.2 and later (if -supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` -implies certificate validation and hostname checks by default. You have to -load certificates into the context. - - -Cipher selection -'''''''''''''''' - -If you have advanced security requirements, fine-tuning of the ciphers -enabled when negotiating a SSL session is possible through the -:meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the -ssl module disables certain weak ciphers by default, but you may want -to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. -If you want to check which ciphers are enabled by a given cipher list, use -:meth:`SSLContext.get_ciphers` or the ``openssl ciphers`` command on your -system. - -Multi-processing -^^^^^^^^^^^^^^^^ - -If using this module as part of a multi-processed application (using, -for example the :mod:`multiprocessing` or :mod:`concurrent.futures` modules), -be aware that OpenSSL's internal random number generator does not properly -handle forked processes. Applications must change the PRNG state of the -parent process if they use any SSL feature with :func:`os.fork`. Any -successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or -:func:`~ssl.RAND_pseudo_bytes` is sufficient. - - -.. ssl-libressl: - -LibreSSL support ----------------- - -LibreSSL is a fork of OpenSSL 1.0.1. The ssl module has limited support for -LibreSSL. Some features are not available when the ssl module is compiled -with LibreSSL. - -* LibreSSL >= 2.6.1 no longer supports NPN. The methods - :meth:`SSLContext.set_npn_protocols` and - :meth:`SSLSocket.selected_npn_protocol` are not available. -* :meth:`SSLContext.set_default_verify_paths` ignores the env vars - :envvar:`SSL_CERT_FILE` and :envvar:`SSL_CERT_PATH` although - :func:`get_default_verify_paths` still reports them. - - -.. seealso:: - - Class :class:`socket.socket` - Documentation of underlying :mod:`socket` class - - `SSL/TLS Strong Encryption: An Introduction `_ - Intro from the Apache HTTP Server documentation - - :rfc:`RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <1422>` - Steve Kent - - :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller - - :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper - - :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. - - :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake - - `IANA TLS: Transport Layer Security (TLS) Parameters `_ - IANA - - :rfc:`RFC 7525: Recommendations for Secure Use of Transport Layer Security (TLS) and Datagram Transport Layer Security (DTLS) <7525>` - IETF - - `Mozilla's Server Side TLS recommendations `_ - Mozilla diff --git a/third_party/python/Doc/library/stat.rst b/third_party/python/Doc/library/stat.rst deleted file mode 100644 index c8f6904f9..000000000 --- a/third_party/python/Doc/library/stat.rst +++ /dev/null @@ -1,427 +0,0 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== - -.. module:: stat - :synopsis: Utilities for interpreting the results of os.stat(), - os.lstat() and os.fstat(). - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/stat.py` - --------------- - -The :mod:`stat` module defines constants and functions for interpreting the -results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they -exist). For complete details about the :c:func:`stat`, :c:func:`fstat` and -:c:func:`lstat` calls, consult the documentation for your system. - -.. versionchanged:: 3.4 - The stat module is backed by a C implementation. - -The :mod:`stat` module defines the following functions to test for specific file -types: - - -.. function:: S_ISDIR(mode) - - Return non-zero if the mode is from a directory. - - -.. function:: S_ISCHR(mode) - - Return non-zero if the mode is from a character special device file. - - -.. function:: S_ISBLK(mode) - - Return non-zero if the mode is from a block special device file. - - -.. function:: S_ISREG(mode) - - Return non-zero if the mode is from a regular file. - - -.. function:: S_ISFIFO(mode) - - Return non-zero if the mode is from a FIFO (named pipe). - - -.. function:: S_ISLNK(mode) - - Return non-zero if the mode is from a symbolic link. - - -.. function:: S_ISSOCK(mode) - - Return non-zero if the mode is from a socket. - -.. function:: S_ISDOOR(mode) - - Return non-zero if the mode is from a door. - - .. versionadded:: 3.4 - -.. function:: S_ISPORT(mode) - - Return non-zero if the mode is from an event port. - - .. versionadded:: 3.4 - -.. function:: S_ISWHT(mode) - - Return non-zero if the mode is from a whiteout. - - .. versionadded:: 3.4 - -Two additional functions are defined for more general manipulation of the file's -mode: - - -.. function:: S_IMODE(mode) - - Return the portion of the file's mode that can be set by - :func:`os.chmod`\ ---that is, the file's permission bits, plus the sticky - bit, set-group-id, and set-user-id bits (on systems that support them). - - -.. function:: S_IFMT(mode) - - Return the portion of the file's mode that describes the file type (used by the - :func:`S_IS\*` functions above). - -Normally, you would use the :func:`os.path.is\*` functions for testing the type -of a file; the functions here are useful when you are doing multiple tests of -the same file and wish to avoid the overhead of the :c:func:`stat` system call -for each test. These are also useful when checking for information about a file -that isn't handled by :mod:`os.path`, like the tests for block and character -devices. - -Example:: - - import os, sys - from stat import * - - def walktree(top, callback): - '''recursively descend the directory tree rooted at top, - calling the callback function for each regular file''' - - for f in os.listdir(top): - pathname = os.path.join(top, f) - mode = os.stat(pathname).st_mode - if S_ISDIR(mode): - # It's a directory, recurse into it - walktree(pathname, callback) - elif S_ISREG(mode): - # It's a file, call the callback function - callback(pathname) - else: - # Unknown file type, print a message - print('Skipping %s' % pathname) - - def visitfile(file): - print('visiting', file) - - if __name__ == '__main__': - walktree(sys.argv[1], visitfile) - -An additional utility function is provided to convert a file's mode in a human -readable string: - -.. function:: filemode(mode) - - Convert a file's mode to a string of the form '-rwxrwxrwx'. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The function supports :data:`S_IFDOOR`, :data:`S_IFPORT` and - :data:`S_IFWHT`. - - -All the variables below are simply symbolic indexes into the 10-tuple returned -by :func:`os.stat`, :func:`os.fstat` or :func:`os.lstat`. - - -.. data:: ST_MODE - - Inode protection mode. - - -.. data:: ST_INO - - Inode number. - - -.. data:: ST_DEV - - Device inode resides on. - - -.. data:: ST_NLINK - - Number of links to the inode. - - -.. data:: ST_UID - - User id of the owner. - - -.. data:: ST_GID - - Group id of the owner. - - -.. data:: ST_SIZE - - Size in bytes of a plain file; amount of data waiting on some special files. - - -.. data:: ST_ATIME - - Time of last access. - - -.. data:: ST_MTIME - - Time of last modification. - - -.. data:: ST_CTIME - - The "ctime" as reported by the operating system. On some systems (like Unix) is - the time of the last metadata change, and, on others (like Windows), is the - creation time (see platform documentation for details). - -The interpretation of "file size" changes according to the file type. For plain -files this is the size of the file in bytes. For FIFOs and sockets under most -flavors of Unix (including Linux in particular), the "size" is the number of -bytes waiting to be read at the time of the call to :func:`os.stat`, -:func:`os.fstat`, or :func:`os.lstat`; this can sometimes be useful, especially -for polling one of these special files after a non-blocking open. The meaning -of the size field for other character and block devices varies more, depending -on the implementation of the underlying system call. - -The variables below define the flags used in the :data:`ST_MODE` field. - -Use of the functions above is more portable than use of the first set of flags: - -.. data:: S_IFSOCK - - Socket. - -.. data:: S_IFLNK - - Symbolic link. - -.. data:: S_IFREG - - Regular file. - -.. data:: S_IFBLK - - Block device. - -.. data:: S_IFDIR - - Directory. - -.. data:: S_IFCHR - - Character device. - -.. data:: S_IFIFO - - FIFO. - -.. data:: S_IFDOOR - - Door. - - .. versionadded:: 3.4 - -.. data:: S_IFPORT - - Event port. - - .. versionadded:: 3.4 - -.. data:: S_IFWHT - - Whiteout. - - .. versionadded:: 3.4 - -.. note:: - - :data:`S_IFDOOR`, :data:`S_IFPORT` or :data:`S_IFWHT` are defined as - 0 when the platform does not have support for the file types. - -The following flags can also be used in the *mode* argument of :func:`os.chmod`: - -.. data:: S_ISUID - - Set UID bit. - -.. data:: S_ISGID - - Set-group-ID bit. This bit has several special uses. For a directory - it indicates that BSD semantics is to be used for that directory: - files created there inherit their group ID from the directory, not - from the effective group ID of the creating process, and directories - created there will also get the :data:`S_ISGID` bit set. For a - file that does not have the group execution bit (:data:`S_IXGRP`) - set, the set-group-ID bit indicates mandatory file/record locking - (see also :data:`S_ENFMT`). - -.. data:: S_ISVTX - - Sticky bit. When this bit is set on a directory it means that a file - in that directory can be renamed or deleted only by the owner of the - file, by the owner of the directory, or by a privileged process. - -.. data:: S_IRWXU - - Mask for file owner permissions. - -.. data:: S_IRUSR - - Owner has read permission. - -.. data:: S_IWUSR - - Owner has write permission. - -.. data:: S_IXUSR - - Owner has execute permission. - -.. data:: S_IRWXG - - Mask for group permissions. - -.. data:: S_IRGRP - - Group has read permission. - -.. data:: S_IWGRP - - Group has write permission. - -.. data:: S_IXGRP - - Group has execute permission. - -.. data:: S_IRWXO - - Mask for permissions for others (not in group). - -.. data:: S_IROTH - - Others have read permission. - -.. data:: S_IWOTH - - Others have write permission. - -.. data:: S_IXOTH - - Others have execute permission. - -.. data:: S_ENFMT - - System V file locking enforcement. This flag is shared with :data:`S_ISGID`: - file/record locking is enforced on files that do not have the group - execution bit (:data:`S_IXGRP`) set. - -.. data:: S_IREAD - - Unix V7 synonym for :data:`S_IRUSR`. - -.. data:: S_IWRITE - - Unix V7 synonym for :data:`S_IWUSR`. - -.. data:: S_IEXEC - - Unix V7 synonym for :data:`S_IXUSR`. - -The following flags can be used in the *flags* argument of :func:`os.chflags`: - -.. data:: UF_NODUMP - - Do not dump the file. - -.. data:: UF_IMMUTABLE - - The file may not be changed. - -.. data:: UF_APPEND - - The file may only be appended to. - -.. data:: UF_OPAQUE - - The directory is opaque when viewed through a union stack. - -.. data:: UF_NOUNLINK - - The file may not be renamed or deleted. - -.. data:: UF_COMPRESSED - - The file is stored compressed (Mac OS X 10.6+). - -.. data:: UF_HIDDEN - - The file should not be displayed in a GUI (Mac OS X 10.5+). - -.. data:: SF_ARCHIVED - - The file may be archived. - -.. data:: SF_IMMUTABLE - - The file may not be changed. - -.. data:: SF_APPEND - - The file may only be appended to. - -.. data:: SF_NOUNLINK - - The file may not be renamed or deleted. - -.. data:: SF_SNAPSHOT - - The file is a snapshot file. - -See the \*BSD or Mac OS systems man page :manpage:`chflags(2)` for more information. - -On Windows, the following file attribute constants are available for use when -testing bits in the ``st_file_attributes`` member returned by :func:`os.stat`. -See the `Windows API documentation -`_ -for more detail on the meaning of these constants. - -.. data:: FILE_ATTRIBUTE_ARCHIVE - FILE_ATTRIBUTE_COMPRESSED - FILE_ATTRIBUTE_DEVICE - FILE_ATTRIBUTE_DIRECTORY - FILE_ATTRIBUTE_ENCRYPTED - FILE_ATTRIBUTE_HIDDEN - FILE_ATTRIBUTE_INTEGRITY_STREAM - FILE_ATTRIBUTE_NORMAL - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED - FILE_ATTRIBUTE_NO_SCRUB_DATA - FILE_ATTRIBUTE_OFFLINE - FILE_ATTRIBUTE_READONLY - FILE_ATTRIBUTE_REPARSE_POINT - FILE_ATTRIBUTE_SPARSE_FILE - FILE_ATTRIBUTE_SYSTEM - FILE_ATTRIBUTE_TEMPORARY - FILE_ATTRIBUTE_VIRTUAL - - .. versionadded:: 3.5 diff --git a/third_party/python/Doc/library/statistics.rst b/third_party/python/Doc/library/statistics.rst deleted file mode 100644 index 652e75136..000000000 --- a/third_party/python/Doc/library/statistics.rst +++ /dev/null @@ -1,456 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: mathematical statistics functions - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/statistics.py` - -.. testsetup:: * - - from statistics import * - __name__ = '' - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`Real`-valued) data. - -.. note:: - - Unless explicitly noted otherwise, these functions support :class:`int`, - :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`. - Behaviour with other types (whether in the numeric tower or not) is - currently unsupported. Mixed types are also undefined and - implementation-dependent. If your input data consists of mixed types, - you may be able to use :func:`map` to ensure a consistent result, e.g. - ``map(float, input_data)``. - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= ============================================= -:func:`mean` Arithmetic mean ("average") of data. -:func:`harmonic_mean` Harmonic mean of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Mode (most common value) of discrete data. -======================= ============================================= - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data* which can be a sequence or iterator. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of the - data points. For more robust, although less efficient, measures of - central location, see :func:`median` and :func:`mode`. (In this case, - "efficient" refers to statistical efficiency rather than computational - efficiency.) - - The sample mean gives an unbiased estimate of the true population mean, - which means that, taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: harmonic_mean(data) - - Return the harmonic mean of *data*, a sequence or iterator of - real-valued numbers. - - The harmonic mean, sometimes called the subcontrary mean, is the - reciprocal of the arithmetic :func:`mean` of the reciprocals of the - data. For example, the harmonic mean of three values *a*, *b* and *c* - will be equivalent to ``3/(1/a + 1/b + 1/c)``. - - The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging quantities - which are rates or ratios, for example speeds. For example: - - Suppose an investor purchases an equal value of shares in each of - three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. - What is the average P/E ratio for the investor's portfolio? - - .. doctest:: - - >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. - 3.6 - - Using the arithmetic mean would give an average of about 5.167, which - is too high. - - :exc:`StatisticsError` is raised if *data* is empty, or any element - is less than zero. - - .. versionadded:: 3.6 - - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterator. - - The median is a robust measure of central location, and is less affected by - the presence of outliers in your data. When the number of data points is - odd, the middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - If your data is ordinal (supports order operations) but not numeric (doesn't - support addition), you should use :func:`median_low` or :func:`median_high` - instead. - - .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` - - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterator. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * Calculating the `median `_. - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the most common data point from discrete or nominal *data*. The mode - (when it exists) is the most typical value, and is a robust measure of - central location. - - If *data* is empty, or if there is not exactly one most common value, - :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data, and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic which also applies - to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty iterable of real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *mu* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - This function does not attempt to verify that you have passed the actual mean - as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible - results. - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this function - to calculate the variance of a sample, giving the known population mean as - the second argument. Provided the data points are representative - (e.g. independent and identically distributed), the result will be an - unbiased estimate of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/third_party/python/Doc/library/stdtypes.rst b/third_party/python/Doc/library/stdtypes.rst deleted file mode 100644 index 00e1f4c1b..000000000 --- a/third_party/python/Doc/library/stdtypes.rst +++ /dev/null @@ -1,4688 +0,0 @@ -.. XXX: reference/datamodel and this have quite a few overlaps! - - -.. _bltin-types: - -************** -Built-in Types -************** - -The following sections describe the standard types that are built into the -interpreter. - -.. index:: pair: built-in; types - -The principal built-in types are numerics, sequences, mappings, classes, -instances and exceptions. - -Some collection classes are mutable. The methods that add, subtract, or -rearrange their members in place, and don't return a specific item, never return -the collection instance itself but ``None``. - -Some operations are supported by several object types; in particular, -practically all objects can be compared, tested for truth value, and converted -to a string (with the :func:`repr` function or the slightly different -:func:`str` function). The latter function is implicitly used when an object is -written by the :func:`print` function. - - -.. _truth: - -Truth Value Testing -=================== - -.. index:: - statement: if - statement: while - pair: truth; value - pair: Boolean; operations - single: false - -Any object can be tested for truth value, for use in an :keyword:`if` or -:keyword:`while` condition or as operand of the Boolean operations below. - -.. index:: single: true - -By default, an object is considered true unless its class defines either a -:meth:`__bool__` method that returns ``False`` or a :meth:`__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: - - .. index:: - single: None (Built-in object) - single: False (Built-in object) - -* constants defined to be false: ``None`` and ``False``. - -* zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, - ``Fraction(0, 1)`` - -* empty sequences and collections: ``''``, ``()``, ``[]``, ``{}``, ``set()``, - ``range(0)`` - -.. index:: - operator: or - operator: and - single: False - single: True - -Operations and built-in functions that have a Boolean result always return ``0`` -or ``False`` for false and ``1`` or ``True`` for true, unless otherwise stated. -(Important exception: the Boolean operations ``or`` and ``and`` always return -one of their operands.) - - -.. _boolean: - -Boolean Operations --- :keyword:`and`, :keyword:`or`, :keyword:`not` -==================================================================== - -.. index:: pair: Boolean; operations - -These are the Boolean operations, ordered by ascending priority: - -+-------------+---------------------------------+-------+ -| Operation | Result | Notes | -+=============+=================================+=======+ -| ``x or y`` | if *x* is false, then *y*, else | \(1) | -| | *x* | | -+-------------+---------------------------------+-------+ -| ``x and y`` | if *x* is false, then *x*, else | \(2) | -| | *y* | | -+-------------+---------------------------------+-------+ -| ``not x`` | if *x* is false, then ``True``, | \(3) | -| | else ``False`` | | -+-------------+---------------------------------+-------+ - -.. index:: - operator: and - operator: or - operator: not - -Notes: - -(1) - This is a short-circuit operator, so it only evaluates the second - argument if the first one is false. - -(2) - This is a short-circuit operator, so it only evaluates the second - argument if the first one is true. - -(3) - ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is - interpreted as ``not (a == b)``, and ``a == not b`` is a syntax error. - - -.. _stdcomparisons: - -Comparisons -=========== - -.. index:: - pair: chaining; comparisons - pair: operator; comparison - operator: == - operator: < (less) - operator: <= - operator: > (greater) - operator: >= - operator: != - operator: is - operator: is not - -There are eight comparison operations in Python. They all have the same -priority (which is higher than that of the Boolean operations). Comparisons can -be chained arbitrarily; for example, ``x < y <= z`` is equivalent to ``x < y and -y <= z``, except that *y* is evaluated only once (but in both cases *z* is not -evaluated at all when ``x < y`` is found to be false). - -This table summarizes the comparison operations: - -+------------+-------------------------+ -| Operation | Meaning | -+============+=========================+ -| ``<`` | strictly less than | -+------------+-------------------------+ -| ``<=`` | less than or equal | -+------------+-------------------------+ -| ``>`` | strictly greater than | -+------------+-------------------------+ -| ``>=`` | greater than or equal | -+------------+-------------------------+ -| ``==`` | equal | -+------------+-------------------------+ -| ``!=`` | not equal | -+------------+-------------------------+ -| ``is`` | object identity | -+------------+-------------------------+ -| ``is not`` | negated object identity | -+------------+-------------------------+ - -.. index:: - pair: object; numeric - pair: objects; comparing - -Objects of different types, except different numeric types, never compare equal. -Furthermore, some types (for example, function objects) support only a degenerate -notion of comparison where any two objects of that type are unequal. The ``<``, -``<=``, ``>`` and ``>=`` operators will raise a :exc:`TypeError` exception when -comparing a complex number with another built-in numeric type, when the objects -are of different types that cannot be compared, or in other cases where there is -no defined ordering. - -.. index:: - single: __eq__() (instance method) - single: __ne__() (instance method) - single: __lt__() (instance method) - single: __le__() (instance method) - single: __gt__() (instance method) - single: __ge__() (instance method) - -Non-identical instances of a class normally compare as non-equal unless the -class defines the :meth:`__eq__` method. - -Instances of a class cannot be ordered with respect to other instances of the -same class, or other types of object, unless the class defines enough of the -methods :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` (in -general, :meth:`__lt__` and :meth:`__eq__` are sufficient, if you want the -conventional meanings of the comparison operators). - -The behavior of the :keyword:`is` and :keyword:`is not` operators cannot be -customized; also they can be applied to any two objects and never raise an -exception. - -.. index:: - operator: in - operator: not in - -Two more operations with the same syntactic priority, :keyword:`in` and -:keyword:`not in`, are supported by types that are :term:`iterable` or -implement the :meth:`__contains__` method. - -.. _typesnumeric: - -Numeric Types --- :class:`int`, :class:`float`, :class:`complex` -================================================================ - -.. index:: - object: numeric - object: Boolean - object: integer - object: floating point - object: complex number - pair: C; language - -There are three distinct numeric types: :dfn:`integers`, :dfn:`floating -point numbers`, and :dfn:`complex numbers`. In addition, Booleans are a -subtype of integers. Integers have unlimited precision. Floating point -numbers are usually implemented using :c:type:`double` in C; information -about the precision and internal representation of floating point -numbers for the machine on which your program is running is available -in :data:`sys.float_info`. Complex numbers have a real and imaginary -part, which are each a floating point number. To extract these parts -from a complex number *z*, use ``z.real`` and ``z.imag``. (The standard -library includes additional numeric types, :mod:`fractions` that hold -rationals, and :mod:`decimal` that hold floating-point numbers with -user-definable precision.) - -.. index:: - pair: numeric; literals - pair: integer; literals - pair: floating point; literals - pair: complex number; literals - pair: hexadecimal; literals - pair: octal; literals - pair: binary; literals - -Numbers are created by numeric literals or as the result of built-in functions -and operators. Unadorned integer literals (including hex, octal and binary -numbers) yield integers. Numeric literals containing a decimal point or an -exponent sign yield floating point numbers. Appending ``'j'`` or ``'J'`` to a -numeric literal yields an imaginary number (a complex number with a zero real -part) which you can add to an integer or float to get a complex number with real -and imaginary parts. - -.. index:: - single: arithmetic - builtin: int - builtin: float - builtin: complex - single: operator; + (plus) - single: + (plus); unary operator - single: + (plus); binary operator - single: operator; - (minus) - single: - (minus); unary operator - single: - (minus); binary operator - operator: * (asterisk) - operator: / (slash) - operator: // - operator: % (percent) - operator: ** - -Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point, -which is narrower than complex. Comparisons between numbers of mixed type use -the same rule. [2]_ The constructors :func:`int`, :func:`float`, and -:func:`complex` can be used to produce numbers of a specific type. - -All numeric types (except complex) support the following operations, sorted by -ascending priority (all numeric operations have a higher priority than -comparison operations): - -+---------------------+---------------------------------+---------+--------------------+ -| Operation | Result | Notes | Full documentation | -+=====================+=================================+=========+====================+ -| ``x + y`` | sum of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x - y`` | difference of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x * y`` | product of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x / y`` | quotient of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x // y`` | floored quotient of *x* and | \(1) | | -| | *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x % y`` | remainder of ``x / y`` | \(2) | | -+---------------------+---------------------------------+---------+--------------------+ -| ``-x`` | *x* negated | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``+x`` | *x* unchanged | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``abs(x)`` | absolute value or magnitude of | | :func:`abs` | -| | *x* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``int(x)`` | *x* converted to integer | \(3)\(6)| :func:`int` | -+---------------------+---------------------------------+---------+--------------------+ -| ``float(x)`` | *x* converted to floating point | \(4)\(6)| :func:`float` | -+---------------------+---------------------------------+---------+--------------------+ -| ``complex(re, im)`` | a complex number with real part | \(6) | :func:`complex` | -| | *re*, imaginary part *im*. | | | -| | *im* defaults to zero. | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``c.conjugate()`` | conjugate of the complex number | | | -| | *c* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``divmod(x, y)`` | the pair ``(x // y, x % y)`` | \(2) | :func:`divmod` | -+---------------------+---------------------------------+---------+--------------------+ -| ``pow(x, y)`` | *x* to the power *y* | \(5) | :func:`pow` | -+---------------------+---------------------------------+---------+--------------------+ -| ``x ** y`` | *x* to the power *y* | \(5) | | -+---------------------+---------------------------------+---------+--------------------+ - -.. index:: - triple: operations on; numeric; types - single: conjugate() (complex number method) - -Notes: - -(1) - Also referred to as integer division. The resultant value is a whole - integer, though the result's type is not necessarily int. The result is - always rounded towards minus infinity: ``1//2`` is ``0``, ``(-1)//2`` is - ``-1``, ``1//(-2)`` is ``-1``, and ``(-1)//(-2)`` is ``0``. - -(2) - Not for complex numbers. Instead convert to floats using :func:`abs` if - appropriate. - -(3) - .. index:: - module: math - single: floor() (in module math) - single: ceil() (in module math) - single: trunc() (in module math) - pair: numeric; conversions - pair: C; language - - Conversion from floating point to integer may round or truncate - as in C; see functions :func:`math.floor` and :func:`math.ceil` for - well-defined conversions. - -(4) - float also accepts the strings "nan" and "inf" with an optional prefix "+" - or "-" for Not a Number (NaN) and positive or negative infinity. - -(5) - Python defines ``pow(0, 0)`` and ``0 ** 0`` to be ``1``, as is common for - programming languages. - -(6) - The numeric literals accepted include the digits ``0`` to ``9`` or any - Unicode equivalent (code points with the ``Nd`` property). - - See http://www.unicode.org/Public/9.0.0/ucd/extracted/DerivedNumericType.txt - for a complete list of code points with the ``Nd`` property. - - -All :class:`numbers.Real` types (:class:`int` and :class:`float`) also include -the following operations: - -+--------------------+---------------------------------------------+ -| Operation | Result | -+====================+=============================================+ -| :func:`math.trunc(\| *x* truncated to :class:`~numbers.Integral` | -| x) ` | | -+--------------------+---------------------------------------------+ -| :func:`round(x[, | *x* rounded to *n* digits, | -| n]) ` | rounding half to even. If *n* is | -| | omitted, it defaults to 0. | -+--------------------+---------------------------------------------+ -| :func:`math.floor(\| the greatest :class:`~numbers.Integral` | -| x) ` | <= *x* | -+--------------------+---------------------------------------------+ -| :func:`math.ceil(x)| the least :class:`~numbers.Integral` >= *x* | -| ` | | -+--------------------+---------------------------------------------+ - -For additional numeric operations see the :mod:`math` and :mod:`cmath` -modules. - -.. XXXJH exceptions: overflow (when? what operations?) zerodivision - - -.. _bitstring-ops: - -Bitwise Operations on Integer Types ------------------------------------ - -.. index:: - triple: operations on; integer; types - pair: bitwise; operations - pair: shifting; operations - pair: masking; operations - operator: | (vertical bar) - operator: ^ (caret) - operator: & (ampersand) - operator: << - operator: >> - operator: ~ (tilde) - -Bitwise operations only make sense for integers. The result of bitwise -operations is calculated as though carried out in two's complement with an -infinite number of sign bits. - -The priorities of the binary bitwise operations are all lower than the numeric -operations and higher than the comparisons; the unary operation ``~`` has the -same priority as the other unary numeric operations (``+`` and ``-``). - -This table lists the bitwise operations sorted in ascending priority: - -+------------+--------------------------------+----------+ -| Operation | Result | Notes | -+============+================================+==========+ -| ``x | y`` | bitwise :dfn:`or` of *x* and | \(4) | -| | *y* | | -+------------+--------------------------------+----------+ -| ``x ^ y`` | bitwise :dfn:`exclusive or` of | \(4) | -| | *x* and *y* | | -+------------+--------------------------------+----------+ -| ``x & y`` | bitwise :dfn:`and` of *x* and | \(4) | -| | *y* | | -+------------+--------------------------------+----------+ -| ``x << n`` | *x* shifted left by *n* bits | (1)(2) | -+------------+--------------------------------+----------+ -| ``x >> n`` | *x* shifted right by *n* bits | (1)(3) | -+------------+--------------------------------+----------+ -| ``~x`` | the bits of *x* inverted | | -+------------+--------------------------------+----------+ - -Notes: - -(1) - Negative shift counts are illegal and cause a :exc:`ValueError` to be raised. - -(2) - A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)`` - without overflow check. - -(3) - A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without - overflow check. - -(4) - Performing these calculations with at least one extra sign extension bit in - a finite two's complement representation (a working bit-width of - ``1 + max(x.bit_length(), y.bit_length())`` or more) is sufficient to get the - same result as if there were an infinite number of sign bits. - - -Additional Methods on Integer Types ------------------------------------ - -The int type implements the :class:`numbers.Integral` :term:`abstract base -class`. In addition, it provides a few more methods: - -.. method:: int.bit_length() - - Return the number of bits necessary to represent an integer in binary, - excluding the sign and leading zeros:: - - >>> n = -37 - >>> bin(n) - '-0b100101' - >>> n.bit_length() - 6 - - More precisely, if ``x`` is nonzero, then ``x.bit_length()`` is the - unique positive integer ``k`` such that ``2**(k-1) <= abs(x) < 2**k``. - Equivalently, when ``abs(x)`` is small enough to have a correctly - rounded logarithm, then ``k = 1 + int(log(abs(x), 2))``. - If ``x`` is zero, then ``x.bit_length()`` returns ``0``. - - Equivalent to:: - - def bit_length(self): - s = bin(self) # binary representation: bin(-37) --> '-0b100101' - s = s.lstrip('-0b') # remove leading zeros and minus sign - return len(s) # len('100101') --> 6 - - .. versionadded:: 3.1 - -.. method:: int.to_bytes(length, byteorder, \*, signed=False) - - Return an array of bytes representing an integer. - - >>> (1024).to_bytes(2, byteorder='big') - b'\x04\x00' - >>> (1024).to_bytes(10, byteorder='big') - b'\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00' - >>> (-1024).to_bytes(10, byteorder='big', signed=True) - b'\xff\xff\xff\xff\xff\xff\xff\xff\xfc\x00' - >>> x = 1000 - >>> x.to_bytes((x.bit_length() + 7) // 8, byteorder='little') - b'\xe8\x03' - - The integer is represented using *length* bytes. An :exc:`OverflowError` - is raised if the integer is not representable with the given number of - bytes. - - The *byteorder* argument determines the byte order used to represent the - integer. If *byteorder* is ``"big"``, the most significant byte is at the - beginning of the byte array. If *byteorder* is ``"little"``, the most - significant byte is at the end of the byte array. To request the native - byte order of the host system, use :data:`sys.byteorder` as the byte order - value. - - The *signed* argument determines whether two's complement is used to - represent the integer. If *signed* is ``False`` and a negative integer is - given, an :exc:`OverflowError` is raised. The default value for *signed* - is ``False``. - - .. versionadded:: 3.2 - -.. classmethod:: int.from_bytes(bytes, byteorder, \*, signed=False) - - Return the integer represented by the given array of bytes. - - >>> int.from_bytes(b'\x00\x10', byteorder='big') - 16 - >>> int.from_bytes(b'\x00\x10', byteorder='little') - 4096 - >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=True) - -1024 - >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=False) - 64512 - >>> int.from_bytes([255, 0, 0], byteorder='big') - 16711680 - - The argument *bytes* must either be a :term:`bytes-like object` or an - iterable producing bytes. - - The *byteorder* argument determines the byte order used to represent the - integer. If *byteorder* is ``"big"``, the most significant byte is at the - beginning of the byte array. If *byteorder* is ``"little"``, the most - significant byte is at the end of the byte array. To request the native - byte order of the host system, use :data:`sys.byteorder` as the byte order - value. - - The *signed* argument indicates whether two's complement is used to - represent the integer. - - .. versionadded:: 3.2 - - -Additional Methods on Float ---------------------------- - -The float type implements the :class:`numbers.Real` :term:`abstract base -class`. float also has the following additional methods. - -.. method:: float.as_integer_ratio() - - Return a pair of integers whose ratio is exactly equal to the - original float and with a positive denominator. Raises - :exc:`OverflowError` on infinities and a :exc:`ValueError` on - NaNs. - -.. method:: float.is_integer() - - Return ``True`` if the float instance is finite with integral - value, and ``False`` otherwise:: - - >>> (-2.0).is_integer() - True - >>> (3.2).is_integer() - False - -Two methods support conversion to -and from hexadecimal strings. Since Python's floats are stored -internally as binary numbers, converting a float to or from a -*decimal* string usually involves a small rounding error. In -contrast, hexadecimal strings allow exact representation and -specification of floating-point numbers. This can be useful when -debugging, and in numerical work. - - -.. method:: float.hex() - - Return a representation of a floating-point number as a hexadecimal - string. For finite floating-point numbers, this representation - will always include a leading ``0x`` and a trailing ``p`` and - exponent. - - -.. classmethod:: float.fromhex(s) - - Class method to return the float represented by a hexadecimal - string *s*. The string *s* may have leading and trailing - whitespace. - - -Note that :meth:`float.hex` is an instance method, while -:meth:`float.fromhex` is a class method. - -A hexadecimal string takes the form:: - - [sign] ['0x'] integer ['.' fraction] ['p' exponent] - -where the optional ``sign`` may by either ``+`` or ``-``, ``integer`` -and ``fraction`` are strings of hexadecimal digits, and ``exponent`` -is a decimal integer with an optional leading sign. Case is not -significant, and there must be at least one hexadecimal digit in -either the integer or the fraction. This syntax is similar to the -syntax specified in section 6.4.4.2 of the C99 standard, and also to -the syntax used in Java 1.5 onwards. In particular, the output of -:meth:`float.hex` is usable as a hexadecimal floating-point literal in -C or Java code, and hexadecimal strings produced by C's ``%a`` format -character or Java's ``Double.toHexString`` are accepted by -:meth:`float.fromhex`. - - -Note that the exponent is written in decimal rather than hexadecimal, -and that it gives the power of 2 by which to multiply the coefficient. -For example, the hexadecimal string ``0x3.a7p10`` represents the -floating-point number ``(3 + 10./16 + 7./16**2) * 2.0**10``, or -``3740.0``:: - - >>> float.fromhex('0x3.a7p10') - 3740.0 - - -Applying the reverse conversion to ``3740.0`` gives a different -hexadecimal string representing the same number:: - - >>> float.hex(3740.0) - '0x1.d380000000000p+11' - - -.. _numeric-hash: - -Hashing of numeric types ------------------------- - -For numbers ``x`` and ``y``, possibly of different types, it's a requirement -that ``hash(x) == hash(y)`` whenever ``x == y`` (see the :meth:`__hash__` -method documentation for more details). For ease of implementation and -efficiency across a variety of numeric types (including :class:`int`, -:class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`) -Python's hash for numeric types is based on a single mathematical function -that's defined for any rational number, and hence applies to all instances of -:class:`int` and :class:`fractions.Fraction`, and all finite instances of -:class:`float` and :class:`decimal.Decimal`. Essentially, this function is -given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is -made available to Python as the :attr:`modulus` attribute of -:data:`sys.hash_info`. - -.. impl-detail:: - - Currently, the prime used is ``P = 2**31 - 1`` on machines with 32-bit C - longs and ``P = 2**61 - 1`` on machines with 64-bit C longs. - -Here are the rules in detail: - -- If ``x = m / n`` is a nonnegative rational number and ``n`` is not divisible - by ``P``, define ``hash(x)`` as ``m * invmod(n, P) % P``, where ``invmod(n, - P)`` gives the inverse of ``n`` modulo ``P``. - -- If ``x = m / n`` is a nonnegative rational number and ``n`` is - divisible by ``P`` (but ``m`` is not) then ``n`` has no inverse - modulo ``P`` and the rule above doesn't apply; in this case define - ``hash(x)`` to be the constant value ``sys.hash_info.inf``. - -- If ``x = m / n`` is a negative rational number define ``hash(x)`` - as ``-hash(-x)``. If the resulting hash is ``-1``, replace it with - ``-2``. - -- The particular values ``sys.hash_info.inf``, ``-sys.hash_info.inf`` - and ``sys.hash_info.nan`` are used as hash values for positive - infinity, negative infinity, or nans (respectively). (All hashable - nans have the same hash value.) - -- For a :class:`complex` number ``z``, the hash values of the real - and imaginary parts are combined by computing ``hash(z.real) + - sys.hash_info.imag * hash(z.imag)``, reduced modulo - ``2**sys.hash_info.width`` so that it lies in - ``range(-2**(sys.hash_info.width - 1), 2**(sys.hash_info.width - - 1))``. Again, if the result is ``-1``, it's replaced with ``-2``. - - -To clarify the above rules, here's some example Python code, -equivalent to the built-in hash, for computing the hash of a rational -number, :class:`float`, or :class:`complex`:: - - - import sys, math - - def hash_fraction(m, n): - """Compute the hash of a rational number m / n. - - Assumes m and n are integers, with n positive. - Equivalent to hash(fractions.Fraction(m, n)). - - """ - P = sys.hash_info.modulus - # Remove common factors of P. (Unnecessary if m and n already coprime.) - while m % P == n % P == 0: - m, n = m // P, n // P - - if n % P == 0: - hash_value = sys.hash_info.inf - else: - # Fermat's Little Theorem: pow(n, P-1, P) is 1, so - # pow(n, P-2, P) gives the inverse of n modulo P. - hash_value = (abs(m) % P) * pow(n, P - 2, P) % P - if m < 0: - hash_value = -hash_value - if hash_value == -1: - hash_value = -2 - return hash_value - - def hash_float(x): - """Compute the hash of a float x.""" - - if math.isnan(x): - return sys.hash_info.nan - elif math.isinf(x): - return sys.hash_info.inf if x > 0 else -sys.hash_info.inf - else: - return hash_fraction(*x.as_integer_ratio()) - - def hash_complex(z): - """Compute the hash of a complex number z.""" - - hash_value = hash_float(z.real) + sys.hash_info.imag * hash_float(z.imag) - # do a signed reduction modulo 2**sys.hash_info.width - M = 2**(sys.hash_info.width - 1) - hash_value = (hash_value & (M - 1)) - (hash_value & M) - if hash_value == -1: - hash_value = -2 - return hash_value - -.. _typeiter: - -Iterator Types -============== - -.. index:: - single: iterator protocol - single: protocol; iterator - single: sequence; iteration - single: container; iteration over - -Python supports a concept of iteration over containers. This is implemented -using two distinct methods; these are used to allow user-defined classes to -support iteration. Sequences, described below in more detail, always support -the iteration methods. - -One method needs to be defined for container objects to provide iteration -support: - -.. XXX duplicated in reference/datamodel! - -.. method:: container.__iter__() - - Return an iterator object. The object is required to support the iterator - protocol described below. If a container supports different types of - iteration, additional methods can be provided to specifically request - iterators for those iteration types. (An example of an object supporting - multiple forms of iteration would be a tree structure which supports both - breadth-first and depth-first traversal.) This method corresponds to the - :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C - API. - -The iterator objects themselves are required to support the following two -methods, which together form the :dfn:`iterator protocol`: - - -.. method:: iterator.__iter__() - - Return the iterator object itself. This is required to allow both containers - and iterators to be used with the :keyword:`for` and :keyword:`in` statements. - This method corresponds to the :c:member:`~PyTypeObject.tp_iter` slot of the type structure for - Python objects in the Python/C API. - - -.. method:: iterator.__next__() - - Return the next item from the container. If there are no further items, raise - the :exc:`StopIteration` exception. This method corresponds to the - :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for Python objects in the - Python/C API. - -Python defines several iterator objects to support iteration over general and -specific sequence types, dictionaries, and other more specialized forms. The -specific types are not important beyond their implementation of the iterator -protocol. - -Once an iterator's :meth:`~iterator.__next__` method raises -:exc:`StopIteration`, it must continue to do so on subsequent calls. -Implementations that do not obey this property are deemed broken. - - -.. _generator-types: - -Generator Types ---------------- - -Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`__iter__` method is implemented as a -generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` -methods. -More information about generators can be found in :ref:`the documentation for -the yield expression `. - - -.. _typesseq: - -Sequence Types --- :class:`list`, :class:`tuple`, :class:`range` -================================================================ - -There are three basic sequence types: lists, tuples, and range objects. -Additional sequence types tailored for processing of -:ref:`binary data ` and :ref:`text strings ` are -described in dedicated sections. - - -.. _typesseq-common: - -Common Sequence Operations --------------------------- - -.. index:: object: sequence - -The operations in the following table are supported by most sequence types, -both mutable and immutable. The :class:`collections.abc.Sequence` ABC is -provided to make it easier to correctly implement these operations on -custom sequence types. - -This table lists the sequence operations sorted in ascending priority. In the -table, *s* and *t* are sequences of the same type, *n*, *i*, *j* and *k* are -integers and *x* is an arbitrary object that meets any type and value -restrictions imposed by *s*. - -The ``in`` and ``not in`` operations have the same priorities as the -comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. [3]_ - -.. index:: - triple: operations on; sequence; types - builtin: len - builtin: min - builtin: max - pair: concatenation; operation - pair: repetition; operation - pair: subscript; operation - pair: slice; operation - operator: in - operator: not in - single: count() (sequence method) - single: index() (sequence method) - -+--------------------------+--------------------------------+----------+ -| Operation | Result | Notes | -+==========================+================================+==========+ -| ``x in s`` | ``True`` if an item of *s* is | \(1) | -| | equal to *x*, else ``False`` | | -+--------------------------+--------------------------------+----------+ -| ``x not in s`` | ``False`` if an item of *s* is | \(1) | -| | equal to *x*, else ``True`` | | -+--------------------------+--------------------------------+----------+ -| ``s + t`` | the concatenation of *s* and | (6)(7) | -| | *t* | | -+--------------------------+--------------------------------+----------+ -| ``s * n`` or | equivalent to adding *s* to | (2)(7) | -| ``n * s`` | itself *n* times | | -+--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) | -| | with step *k* | | -+--------------------------+--------------------------------+----------+ -| ``len(s)`` | length of *s* | | -+--------------------------+--------------------------------+----------+ -| ``min(s)`` | smallest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``max(s)`` | largest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ - -Sequences of the same type also support comparisons. In particular, tuples -and lists are compared lexicographically by comparing corresponding elements. -This means that to compare equal, every element must compare equal and the -two sequences must be of the same type and have the same length. (For full -details see :ref:`comparisons` in the language reference.) - -Notes: - -(1) - While the ``in`` and ``not in`` operations are used only for simple - containment testing in the general case, some specialised sequences - (such as :class:`str`, :class:`bytes` and :class:`bytearray`) also use - them for subsequence testing:: - - >>> "gg" in "eggs" - True - -(2) - Values of *n* less than ``0`` are treated as ``0`` (which yields an empty - sequence of the same type as *s*). Note that items in the sequence *s* - are not copied; they are referenced multiple times. This often haunts - new Python programmers; consider:: - - >>> lists = [[]] * 3 - >>> lists - [[], [], []] - >>> lists[0].append(3) - >>> lists - [[3], [3], [3]] - - What has happened is that ``[[]]`` is a one-element list containing an empty - list, so all three elements of ``[[]] * 3`` are references to this single empty - list. Modifying any of the elements of ``lists`` modifies this single list. - You can create a list of different lists this way:: - - >>> lists = [[] for i in range(3)] - >>> lists[0].append(3) - >>> lists[1].append(5) - >>> lists[2].append(7) - >>> lists - [[3], [5], [7]] - - Further explanation is available in the FAQ entry - :ref:`faq-multidimensional-list`. - -(3) - If *i* or *j* is negative, the index is relative to the end of sequence *s*: - ``len(s) + i`` or ``len(s) + j`` is substituted. But note that ``-0`` is - still ``0``. - -(4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. - -(5) - The slice of *s* from *i* to *j* with step *k* is defined as the sequence of - items with index ``x = i + n*k`` such that ``0 <= n < (j-i)/k``. In other words, - the indices are ``i``, ``i+k``, ``i+2*k``, ``i+3*k`` and so on, stopping when - *j* is reached (but never including *j*). When *k* is positive, - *i* and *j* are reduced to ``len(s)`` if they are greater. - When *k* is negative, *i* and *j* are reduced to ``len(s) - 1`` if - they are greater. If *i* or *j* are omitted or ``None``, they become - "end" values (which end depends on the sign of *k*). Note, *k* cannot be zero. - If *k* is ``None``, it is treated like ``1``. - -(6) - Concatenating immutable sequences always results in a new object. This - means that building up a sequence by repeated concatenation will have a - quadratic runtime cost in the total sequence length. To get a linear - runtime cost, you must switch to one of the alternatives below: - - * if concatenating :class:`str` objects, you can build a list and use - :meth:`str.join` at the end or else write to an :class:`io.StringIO` - instance and retrieve its value when complete - - * if concatenating :class:`bytes` objects, you can similarly use - :meth:`bytes.join` or :class:`io.BytesIO`, or you can do in-place - concatenation with a :class:`bytearray` object. :class:`bytearray` - objects are mutable and have an efficient overallocation mechanism - - * if concatenating :class:`tuple` objects, extend a :class:`list` instead - - * for other types, investigate the relevant class documentation - - -(7) - Some sequence types (such as :class:`range`) only support item sequences - that follow specific patterns, and hence don't support sequence - concatenation or repetition. - -(8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - Not all implementations support passing the additional arguments *i* and *j*. - These arguments allow efficient searching of subsections of the sequence. Passing - the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. - - -.. _typesseq-immutable: - -Immutable Sequence Types ------------------------- - -.. index:: - triple: immutable; sequence; types - object: tuple - builtin: hash - -The only operation that immutable sequence types generally implement that is -not also implemented by mutable sequence types is support for the :func:`hash` -built-in. - -This support allows immutable sequences, such as :class:`tuple` instances, to -be used as :class:`dict` keys and stored in :class:`set` and :class:`frozenset` -instances. - -Attempting to hash an immutable sequence that contains unhashable values will -result in :exc:`TypeError`. - - -.. _typesseq-mutable: - -Mutable Sequence Types ----------------------- - -.. index:: - triple: mutable; sequence; types - object: list - object: bytearray - -The operations in the following table are defined on mutable sequence types. -The :class:`collections.abc.MutableSequence` ABC is provided to make it -easier to correctly implement these operations on custom sequence types. - -In the table *s* is an instance of a mutable sequence type, *t* is any -iterable object and *x* is an arbitrary object that meets any type -and value restrictions imposed by *s* (for example, :class:`bytearray` only -accepts integers that meet the value restriction ``0 <= x <= 255``). - - -.. index:: - triple: operations on; sequence; types - triple: operations on; list; type - pair: subscript; assignment - pair: slice; assignment - statement: del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) - -+------------------------------+--------------------------------+---------------------+ -| Operation | Result | Notes | -+==============================+================================+=====================+ -| ``s[i] = x`` | item *i* of *s* is replaced by | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j] = t`` | slice of *s* from *i* to *j* | | -| | is replaced by the contents of | | -| | the iterable *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | -| | are replaced by those of *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j:k]`` | removes the elements of | | -| | ``s[i:j:k]`` from the list | | -+------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from *s* | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of *s* | \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | -| | most part the same as | | -| | ``s[len(s):len(s)] = t``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | -| | repeated *n* times | | -+------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop([i])`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i] == x`` | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - - -Notes: - -(1) - *t* must have the same length as the slice it is replacing. - -(2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. - -(3) - ``remove`` raises :exc:`ValueError` when *x* is not found in *s*. - -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. - -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`) - - .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. - -(6) - The value *n* is an integer, or an object implementing - :meth:`~object.__index__`. Zero and negative values of *n* clear - the sequence. Items in the sequence are not copied; they are referenced - multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. - - -.. _typesseq-list: - -Lists ------ - -.. index:: object: list - -Lists are mutable sequences, typically used to store collections of -homogeneous items (where the precise degree of similarity will vary by -application). - -.. class:: list([iterable]) - - Lists may be constructed in several ways: - - * Using a pair of square brackets to denote the empty list: ``[]`` - * Using square brackets, separating items with commas: ``[a]``, ``[a, b, c]`` - * Using a list comprehension: ``[x for x in iterable]`` - * Using the type constructor: ``list()`` or ``list(iterable)`` - - The constructor builds a list whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a list, a copy is made and returned, similar to ``iterable[:]``. - For example, ``list('abc')`` returns ``['a', 'b', 'c']`` and - ``list( (1, 2, 3) )`` returns ``[1, 2, 3]``. - If no argument is given, the constructor creates a new empty list, ``[]``. - - - Many other operations also produce lists, including the :func:`sorted` - built-in. - - Lists implement all of the :ref:`common ` and - :ref:`mutable ` sequence operations. Lists also provide the - following additional method: - - .. method:: list.sort(*, key=None, reverse=False) - - This method sorts the list in place, using only ``<`` comparisons - between items. Exceptions are not suppressed - if any comparison operations - fail, the entire sort operation will fail (and the list will likely be left - in a partially modified state). - - :meth:`sort` accepts two arguments that can only be passed by keyword - (:ref:`keyword-only arguments `): - - *key* specifies a function of one argument that is used to extract a - comparison key from each list element (for example, ``key=str.lower``). - The key corresponding to each item in the list is calculated once and - then used for the entire sorting process. The default value of ``None`` - means that list items are sorted directly without calculating a separate - key value. - - The :func:`functools.cmp_to_key` utility is available to convert a 2.x - style *cmp* function to a *key* function. - - *reverse* is a boolean value. If set to ``True``, then the list elements - are sorted as if each comparison were reversed. - - This method modifies the sequence in place for economy of space when - sorting a large sequence. To remind users that it operates by side - effect, it does not return the sorted sequence (use :func:`sorted` to - explicitly request a new sorted list instance). - - The :meth:`sort` method is guaranteed to be stable. A sort is stable if it - guarantees not to change the relative order of elements that compare equal - --- this is helpful for sorting in multiple passes (for example, sort by - department, then by salary grade). - - .. impl-detail:: - - While a list is being sorted, the effect of attempting to mutate, or even - inspect, the list is undefined. The C implementation of Python makes the - list appear empty for the duration, and raises :exc:`ValueError` if it can - detect that the list has been mutated during a sort. - - -.. _typesseq-tuple: - -Tuples ------- - -.. index:: object: tuple - -Tuples are immutable sequences, typically used to store collections of -heterogeneous data (such as the 2-tuples produced by the :func:`enumerate` -built-in). Tuples are also used for cases where an immutable sequence of -homogeneous data is needed (such as allowing storage in a :class:`set` or -:class:`dict` instance). - -.. class:: tuple([iterable]) - - Tuples may be constructed in a number of ways: - - * Using a pair of parentheses to denote the empty tuple: ``()`` - * Using a trailing comma for a singleton tuple: ``a,`` or ``(a,)`` - * Separating items with commas: ``a, b, c`` or ``(a, b, c)`` - * Using the :func:`tuple` built-in: ``tuple()`` or ``tuple(iterable)`` - - The constructor builds a tuple whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a tuple, it is returned unchanged. For example, - ``tuple('abc')`` returns ``('a', 'b', 'c')`` and - ``tuple( [1, 2, 3] )`` returns ``(1, 2, 3)``. - If no argument is given, the constructor creates a new empty tuple, ``()``. - - Note that it is actually the comma which makes a tuple, not the parentheses. - The parentheses are optional, except in the empty tuple case, or - when they are needed to avoid syntactic ambiguity. For example, - ``f(a, b, c)`` is a function call with three arguments, while - ``f((a, b, c))`` is a function call with a 3-tuple as the sole argument. - - Tuples implement all of the :ref:`common ` sequence - operations. - -For heterogeneous collections of data where access by name is clearer than -access by index, :func:`collections.namedtuple` may be a more appropriate -choice than a simple tuple object. - - -.. _typesseq-range: - -Ranges ------- - -.. index:: object: range - -The :class:`range` type represents an immutable sequence of numbers and is -commonly used for looping a specific number of times in :keyword:`for` -loops. - -.. class:: range(stop) - range(start, stop[, step]) - - The arguments to the range constructor must be integers (either built-in - :class:`int` or any object that implements the ``__index__`` special - method). If the *step* argument is omitted, it defaults to ``1``. - If the *start* argument is omitted, it defaults to ``0``. - If *step* is zero, :exc:`ValueError` is raised. - - For a positive *step*, the contents of a range ``r`` are determined by the - formula ``r[i] = start + step*i`` where ``i >= 0`` and - ``r[i] < stop``. - - For a negative *step*, the contents of the range are still determined by - the formula ``r[i] = start + step*i``, but the constraints are ``i >= 0`` - and ``r[i] > stop``. - - A range object will be empty if ``r[0]`` does not meet the value - constraint. Ranges do support negative indices, but these are interpreted - as indexing from the end of the sequence determined by the positive - indices. - - Ranges containing absolute values larger than :data:`sys.maxsize` are - permitted but some features (such as :func:`len`) may raise - :exc:`OverflowError`. - - Range examples:: - - >>> list(range(10)) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> list(range(1, 11)) - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - >>> list(range(0, 30, 5)) - [0, 5, 10, 15, 20, 25] - >>> list(range(0, 10, 3)) - [0, 3, 6, 9] - >>> list(range(0, -10, -1)) - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] - >>> list(range(0)) - [] - >>> list(range(1, 0)) - [] - - Ranges implement all of the :ref:`common ` sequence operations - except concatenation and repetition (due to the fact that range objects can - only represent sequences that follow a strict pattern and repetition and - concatenation will usually violate that pattern). - - .. attribute:: start - - The value of the *start* parameter (or ``0`` if the parameter was - not supplied) - - .. attribute:: stop - - The value of the *stop* parameter - - .. attribute:: step - - The value of the *step* parameter (or ``1`` if the parameter was - not supplied) - -The advantage of the :class:`range` type over a regular :class:`list` or -:class:`tuple` is that a :class:`range` object will always take the same -(small) amount of memory, no matter the size of the range it represents (as it -only stores the ``start``, ``stop`` and ``step`` values, calculating individual -items and subranges as needed). - -Range objects implement the :class:`collections.abc.Sequence` ABC, and provide -features such as containment tests, element index lookup, slicing and -support for negative indices (see :ref:`typesseq`): - - >>> r = range(0, 20, 2) - >>> r - range(0, 20, 2) - >>> 11 in r - False - >>> 10 in r - True - >>> r.index(10) - 5 - >>> r[5] - 10 - >>> r[:5] - range(0, 10, 2) - >>> r[-1] - 18 - -Testing range objects for equality with ``==`` and ``!=`` compares -them as sequences. That is, two range objects are considered equal if -they represent the same sequence of values. (Note that two range -objects that compare equal might have different :attr:`~range.start`, -:attr:`~range.stop` and :attr:`~range.step` attributes, for example -``range(0) == range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.) - -.. versionchanged:: 3.2 - Implement the Sequence ABC. - Support slicing and negative indices. - Test :class:`int` objects for membership in constant time instead of - iterating through all items. - -.. versionchanged:: 3.3 - Define '==' and '!=' to compare range objects based on the - sequence of values they define (instead of comparing based on - object identity). - -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` - attributes. - -.. seealso:: - - * The `linspace recipe `_ - shows how to implement a lazy version of range suitable for floating - point applications. - -.. index:: - single: string; text sequence type - single: str (built-in class); (see also string) - object: string - -.. _textseq: - -Text Sequence Type --- :class:`str` -=================================== - -Textual data in Python is handled with :class:`str` objects, or :dfn:`strings`. -Strings are immutable -:ref:`sequences ` of Unicode code points. String literals are -written in a variety of ways: - -* Single quotes: ``'allows embedded "double" quotes'`` -* Double quotes: ``"allows embedded 'single' quotes"``. -* Triple quoted: ``'''Three single quotes'''``, ``"""Three double quotes"""`` - -Triple quoted strings may span multiple lines - all associated whitespace will -be included in the string literal. - -String literals that are part of a single expression and have only whitespace -between them will be implicitly converted to a single string literal. That -is, ``("spam " "eggs") == "spam eggs"``. - -See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that -disables most escape sequence processing. - -Strings may also be created from other objects using the :class:`str` -constructor. - -Since there is no separate "character" type, indexing a string produces -strings of length 1. That is, for a non-empty string *s*, ``s[0] == s[0:1]``. - -.. index:: - object: io.StringIO - -There is also no mutable string type, but :meth:`str.join` or -:class:`io.StringIO` can be used to efficiently construct strings from -multiple fragments. - -.. versionchanged:: 3.3 - For backwards compatibility with the Python 2 series, the ``u`` prefix is - once again permitted on string literals. It has no effect on the meaning - of string literals and cannot be combined with the ``r`` prefix. - - -.. index:: - single: string; str (built-in class) - -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - - Return a :ref:`string ` version of *object*. If *object* is not - provided, returns the empty string. Otherwise, the behavior of ``str()`` - depends on whether *encoding* or *errors* is given, as follows. - - If neither *encoding* nor *errors* is given, ``str(object)`` returns - :meth:`object.__str__() `, which is the "informal" or nicely - printable string representation of *object*. For string objects, this is - the string itself. If *object* does not have a :meth:`~object.__str__` - method, then :func:`str` falls back to returning - :meth:`repr(object) `. - - .. index:: - single: buffer protocol; str (built-in class) - single: bytes; str (built-in class) - - If at least one of *encoding* or *errors* is given, *object* should be a - :term:`bytes-like object` (e.g. :class:`bytes` or :class:`bytearray`). In - this case, if *object* is a :class:`bytes` (or :class:`bytearray`) object, - then ``str(bytes, encoding, errors)`` is equivalent to - :meth:`bytes.decode(encoding, errors) `. Otherwise, the bytes - object underlying the buffer object is obtained before calling - :meth:`bytes.decode`. See :ref:`binaryseq` and - :ref:`bufferobjects` for information on buffer objects. - - Passing a :class:`bytes` object to :func:`str` without the *encoding* - or *errors* arguments falls under the first case of returning the informal - string representation (see also the :option:`-b` command-line option to - Python). For example:: - - >>> str(b'Zoot!') - "b'Zoot!'" - - For more information on the ``str`` class and its methods, see - :ref:`textseq` and the :ref:`string-methods` section below. To output - formatted strings, see the :ref:`f-strings` and :ref:`formatstrings` - sections. In addition, see the :ref:`stringservices` section. - - -.. index:: - pair: string; methods - -.. _string-methods: - -String Methods --------------- - -.. index:: - module: re - -Strings implement all of the :ref:`common ` sequence -operations, along with the additional methods described below. - -Strings also support two styles of string formatting, one providing a large -degree of flexibility and customization (see :meth:`str.format`, -:ref:`formatstrings` and :ref:`string-formatting`) and the other based on C -``printf`` style formatting that handles a narrower range of types and is -slightly harder to use correctly, but is often faster for the cases it can -handle (:ref:`old-string-formatting`). - -The :ref:`textservices` section of the standard library covers a number of -other modules that provide various text related utilities (including regular -expression support in the :mod:`re` module). - -.. method:: str.capitalize() - - Return a copy of the string with its first character capitalized and the - rest lowercased. - - -.. method:: str.casefold() - - Return a casefolded copy of the string. Casefolded strings may be used for - caseless matching. - - Casefolding is similar to lowercasing but more aggressive because it is - intended to remove all case distinctions in a string. For example, the German - lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already - lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` - converts it to ``"ss"``. - - The casefolding algorithm is described in section 3.13 of the Unicode - Standard. - - .. versionadded:: 3.3 - - -.. method:: str.center(width[, fillchar]) - - Return centered in a string of length *width*. Padding is done using the - specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. - - - -.. method:: str.count(sub[, start[, end]]) - - Return the number of non-overlapping occurrences of substring *sub* in the - range [*start*, *end*]. Optional arguments *start* and *end* are - interpreted as in slice notation. - - -.. method:: str.encode(encoding="utf-8", errors="strict") - - Return an encoded version of the string as a bytes object. Default encoding - is ``'utf-8'``. *errors* may be given to set a different error handling scheme. - The default for *errors* is ``'strict'``, meaning that encoding errors raise - a :exc:`UnicodeError`. Other possible - values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, - ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - .. versionchanged:: 3.1 - Support for keyword arguments added. - - -.. method:: str.endswith(suffix[, start[, end]]) - - Return ``True`` if the string ends with the specified *suffix*, otherwise return - ``False``. *suffix* can also be a tuple of suffixes to look for. With optional - *start*, test beginning at that position. With optional *end*, stop comparing - at that position. - - -.. method:: str.expandtabs(tabsize=8) - - Return a copy of the string where all tab characters are replaced by one or - more spaces, depending on the current column and the given tab size. Tab - positions occur every *tabsize* characters (default is 8, giving tab - positions at columns 0, 8, 16 and so on). To expand the string, the current - column is set to zero and the string is examined character by character. If - the character is a tab (``\t``), one or more space characters are inserted - in the result until the current column is equal to the next tab position. - (The tab character itself is not copied.) If the character is a newline - (``\n``) or return (``\r``), it is copied and the current column is reset to - zero. Any other character is copied unchanged and the current column is - incremented by one regardless of how the character is represented when - printed. - - >>> '01\t012\t0123\t01234'.expandtabs() - '01 012 0123 01234' - >>> '01\t012\t0123\t01234'.expandtabs(4) - '01 012 0123 01234' - - -.. method:: str.find(sub[, start[, end]]) - - Return the lowest index in the string where substring *sub* is found within - the slice ``s[start:end]``. Optional arguments *start* and *end* are - interpreted as in slice notation. Return ``-1`` if *sub* is not found. - - .. note:: - - The :meth:`~str.find` method should be used only if you need to know the - position of *sub*. To check if *sub* is a substring or not, use the - :keyword:`in` operator:: - - >>> 'Py' in 'Python' - True - - -.. method:: str.format(*args, **kwargs) - - Perform a string formatting operation. The string on which this method is - called can contain literal text or replacement fields delimited by braces - ``{}``. Each replacement field contains either the numeric index of a - positional argument, or the name of a keyword argument. Returns a copy of - the string where each replacement field is replaced with the string value of - the corresponding argument. - - >>> "The sum of 1 + 2 is {0}".format(1+2) - 'The sum of 1 + 2 is 3' - - See :ref:`formatstrings` for a description of the various formatting options - that can be specified in format strings. - - .. note:: - When formatting a number (:class:`int`, :class:`float`, :class:`complex`, - :class:`decimal.Decimal` and subclasses) with the ``n`` type - (ex: ``'{:n}'.format(1234)``), the function temporarily sets the - ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale to decode - ``decimal_point`` and ``thousands_sep`` fields of :c:func:`localeconv` if - they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC`` locale is - different than the ``LC_CTYPE`` locale. This temporary change affects - other threads. - - .. versionchanged:: 3.6.5 - When formatting a number with the ``n`` type, the function sets - temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some - cases. - - -.. method:: str.format_map(mapping) - - Similar to ``str.format(**mapping)``, except that ``mapping`` is - used directly and not copied to a :class:`dict`. This is useful - if for example ``mapping`` is a dict subclass: - - >>> class Default(dict): - ... def __missing__(self, key): - ... return key - ... - >>> '{name} was born in {country}'.format_map(Default(name='Guido')) - 'Guido was born in country' - - .. versionadded:: 3.2 - - -.. method:: str.index(sub[, start[, end]]) - - Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. - - -.. method:: str.isalnum() - - Return true if all characters in the string are alphanumeric and there is at - least one character, false otherwise. A character ``c`` is alphanumeric if one - of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. - - -.. method:: str.isalpha() - - Return true if all characters in the string are alphabetic and there is at least - one character, false otherwise. Alphabetic characters are those characters defined - in the Unicode character database as "Letter", i.e., those with general category - property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different - from the "Alphabetic" property defined in the Unicode Standard. - - -.. method:: str.isdecimal() - - Return true if all characters in the string are decimal - characters and there is at least one character, false - otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT - ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". - - -.. method:: str.isdigit() - - Return true if all characters in the string are digits and there is at least one - character, false otherwise. Digits include decimal characters and digits that need - special handling, such as the compatibility superscript digits. - This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the - property value Numeric_Type=Digit or Numeric_Type=Decimal. - - -.. method:: str.isidentifier() - - Return true if the string is a valid identifier according to the language - definition, section :ref:`identifiers`. - - Use :func:`keyword.iskeyword` to test for reserved identifiers such as - :keyword:`def` and :keyword:`class`. - -.. method:: str.islower() - - Return true if all cased characters [4]_ in the string are lowercase and - there is at least one cased character, false otherwise. - - -.. method:: str.isnumeric() - - Return true if all characters in the string are numeric - characters, and there is at least one character, false - otherwise. Numeric characters include digit characters, and all characters - that have the Unicode numeric value property, e.g. U+2155, - VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property - value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. - - -.. method:: str.isprintable() - - Return true if all characters in the string are printable or the string is - empty, false otherwise. Nonprintable characters are those characters defined - in the Unicode character database as "Other" or "Separator", excepting the - ASCII space (0x20) which is considered printable. (Note that printable - characters in this context are those which should not be escaped when - :func:`repr` is invoked on a string. It has no bearing on the handling of - strings written to :data:`sys.stdout` or :data:`sys.stderr`.) - - -.. method:: str.isspace() - - Return true if there are only whitespace characters in the string and there is - at least one character, false otherwise. Whitespace characters are those - characters defined in the Unicode character database as "Other" or "Separator" - and those with bidirectional property being one of "WS", "B", or "S". - -.. method:: str.istitle() - - Return true if the string is a titlecased string and there is at least one - character, for example uppercase characters may only follow uncased characters - and lowercase characters only cased ones. Return false otherwise. - - -.. method:: str.isupper() - - Return true if all cased characters [4]_ in the string are uppercase and - there is at least one cased character, false otherwise. - - -.. method:: str.join(iterable) - - Return a string which is the concatenation of the strings in *iterable*. - A :exc:`TypeError` will be raised if there are any non-string values in - *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. - - -.. method:: str.ljust(width[, fillchar]) - - Return the string left justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. - - -.. method:: str.lower() - - Return a copy of the string with all the cased characters [4]_ converted to - lowercase. - - The lowercasing algorithm used is described in section 3.13 of the Unicode - Standard. - - -.. method:: str.lstrip([chars]) - - Return a copy of the string with leading characters removed. The *chars* - argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a prefix; rather, all combinations of its values are stripped:: - - >>> ' spacious '.lstrip() - 'spacious ' - >>> 'www.example.com'.lstrip('cmowz.') - 'example.com' - - -.. staticmethod:: str.maketrans(x[, y[, z]]) - - This static method returns a translation table usable for :meth:`str.translate`. - - If there is only one argument, it must be a dictionary mapping Unicode - ordinals (integers) or characters (strings of length 1) to Unicode ordinals, - strings (of arbitrary lengths) or ``None``. Character keys will then be - converted to ordinals. - - If there are two arguments, they must be strings of equal length, and in the - resulting dictionary, each character in x will be mapped to the character at - the same position in y. If there is a third argument, it must be a string, - whose characters will be mapped to ``None`` in the result. - - -.. method:: str.partition(sep) - - Split the string at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself, and the part - after the separator. If the separator is not found, return a 3-tuple containing - the string itself, followed by two empty strings. - - -.. method:: str.replace(old, new[, count]) - - Return a copy of the string with all occurrences of substring *old* replaced by - *new*. If the optional argument *count* is given, only the first *count* - occurrences are replaced. - - -.. method:: str.rfind(sub[, start[, end]]) - - Return the highest index in the string where substring *sub* is found, such - that *sub* is contained within ``s[start:end]``. Optional arguments *start* - and *end* are interpreted as in slice notation. Return ``-1`` on failure. - - -.. method:: str.rindex(sub[, start[, end]]) - - Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not - found. - - -.. method:: str.rjust(width[, fillchar]) - - Return the string right justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. - - -.. method:: str.rpartition(sep) - - Split the string at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself, and the part - after the separator. If the separator is not found, return a 3-tuple containing - two empty strings, followed by the string itself. - - -.. method:: str.rsplit(sep=None, maxsplit=-1) - - Return a list of the words in the string, using *sep* as the delimiter string. - If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* - ones. If *sep* is not specified or ``None``, any whitespace string is a - separator. Except for splitting from the right, :meth:`rsplit` behaves like - :meth:`split` which is described in detail below. - - -.. method:: str.rstrip([chars]) - - Return a copy of the string with trailing characters removed. The *chars* - argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: - - >>> ' spacious '.rstrip() - ' spacious' - >>> 'mississippi'.rstrip('ipz') - 'mississ' - - -.. method:: str.split(sep=None, maxsplit=-1) - - Return a list of the words in the string, using *sep* as the delimiter - string. If *maxsplit* is given, at most *maxsplit* splits are done (thus, - the list will have at most ``maxsplit+1`` elements). If *maxsplit* is not - specified or ``-1``, then there is no limit on the number of splits - (all possible splits are made). - - If *sep* is given, consecutive delimiters are not grouped together and are - deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns - ``['1', '', '2']``). The *sep* argument may consist of multiple characters - (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). - Splitting an empty string with a specified separator returns ``['']``. - - For example:: - - >>> '1,2,3'.split(',') - ['1', '2', '3'] - >>> '1,2,3'.split(',', maxsplit=1) - ['1', '2,3'] - >>> '1,2,,3,'.split(',') - ['1', '2', '', '3', ''] - - If *sep* is not specified or is ``None``, a different splitting algorithm is - applied: runs of consecutive whitespace are regarded as a single separator, - and the result will contain no empty strings at the start or end if the - string has leading or trailing whitespace. Consequently, splitting an empty - string or a string consisting of just whitespace with a ``None`` separator - returns ``[]``. - - For example:: - - >>> '1 2 3'.split() - ['1', '2', '3'] - >>> '1 2 3'.split(maxsplit=1) - ['1', '2 3'] - >>> ' 1 2 3 '.split() - ['1', '2', '3'] - - -.. index:: - single: universal newlines; str.splitlines method - -.. method:: str.splitlines([keepends]) - - Return a list of the lines in the string, breaking at line boundaries. Line - breaks are not included in the resulting list unless *keepends* is given and - true. - - This method splits on the following line boundaries. In particular, the - boundaries are a superset of :term:`universal newlines`. - - +-----------------------+-----------------------------+ - | Representation | Description | - +=======================+=============================+ - | ``\n`` | Line Feed | - +-----------------------+-----------------------------+ - | ``\r`` | Carriage Return | - +-----------------------+-----------------------------+ - | ``\r\n`` | Carriage Return + Line Feed | - +-----------------------+-----------------------------+ - | ``\v`` or ``\x0b`` | Line Tabulation | - +-----------------------+-----------------------------+ - | ``\f`` or ``\x0c`` | Form Feed | - +-----------------------+-----------------------------+ - | ``\x1c`` | File Separator | - +-----------------------+-----------------------------+ - | ``\x1d`` | Group Separator | - +-----------------------+-----------------------------+ - | ``\x1e`` | Record Separator | - +-----------------------+-----------------------------+ - | ``\x85`` | Next Line (C1 Control Code) | - +-----------------------+-----------------------------+ - | ``\u2028`` | Line Separator | - +-----------------------+-----------------------------+ - | ``\u2029`` | Paragraph Separator | - +-----------------------+-----------------------------+ - - .. versionchanged:: 3.2 - - ``\v`` and ``\f`` added to list of line boundaries. - - For example:: - - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() - ['ab c', '', 'de fg', 'kl'] - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] - - Unlike :meth:`~str.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> "".splitlines() - [] - >>> "One line\n".splitlines() - ['One line'] - - For comparison, ``split('\n')`` gives:: - - >>> ''.split('\n') - [''] - >>> 'Two lines\n'.split('\n') - ['Two lines', ''] - - -.. method:: str.startswith(prefix[, start[, end]]) - - Return ``True`` if string starts with the *prefix*, otherwise return ``False``. - *prefix* can also be a tuple of prefixes to look for. With optional *start*, - test string beginning at that position. With optional *end*, stop comparing - string at that position. - - -.. method:: str.strip([chars]) - - Return a copy of the string with the leading and trailing characters removed. - The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: - - >>> ' spacious '.strip() - 'spacious' - >>> 'www.example.com'.strip('cmowz.') - 'example' - - The outermost leading and trailing *chars* argument values are stripped - from the string. Characters are removed from the leading end until - reaching a string character that is not contained in the set of - characters in *chars*. A similar action takes place on the trailing end. - For example:: - - >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' - >>> comment_string.strip('.#! ') - 'Section 3.2.1 Issue #32' - - -.. method:: str.swapcase() - - Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. - - -.. method:: str.title() - - Return a titlecased version of the string where words start with an uppercase - character and the remaining characters are lowercase. - - For example:: - - >>> 'Hello world'.title() - 'Hello World' - - The algorithm uses a simple language-independent definition of a word as - groups of consecutive letters. The definition works in many contexts but - it means that apostrophes in contractions and possessives form word - boundaries, which may not be the desired result:: - - >>> "they're bill's friends from the UK".title() - "They'Re Bill'S Friends From The Uk" - - A workaround for apostrophes can be constructed using regular expressions:: - - >>> import re - >>> def titlecase(s): - ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0)[0].upper() + - ... mo.group(0)[1:].lower(), - ... s) - ... - >>> titlecase("they're bill's friends.") - "They're Bill's Friends." - - -.. method:: str.translate(table) - - Return a copy of the string in which each character has been mapped through - the given translation table. The table must be an object that implements - indexing via :meth:`__getitem__`, typically a :term:`mapping` or - :term:`sequence`. When indexed by a Unicode ordinal (an integer), the - table object can do any of the following: return a Unicode ordinal or a - string, to map the character to one or more other characters; return - ``None``, to delete the character from the return string; or raise a - :exc:`LookupError` exception, to map the character to itself. - - You can use :meth:`str.maketrans` to create a translation map from - character-to-character mappings in different formats. - - See also the :mod:`codecs` module for a more flexible approach to custom - character mappings. - - -.. method:: str.upper() - - Return a copy of the string with all the cased characters [4]_ converted to - uppercase. Note that ``s.upper().isupper()`` might be ``False`` if ``s`` - contains uncased characters or if the Unicode category of the resulting - character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, - titlecase). - - The uppercasing algorithm used is described in section 3.13 of the Unicode - Standard. - - -.. method:: str.zfill(width) - - Return a copy of the string left filled with ASCII ``'0'`` digits to - make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) - is handled by inserting the padding *after* the sign character rather - than before. The original string is returned if *width* is less than - or equal to ``len(s)``. - - For example:: - - >>> "42".zfill(5) - '00042' - >>> "-42".zfill(5) - '-0042' - - - -.. _old-string-formatting: - -``printf``-style String Formatting ----------------------------------- - -.. index:: - single: formatting, string (%) - single: interpolation, string (%) - single: string; formatting, printf - single: string; interpolation, printf - single: printf-style formatting - single: sprintf-style formatting - single: % (percent); printf-style formatting - -.. note:: - - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted - string literals ` or the :meth:`str.format` interface - helps avoid these errors. These alternatives also provide more powerful, - flexible and extensible approaches to formatting text. - -String objects have one unique built-in operation: the ``%`` operator (modulo). -This is also known as the string *formatting* or *interpolation* operator. -Given ``format % values`` (where *format* is a string), ``%`` conversion -specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. - -If *format* requires a single argument, *values* may be a single non-tuple -object. [5]_ Otherwise, *values* must be a tuple with exactly the number of -items specified by the format string, or a single mapping object (for example, a -dictionary). - -.. index:: - single: () (parentheses); in printf-style formatting - single: * (asterisk); in printf-style formatting - single: . (dot); in printf-style formatting - -A conversion specifier contains two or more characters and has the following -components, which must occur in this order: - -#. The ``'%'`` character, which marks the start of the specifier. - -#. Mapping key (optional), consisting of a parenthesised sequence of characters - (for example, ``(somename)``). - -#. Conversion flags (optional), which affect the result of some conversion - types. - -#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the - actual width is read from the next element of the tuple in *values*, and the - object to convert comes after the minimum field width and optional precision. - -#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If - specified as ``'*'`` (an asterisk), the actual precision is read from the next - element of the tuple in *values*, and the value to convert comes after the - precision. - -#. Length modifier (optional). - -#. Conversion type. - -When the right argument is a dictionary (or other mapping type), then the -formats in the string *must* include a parenthesised mapping key into that -dictionary inserted immediately after the ``'%'`` character. The mapping key -selects the value to be formatted from the mapping. For example: - - >>> print('%(language)s has %(number)03d quote types.' % - ... {'language': "Python", "number": 2}) - Python has 002 quote types. - -In this case no ``*`` specifiers may occur in a format (since they require a -sequential parameter list). - -The conversion flag characters are: - -.. index:: - single: # (hash); in printf-style formatting - single: - (minus); in printf-style formatting - single: + (plus); in printf-style formatting - single: space; in printf-style formatting - -+---------+---------------------------------------------------------------------+ -| Flag | Meaning | -+=========+=====================================================================+ -| ``'#'`` | The value conversion will use the "alternate form" (where defined | -| | below). | -+---------+---------------------------------------------------------------------+ -| ``'0'`` | The conversion will be zero padded for numeric values. | -+---------+---------------------------------------------------------------------+ -| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` | -| | conversion if both are given). | -+---------+---------------------------------------------------------------------+ -| ``' '`` | (a space) A blank should be left before a positive number (or empty | -| | string) produced by a signed conversion. | -+---------+---------------------------------------------------------------------+ -| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion | -| | (overrides a "space" flag). | -+---------+---------------------------------------------------------------------+ - -A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it -is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``. - -The conversion types are: - -+------------+-----------------------------------------------------+-------+ -| Conversion | Meaning | Notes | -+============+=====================================================+=======+ -| ``'d'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'i'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'o'`` | Signed octal value. | \(1) | -+------------+-----------------------------------------------------+-------+ -| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(6) | -+------------+-----------------------------------------------------+-------+ -| ``'x'`` | Signed hexadecimal (lowercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'X'`` | Signed hexadecimal (uppercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'c'`` | Single character (accepts integer or single | | -| | character string). | | -+------------+-----------------------------------------------------+-------+ -| ``'r'`` | String (converts any Python object using | \(5) | -| | :func:`repr`). | | -+------------+-----------------------------------------------------+-------+ -| ``'s'`` | String (converts any Python object using | \(5) | -| | :func:`str`). | | -+------------+-----------------------------------------------------+-------+ -| ``'a'`` | String (converts any Python object using | \(5) | -| | :func:`ascii`). | | -+------------+-----------------------------------------------------+-------+ -| ``'%'`` | No argument is converted, results in a ``'%'`` | | -| | character in the result. | | -+------------+-----------------------------------------------------+-------+ - -Notes: - -(1) - The alternate form causes a leading octal specifier (``'0o'``) to be - inserted before the first digit. - -(2) - The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether - the ``'x'`` or ``'X'`` format was used) to be inserted before the first digit. - -(3) - The alternate form causes the result to always contain a decimal point, even if - no digits follow it. - - The precision determines the number of digits after the decimal point and - defaults to 6. - -(4) - The alternate form causes the result to always contain a decimal point, and - trailing zeroes are not removed as they would otherwise be. - - The precision determines the number of significant digits before and after the - decimal point and defaults to 6. - -(5) - If precision is ``N``, the output is truncated to ``N`` characters. - -(6) - See :pep:`237`. - -Since Python strings have an explicit length, ``%s`` conversions do not assume -that ``'\0'`` is the end of the string. - -.. XXX Examples? - -.. versionchanged:: 3.1 - ``%f`` conversions for numbers whose absolute value is over 1e50 are no - longer replaced by ``%g`` conversions. - - -.. index:: - single: buffer protocol; binary sequence types - -.. _binaryseq: - -Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview` -================================================================================= - -.. index:: - object: bytes - object: bytearray - object: memoryview - module: array - -The core built-in types for manipulating binary data are :class:`bytes` and -:class:`bytearray`. They are supported by :class:`memoryview` which uses -the :ref:`buffer protocol ` to access the memory of other -binary objects without needing to make a copy. - -The :mod:`array` module supports efficient storage of basic data types like -32-bit integers and IEEE754 double-precision floating values. - -.. _typebytes: - -Bytes Objects -------------- - -.. index:: object: bytes - -Bytes objects are immutable sequences of single bytes. Since many major -binary protocols are based on the ASCII text encoding, bytes objects offer -several methods that are only valid when working with ASCII compatible -data and are closely related to string objects in a variety of other ways. - -.. class:: bytes([source[, encoding[, errors]]]) - - Firstly, the syntax for bytes literals is largely the same as that for string - literals, except that a ``b`` prefix is added: - - * Single quotes: ``b'still allows embedded "double" quotes'`` - * Double quotes: ``b"still allows embedded 'single' quotes"``. - * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` - - Only ASCII characters are permitted in bytes literals (regardless of the - declared source code encoding). Any binary values over 127 must be entered - into bytes literals using the appropriate escape sequence. - - As with string literals, bytes literals may also use a ``r`` prefix to disable - processing of escape sequences. See :ref:`strings` for more about the various - forms of bytes literal, including supported escape sequences. - - While bytes literals and representations are based on ASCII text, bytes - objects actually behave like immutable sequences of integers, with each - value in the sequence restricted such that ``0 <= x < 256`` (attempts to - violate this restriction will trigger :exc:`ValueError`). This is done - deliberately to emphasise that while many binary formats include ASCII based - elements and can be usefully manipulated with some text-oriented algorithms, - this is not generally the case for arbitrary binary data (blindly applying - text processing algorithms to binary data formats that are not ASCII - compatible will usually lead to data corruption). - - In addition to the literal forms, bytes objects can be created in a number of - other ways: - - * A zero-filled bytes object of a specified length: ``bytes(10)`` - * From an iterable of integers: ``bytes(range(20))`` - * Copying existing binary data via the buffer protocol: ``bytes(obj)`` - - Also see the :ref:`bytes ` built-in. - - Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal - numbers are a commonly used format for describing binary data. Accordingly, - the bytes type has an additional class method to read data in that format: - - .. classmethod:: fromhex(string) - - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII whitespace being ignored. - - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' - - A reverse conversion function exists to transform a bytes object into its - hexadecimal representation. - - .. method:: hex() - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' - - .. versionadded:: 3.5 - -Since bytes objects are sequences of integers (akin to a tuple), for a bytes -object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes -object of length 1. (This contrasts with text strings, where both indexing -and slicing will produce a string of length 1) - -The representation of bytes objects uses the literal format (``b'...'``) -since it is often more useful than e.g. ``bytes([46, 46, 46])``. You can -always convert a bytes object into a list of integers using ``list(b)``. - -.. note:: - For Python 2.x users: In the Python 2.x series, a variety of implicit - conversions between 8-bit strings (the closest thing 2.x offers to a - built-in binary data type) and Unicode strings were permitted. This was a - backwards compatibility workaround to account for the fact that Python - originally only supported 8-bit text, and Unicode text was a later - addition. In Python 3.x, those implicit conversions are gone - conversions - between 8-bit binary data and Unicode text must be explicit, and bytes and - string objects will always compare unequal. - - -.. _typebytearray: - -Bytearray Objects ------------------ - -.. index:: object: bytearray - -:class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. - -.. class:: bytearray([source[, encoding[, errors]]]) - - There is no dedicated literal syntax for bytearray objects, instead - they are always created by calling the constructor: - - * Creating an empty instance: ``bytearray()`` - * Creating a zero-filled instance with a given length: ``bytearray(10)`` - * From an iterable of integers: ``bytearray(range(20))`` - * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` - - As bytearray objects are mutable, they support the - :ref:`mutable ` sequence operations in addition to the - common bytes and bytearray operations described in :ref:`bytes-methods`. - - Also see the :ref:`bytearray ` built-in. - - Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal - numbers are a commonly used format for describing binary data. Accordingly, - the bytearray type has an additional class method to read data in that format: - - .. classmethod:: fromhex(string) - - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII whitespace being ignored. - - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') - - A reverse conversion function exists to transform a bytearray object into its - hexadecimal representation. - - .. method:: hex() - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' - - .. versionadded:: 3.5 - -Since bytearray objects are sequences of integers (akin to a list), for a -bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be -a bytearray object of length 1. (This contrasts with text strings, where -both indexing and slicing will produce a string of length 1) - -The representation of bytearray objects uses the bytes literal format -(``bytearray(b'...')``) since it is often more useful than e.g. -``bytearray([46, 46, 46])``. You can always convert a bytearray object into -a list of integers using ``list(b)``. - - -.. _bytes-methods: - -Bytes and Bytearray Operations ------------------------------- - -.. index:: pair: bytes; methods - pair: bytearray; methods - -Both bytes and bytearray objects support the :ref:`common ` -sequence operations. They interoperate not just with operands of the same -type, but with any :term:`bytes-like object`. Due to this flexibility, they can be -freely mixed in operations without causing errors. However, the return type -of the result may depend on the order of operands. - -.. note:: - - The methods on bytes and bytearray objects don't accept strings as their - arguments, just as the methods on strings don't accept bytes as their - arguments. For example, you have to write:: - - a = "abc" - b = a.replace("a", "f") - - and:: - - a = b"abc" - b = a.replace(b"a", b"f") - -Some bytes and bytearray operations assume the use of ASCII compatible -binary formats, and hence should be avoided when working with arbitrary -binary data. These restrictions are covered below. - -.. note:: - Using these ASCII based operations to manipulate binary data that is not - stored in an ASCII based format may lead to data corruption. - -The following methods on bytes and bytearray objects can be used with -arbitrary binary data. - -.. method:: bytes.count(sub[, start[, end]]) - bytearray.count(sub[, start[, end]]) - - Return the number of non-overlapping occurrences of subsequence *sub* in - the range [*start*, *end*]. Optional arguments *start* and *end* are - interpreted as in slice notation. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.decode(encoding="utf-8", errors="strict") - bytearray.decode(encoding="utf-8", errors="strict") - - Return a string decoded from the given bytes. Default encoding is - ``'utf-8'``. *errors* may be given to set a different - error handling scheme. The default for *errors* is ``'strict'``, meaning - that encoding errors raise a :exc:`UnicodeError`. Other possible values are - ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a - list of possible encodings, see section :ref:`standard-encodings`. - - .. note:: - - Passing the *encoding* argument to :class:`str` allows decoding any - :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. - - .. versionchanged:: 3.1 - Added support for keyword arguments. - - -.. method:: bytes.endswith(suffix[, start[, end]]) - bytearray.endswith(suffix[, start[, end]]) - - Return ``True`` if the binary data ends with the specified *suffix*, - otherwise return ``False``. *suffix* can also be a tuple of suffixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The suffix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.find(sub[, start[, end]]) - bytearray.find(sub[, start[, end]]) - - Return the lowest index in the data where the subsequence *sub* is found, - such that *sub* is contained in the slice ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` if *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. note:: - - The :meth:`~bytes.find` method should be used only if you need to know the - position of *sub*. To check if *sub* is a substring or not, use the - :keyword:`in` operator:: - - >>> b'Py' in b'Python' - True - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.index(sub[, start[, end]]) - bytearray.index(sub[, start[, end]]) - - Like :meth:`~bytes.find`, but raise :exc:`ValueError` when the - subsequence is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.join(iterable) - bytearray.join(iterable) - - Return a bytes or bytearray object which is the concatenation of the - binary data sequences in *iterable*. A :exc:`TypeError` will be raised - if there are any values in *iterable* that are not :term:`bytes-like - objects `, including :class:`str` objects. The - separator between elements is the contents of the bytes or - bytearray object providing this method. - - -.. staticmethod:: bytes.maketrans(from, to) - bytearray.maketrans(from, to) - - This static method returns a translation table usable for - :meth:`bytes.translate` that will map each character in *from* into the - character at the same position in *to*; *from* and *to* must both be - :term:`bytes-like objects ` and have the same length. - - .. versionadded:: 3.1 - - -.. method:: bytes.partition(sep) - bytearray.partition(sep) - - Split the sequence at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself or its - bytearray copy, and the part after the separator. - If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) - - Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. - - The subsequence to search for and its replacement may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.rfind(sub[, start[, end]]) - bytearray.rfind(sub[, start[, end]]) - - Return the highest index in the sequence where the subsequence *sub* is - found, such that *sub* is contained within ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` on failure. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rindex(sub[, start[, end]]) - bytearray.rindex(sub[, start[, end]]) - - Like :meth:`~bytes.rfind` but raises :exc:`ValueError` when the - subsequence *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) - - Split the sequence at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself or its - bytearray copy, and the part after the separator. - If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.startswith(prefix[, start[, end]]) - bytearray.startswith(prefix[, start[, end]]) - - Return ``True`` if the binary data starts with the specified *prefix*, - otherwise return ``False``. *prefix* can also be a tuple of prefixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The prefix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.translate(table, delete=b'') - bytearray.translate(table, delete=b'') - - Return a copy of the bytes or bytearray object where all bytes occurring in - the optional argument *delete* are removed, and the remaining bytes have - been mapped through the given translation table, which must be a bytes - object of length 256. - - You can use the :func:`bytes.maketrans` method to create a translation - table. - - Set the *table* argument to ``None`` for translations that only delete - characters:: - - >>> b'read this short text'.translate(None, b'aeiou') - b'rd ths shrt txt' - - .. versionchanged:: 3.6 - *delete* is now supported as a keyword argument. - - -The following methods on bytes and bytearray objects have default behaviours -that assume the use of ASCII compatible binary formats, but can still be used -with arbitrary binary data by passing appropriate arguments. Note that all of -the bytearray methods in this section do *not* operate in place, and instead -produce new objects. - -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) - - Return a copy of the object centered in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) - - Return a copy of the object left justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) - - Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; - rather, all combinations of its values are stripped:: - - >>> b' spacious '.lstrip() - b'spacious ' - >>> b'www.example.com'.lstrip(b'cmowz.') - b'example.com' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) - - Return a copy of the object right justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rsplit(sep=None, maxsplit=-1) - bytearray.rsplit(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits - are done, the *rightmost* ones. If *sep* is not specified or ``None``, - any subsequence consisting solely of ASCII whitespace is a separator. - Except for splitting from the right, :meth:`rsplit` behaves like - :meth:`split` which is described in detail below. - - -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) - - Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, - all combinations of its values are stripped:: - - >>> b' spacious '.rstrip() - b' spacious' - >>> b'mississippi'.rstrip(b'ipz') - b'mississ' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.split(sep=None, maxsplit=-1) - bytearray.split(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given and non-negative, at most - *maxsplit* splits are done (thus, the list will have at most ``maxsplit+1`` - elements). If *maxsplit* is not specified or is ``-1``, then there is no - limit on the number of splits (all possible splits are made). - - If *sep* is given, consecutive delimiters are not grouped together and are - deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` - returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any - :term:`bytes-like object`. - - For example:: - - >>> b'1,2,3'.split(b',') - [b'1', b'2', b'3'] - >>> b'1,2,3'.split(b',', maxsplit=1) - [b'1', b'2,3'] - >>> b'1,2,,3,'.split(b',') - [b'1', b'2', b'', b'3', b''] - - If *sep* is not specified or is ``None``, a different splitting algorithm - is applied: runs of consecutive ASCII whitespace are regarded as a single - separator, and the result will contain no empty strings at the start or - end if the sequence has leading or trailing whitespace. Consequently, - splitting an empty sequence or a sequence consisting solely of ASCII - whitespace without a specified separator returns ``[]``. - - For example:: - - - >>> b'1 2 3'.split() - [b'1', b'2', b'3'] - >>> b'1 2 3'.split(maxsplit=1) - [b'1', b'2 3'] - >>> b' 1 2 3 '.split() - [b'1', b'2', b'3'] - - -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) - - Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is - not a prefix or suffix; rather, all combinations of its values are - stripped:: - - >>> b' spacious '.strip() - b'spacious' - >>> b'www.example.com'.strip(b'cmowz.') - b'example' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -The following methods on bytes and bytearray objects assume the use of ASCII -compatible binary formats and should not be applied to arbitrary binary data. -Note that all of the bytearray methods in this section do *not* operate in -place, and instead produce new objects. - -.. method:: bytes.capitalize() - bytearray.capitalize() - - Return a copy of the sequence with each byte interpreted as an ASCII - character, and the first byte capitalized and the rest lowercased. - Non-ASCII byte values are passed through unchanged. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.expandtabs(tabsize=8) - bytearray.expandtabs(tabsize=8) - - Return a copy of the sequence where all ASCII tab characters are replaced - by one or more ASCII spaces, depending on the current column and the given - tab size. Tab positions occur every *tabsize* bytes (default is 8, - giving tab positions at columns 0, 8, 16 and so on). To expand the - sequence, the current column is set to zero and the sequence is examined - byte by byte. If the byte is an ASCII tab character (``b'\t'``), one or - more space characters are inserted in the result until the current column - is equal to the next tab position. (The tab character itself is not - copied.) If the current byte is an ASCII newline (``b'\n'``) or - carriage return (``b'\r'``), it is copied and the current column is reset - to zero. Any other byte value is copied unchanged and the current column - is incremented by one regardless of how the byte value is represented when - printed:: - - >>> b'01\t012\t0123\t01234'.expandtabs() - b'01 012 0123 01234' - >>> b'01\t012\t0123\t01234'.expandtabs(4) - b'01 012 0123 01234' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.isalnum() - bytearray.isalnum() - - Return true if all bytes in the sequence are alphabetical ASCII characters - or ASCII decimal digits and the sequence is not empty, false otherwise. - Alphabetic ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. ASCII decimal - digits are those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'ABCabc1'.isalnum() - True - >>> b'ABC abc1'.isalnum() - False - - -.. method:: bytes.isalpha() - bytearray.isalpha() - - Return true if all bytes in the sequence are alphabetic ASCII characters - and the sequence is not empty, false otherwise. Alphabetic ASCII - characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - For example:: - - >>> b'ABCabc'.isalpha() - True - >>> b'ABCabc1'.isalpha() - False - - -.. method:: bytes.isdigit() - bytearray.isdigit() - - Return true if all bytes in the sequence are ASCII decimal digits - and the sequence is not empty, false otherwise. ASCII decimal digits are - those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'1234'.isdigit() - True - >>> b'1.23'.isdigit() - False - - -.. method:: bytes.islower() - bytearray.islower() - - Return true if there is at least one lowercase ASCII character - in the sequence and no uppercase ASCII characters, false otherwise. - - For example:: - - >>> b'hello world'.islower() - True - >>> b'Hello world'.islower() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.isspace() - bytearray.isspace() - - Return true if all bytes in the sequence are ASCII whitespace and the - sequence is not empty, false otherwise. ASCII whitespace characters are - those byte values in the sequence ``b' \t\n\r\x0b\f'`` (space, tab, newline, - carriage return, vertical tab, form feed). - - -.. method:: bytes.istitle() - bytearray.istitle() - - Return true if the sequence is ASCII titlecase and the sequence is not - empty, false otherwise. See :meth:`bytes.title` for more details on the - definition of "titlecase". - - For example:: - - >>> b'Hello World'.istitle() - True - >>> b'Hello world'.istitle() - False - - -.. method:: bytes.isupper() - bytearray.isupper() - - Return true if there is at least one uppercase alphabetic ASCII character - in the sequence and no lowercase ASCII characters, false otherwise. - - For example:: - - >>> b'HELLO WORLD'.isupper() - True - >>> b'Hello world'.isupper() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.lower() - bytearray.lower() - - Return a copy of the sequence with all the uppercase ASCII characters - converted to their corresponding lowercase counterpart. - - For example:: - - >>> b'Hello World'.lower() - b'hello world' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. index:: - single: universal newlines; bytes.splitlines method - single: universal newlines; bytearray.splitlines method - -.. method:: bytes.splitlines(keepends=False) - bytearray.splitlines(keepends=False) - - Return a list of the lines in the binary sequence, breaking at ASCII - line boundaries. This method uses the :term:`universal newlines` approach - to splitting lines. Line breaks are not included in the resulting list - unless *keepends* is given and true. - - For example:: - - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines() - [b'ab c', b'', b'de fg', b'kl'] - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - [b'ab c\n', b'\n', b'de fg\r', b'kl\r\n'] - - Unlike :meth:`~bytes.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> b"".split(b'\n'), b"Two lines\n".split(b'\n') - ([b''], [b'Two lines', b'']) - >>> b"".splitlines(), b"One line\n".splitlines() - ([], [b'One line']) - - -.. method:: bytes.swapcase() - bytearray.swapcase() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart and vice-versa. - - For example:: - - >>> b'Hello World'.swapcase() - b'hELLO wORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - Unlike :func:`str.swapcase()`, it is always the case that - ``bin.swapcase().swapcase() == bin`` for the binary versions. Case - conversions are symmetrical in ASCII, even though that is not generally - true for arbitrary Unicode code points. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.title() - bytearray.title() - - Return a titlecased version of the binary sequence where words start with - an uppercase ASCII character and the remaining characters are lowercase. - Uncased byte values are left unmodified. - - For example:: - - >>> b'Hello world'.title() - b'Hello World' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - All other byte values are uncased. - - The algorithm uses a simple language-independent definition of a word as - groups of consecutive letters. The definition works in many contexts but - it means that apostrophes in contractions and possessives form word - boundaries, which may not be the desired result:: - - >>> b"they're bill's friends from the UK".title() - b"They'Re Bill'S Friends From The Uk" - - A workaround for apostrophes can be constructed using regular expressions:: - - >>> import re - >>> def titlecase(s): - ... return re.sub(rb"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0)[0:1].upper() + - ... mo.group(0)[1:].lower(), - ... s) - ... - >>> titlecase(b"they're bill's friends.") - b"They're Bill's Friends." - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.upper() - bytearray.upper() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart. - - For example:: - - >>> b'Hello World'.upper() - b'HELLO WORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.zfill(width) - bytearray.zfill(width) - - Return a copy of the sequence left filled with ASCII ``b'0'`` digits to - make a sequence of length *width*. A leading sign prefix (``b'+'``/ - ``b'-'``) is handled by inserting the padding *after* the sign character - rather than before. For :class:`bytes` objects, the original sequence is - returned if *width* is less than or equal to ``len(seq)``. - - For example:: - - >>> b"42".zfill(5) - b'00042' - >>> b"-42".zfill(5) - b'-0042' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. _bytes-formatting: - -``printf``-style Bytes Formatting ----------------------------------- - -.. index:: - single: formatting; bytes (%) - single: formatting; bytearray (%) - single: interpolation; bytes (%) - single: interpolation; bytearray (%) - single: bytes; formatting - single: bytearray; formatting - single: bytes; interpolation - single: bytearray; interpolation - single: printf-style formatting - single: sprintf-style formatting - single: % (percent); printf-style formatting - -.. note:: - - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). If the value being printed may be a tuple or - dictionary, wrap it in a tuple. - -Bytes objects (``bytes``/``bytearray``) have one unique built-in operation: -the ``%`` operator (modulo). -This is also known as the bytes *formatting* or *interpolation* operator. -Given ``format % values`` (where *format* is a bytes object), ``%`` conversion -specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. - -If *format* requires a single argument, *values* may be a single non-tuple -object. [5]_ Otherwise, *values* must be a tuple with exactly the number of -items specified by the format bytes object, or a single mapping object (for -example, a dictionary). - -.. index:: - single: () (parentheses); in printf-style formatting - single: * (asterisk); in printf-style formatting - single: . (dot); in printf-style formatting - -A conversion specifier contains two or more characters and has the following -components, which must occur in this order: - -#. The ``'%'`` character, which marks the start of the specifier. - -#. Mapping key (optional), consisting of a parenthesised sequence of characters - (for example, ``(somename)``). - -#. Conversion flags (optional), which affect the result of some conversion - types. - -#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the - actual width is read from the next element of the tuple in *values*, and the - object to convert comes after the minimum field width and optional precision. - -#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If - specified as ``'*'`` (an asterisk), the actual precision is read from the next - element of the tuple in *values*, and the value to convert comes after the - precision. - -#. Length modifier (optional). - -#. Conversion type. - -When the right argument is a dictionary (or other mapping type), then the -formats in the bytes object *must* include a parenthesised mapping key into that -dictionary inserted immediately after the ``'%'`` character. The mapping key -selects the value to be formatted from the mapping. For example: - - >>> print(b'%(language)s has %(number)03d quote types.' % - ... {b'language': b"Python", b"number": 2}) - b'Python has 002 quote types.' - -In this case no ``*`` specifiers may occur in a format (since they require a -sequential parameter list). - -The conversion flag characters are: - -.. index:: - single: # (hash); in printf-style formatting - single: - (minus); in printf-style formatting - single: + (plus); in printf-style formatting - single: space; in printf-style formatting - -+---------+---------------------------------------------------------------------+ -| Flag | Meaning | -+=========+=====================================================================+ -| ``'#'`` | The value conversion will use the "alternate form" (where defined | -| | below). | -+---------+---------------------------------------------------------------------+ -| ``'0'`` | The conversion will be zero padded for numeric values. | -+---------+---------------------------------------------------------------------+ -| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` | -| | conversion if both are given). | -+---------+---------------------------------------------------------------------+ -| ``' '`` | (a space) A blank should be left before a positive number (or empty | -| | string) produced by a signed conversion. | -+---------+---------------------------------------------------------------------+ -| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion | -| | (overrides a "space" flag). | -+---------+---------------------------------------------------------------------+ - -A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it -is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``. - -The conversion types are: - -+------------+-----------------------------------------------------+-------+ -| Conversion | Meaning | Notes | -+============+=====================================================+=======+ -| ``'d'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'i'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'o'`` | Signed octal value. | \(1) | -+------------+-----------------------------------------------------+-------+ -| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(8) | -+------------+-----------------------------------------------------+-------+ -| ``'x'`` | Signed hexadecimal (lowercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'X'`` | Signed hexadecimal (uppercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'c'`` | Single byte (accepts integer or single | | -| | byte objects). | | -+------------+-----------------------------------------------------+-------+ -| ``'b'`` | Bytes (any object that follows the | \(5) | -| | :ref:`buffer protocol ` or has | | -| | :meth:`__bytes__`). | | -+------------+-----------------------------------------------------+-------+ -| ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'a'`` | Bytes (converts any Python object using | \(5) | -| | ``repr(obj).encode('ascii','backslashreplace)``). | | -+------------+-----------------------------------------------------+-------+ -| ``'r'`` | ``'r'`` is an alias for ``'a'`` and should only | \(7) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'%'`` | No argument is converted, results in a ``'%'`` | | -| | character in the result. | | -+------------+-----------------------------------------------------+-------+ - -Notes: - -(1) - The alternate form causes a leading octal specifier (``'0o'``) to be - inserted before the first digit. - -(2) - The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether - the ``'x'`` or ``'X'`` format was used) to be inserted before the first digit. - -(3) - The alternate form causes the result to always contain a decimal point, even if - no digits follow it. - - The precision determines the number of digits after the decimal point and - defaults to 6. - -(4) - The alternate form causes the result to always contain a decimal point, and - trailing zeroes are not removed as they would otherwise be. - - The precision determines the number of significant digits before and after the - decimal point and defaults to 6. - -(5) - If precision is ``N``, the output is truncated to ``N`` characters. - -(6) - ``b'%s'`` is deprecated, but will not be removed during the 3.x series. - -(7) - ``b'%r'`` is deprecated, but will not be removed during the 3.x series. - -(8) - See :pep:`237`. - -.. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - -.. seealso:: - - :pep:`461` - Adding % formatting to bytes and bytearray - -.. versionadded:: 3.5 - -.. _typememoryview: - -Memory Views ------------- - -:class:`memoryview` objects allow Python code to access the internal data -of an object that supports the :ref:`buffer protocol ` without -copying. - -.. class:: memoryview(obj) - - Create a :class:`memoryview` that references *obj*. *obj* must support the - buffer protocol. Built-in objects that support the buffer protocol include - :class:`bytes` and :class:`bytearray`. - - A :class:`memoryview` has the notion of an *element*, which is the - atomic memory unit handled by the originating object *obj*. For many - simple types such as :class:`bytes` and :class:`bytearray`, an element - is a single byte, but other types such as :class:`array.array` may have - bigger elements. - - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`. - If ``view.ndim = 0``, the length is 1. If ``view.ndim = 1``, the length - is equal to the number of elements in the view. For higher dimensions, - the length is equal to the length of the nested list representation of - the view. The :class:`~memoryview.itemsize` attribute will give you the - number of bytes in a single element. - - A :class:`memoryview` supports slicing and indexing to expose its data. - One-dimensional slicing will result in a subview:: - - >>> v = memoryview(b'abcefg') - >>> v[1] - 98 - >>> v[-1] - 103 - >>> v[1:4] - - >>> bytes(v[1:4]) - b'bce' - - If :class:`~memoryview.format` is one of the native format specifiers - from the :mod:`struct` module, indexing with an integer or a tuple of - integers is also supported and returns a single *element* with - the correct type. One-dimensional memoryviews can be indexed - with an integer or a one-integer tuple. Multi-dimensional memoryviews - can be indexed with tuples of exactly *ndim* integers where *ndim* is - the number of dimensions. Zero-dimensional memoryviews can be indexed - with the empty tuple. - - Here is an example with a non-byte format:: - - >>> import array - >>> a = array.array('l', [-11111111, 22222222, -33333333, 44444444]) - >>> m = memoryview(a) - >>> m[0] - -11111111 - >>> m[-1] - 44444444 - >>> m[::2].tolist() - [-11111111, -33333333] - - If the underlying object is writable, the memoryview supports - one-dimensional slice assignment. Resizing is not allowed:: - - >>> data = bytearray(b'abcefg') - >>> v = memoryview(data) - >>> v.readonly - False - >>> v[0] = ord(b'z') - >>> data - bytearray(b'zbcefg') - >>> v[1:4] = b'123' - >>> data - bytearray(b'z123fg') - >>> v[2:3] = b'spam' - Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview assignment: lvalue and rvalue have different structures - >>> v[2:6] = b'spam' - >>> data - bytearray(b'z1spam') - - One-dimensional memoryviews of hashable (read-only) types with formats - 'B', 'b' or 'c' are also hashable. The hash is defined as - ``hash(m) == hash(m.tobytes())``:: - - >>> v = memoryview(b'abcefg') - >>> hash(v) == hash(b'abcefg') - True - >>> hash(v[2:4]) == hash(b'ce') - True - >>> hash(v[::-2]) == hash(b'abcefg'[::-2]) - True - - .. versionchanged:: 3.3 - One-dimensional memoryviews can now be sliced. - One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable. - - .. versionchanged:: 3.4 - memoryview is now registered automatically with - :class:`collections.abc.Sequence` - - .. versionchanged:: 3.5 - memoryviews can now be indexed with tuple of integers. - - :class:`memoryview` has several methods: - - .. method:: __eq__(exporter) - - A memoryview and a :pep:`3118` exporter are equal if their shapes are - equivalent and if all corresponding values are equal when the operands' - respective format codes are interpreted using :mod:`struct` syntax. - - For the subset of :mod:`struct` format strings currently supported by - :meth:`tolist`, ``v`` and ``w`` are equal if ``v.tolist() == w.tolist()``:: - - >>> import array - >>> a = array.array('I', [1, 2, 3, 4, 5]) - >>> b = array.array('d', [1.0, 2.0, 3.0, 4.0, 5.0]) - >>> c = array.array('b', [5, 3, 1]) - >>> x = memoryview(a) - >>> y = memoryview(b) - >>> x == a == y == b - True - >>> x.tolist() == a.tolist() == y.tolist() == b.tolist() - True - >>> z = y[::-2] - >>> z == c - True - >>> z.tolist() == c.tolist() - True - - If either format string is not supported by the :mod:`struct` module, - then the objects will always compare as unequal (even if the format - strings and buffer contents are identical):: - - >>> from ctypes import BigEndianStructure, c_long - >>> class BEPoint(BigEndianStructure): - ... _fields_ = [("x", c_long), ("y", c_long)] - ... - >>> point = BEPoint(100, 200) - >>> a = memoryview(point) - >>> b = memoryview(point) - >>> a == point - False - >>> a == b - False - - Note that, as with floating point numbers, ``v is w`` does *not* imply - ``v == w`` for memoryview objects. - - .. versionchanged:: 3.3 - Previous versions compared the raw memory disregarding the item format - and the logical array structure. - - .. method:: tobytes() - - Return the data in the buffer as a bytestring. This is equivalent to - calling the :class:`bytes` constructor on the memoryview. :: - - >>> m = memoryview(b"abc") - >>> m.tobytes() - b'abc' - >>> bytes(m) - b'abc' - - For non-contiguous arrays the result is equal to the flattened list - representation with all elements converted to bytes. :meth:`tobytes` - supports all format strings, including those that are not in - :mod:`struct` module syntax. - - .. method:: hex() - - Return a string object containing two hexadecimal digits for each - byte in the buffer. :: - - >>> m = memoryview(b"abc") - >>> m.hex() - '616263' - - .. versionadded:: 3.5 - - .. method:: tolist() - - Return the data in the buffer as a list of elements. :: - - >>> memoryview(b'abc').tolist() - [97, 98, 99] - >>> import array - >>> a = array.array('d', [1.1, 2.2, 3.3]) - >>> m = memoryview(a) - >>> m.tolist() - [1.1, 2.2, 3.3] - - .. versionchanged:: 3.3 - :meth:`tolist` now supports all single character native formats in - :mod:`struct` module syntax as well as multi-dimensional - representations. - - .. method:: release() - - Release the underlying buffer exposed by the memoryview object. Many - objects take special actions when a view is held on them (for example, - a :class:`bytearray` would temporarily forbid resizing); therefore, - calling release() is handy to remove these restrictions (and free any - dangling resources) as soon as possible. - - After this method has been called, any further operation on the view - raises a :class:`ValueError` (except :meth:`release()` itself which can - be called multiple times):: - - >>> m = memoryview(b'abc') - >>> m.release() - >>> m[0] - Traceback (most recent call last): - File "", line 1, in - ValueError: operation forbidden on released memoryview object - - The context management protocol can be used for a similar effect, - using the ``with`` statement:: - - >>> with memoryview(b'abc') as m: - ... m[0] - ... - 97 - >>> m[0] - Traceback (most recent call last): - File "", line 1, in - ValueError: operation forbidden on released memoryview object - - .. versionadded:: 3.2 - - .. method:: cast(format[, shape]) - - Cast a memoryview to a new format or shape. *shape* defaults to - ``[byte_length//new_itemsize]``, which means that the result view - will be one-dimensional. The return value is a new memoryview, but - the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous` - and C-contiguous -> 1D. - - The destination format is restricted to a single element native format in - :mod:`struct` syntax. One of the formats must be a byte format - ('B', 'b' or 'c'). The byte length of the result must be the same - as the original length. - - Cast 1D/long to 1D/unsigned bytes:: - - >>> import array - >>> a = array.array('l', [1,2,3]) - >>> x = memoryview(a) - >>> x.format - 'l' - >>> x.itemsize - 8 - >>> len(x) - 3 - >>> x.nbytes - 24 - >>> y = x.cast('B') - >>> y.format - 'B' - >>> y.itemsize - 1 - >>> len(y) - 24 - >>> y.nbytes - 24 - - Cast 1D/unsigned bytes to 1D/char:: - - >>> b = bytearray(b'zyz') - >>> x = memoryview(b) - >>> x[0] = b'a' - Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview: invalid value for format "B" - >>> y = x.cast('c') - >>> y[0] = b'a' - >>> b - bytearray(b'ayz') - - Cast 1D/bytes to 3D/ints to 1D/signed char:: - - >>> import struct - >>> buf = struct.pack("i"*12, *list(range(12))) - >>> x = memoryview(buf) - >>> y = x.cast('i', shape=[2,2,3]) - >>> y.tolist() - [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]] - >>> y.format - 'i' - >>> y.itemsize - 4 - >>> len(y) - 2 - >>> y.nbytes - 48 - >>> z = y.cast('b') - >>> z.format - 'b' - >>> z.itemsize - 1 - >>> len(z) - 48 - >>> z.nbytes - 48 - - Cast 1D/unsigned char to 2D/unsigned long:: - - >>> buf = struct.pack("L"*6, *list(range(6))) - >>> x = memoryview(buf) - >>> y = x.cast('L', shape=[2,3]) - >>> len(y) - 2 - >>> y.nbytes - 48 - >>> y.tolist() - [[0, 1, 2], [3, 4, 5]] - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The source format is no longer restricted when casting to a byte view. - - There are also several readonly attributes available: - - .. attribute:: obj - - The underlying object of the memoryview:: - - >>> b = bytearray(b'xyz') - >>> m = memoryview(b) - >>> m.obj is b - True - - .. versionadded:: 3.3 - - .. attribute:: nbytes - - ``nbytes == product(shape) * itemsize == len(m.tobytes())``. This is - the amount of space in bytes that the array would use in a contiguous - representation. It is not necessarily equal to ``len(m)``:: - - >>> import array - >>> a = array.array('i', [1,2,3,4,5]) - >>> m = memoryview(a) - >>> len(m) - 5 - >>> m.nbytes - 20 - >>> y = m[::2] - >>> len(y) - 3 - >>> y.nbytes - 12 - >>> len(y.tobytes()) - 12 - - Multi-dimensional arrays:: - - >>> import struct - >>> buf = struct.pack("d"*12, *[1.5*x for x in range(12)]) - >>> x = memoryview(buf) - >>> y = x.cast('d', shape=[3,4]) - >>> y.tolist() - [[0.0, 1.5, 3.0, 4.5], [6.0, 7.5, 9.0, 10.5], [12.0, 13.5, 15.0, 16.5]] - >>> len(y) - 3 - >>> y.nbytes - 96 - - .. versionadded:: 3.3 - - .. attribute:: readonly - - A bool indicating whether the memory is read only. - - .. attribute:: format - - A string containing the format (in :mod:`struct` module style) for each - element in the view. A memoryview can be created from exporters with - arbitrary format strings, but some methods (e.g. :meth:`tolist`) are - restricted to native single element formats. - - .. versionchanged:: 3.3 - format ``'B'`` is now handled according to the struct module syntax. - This means that ``memoryview(b'abc')[0] == b'abc'[0] == 97``. - - .. attribute:: itemsize - - The size in bytes of each element of the memoryview:: - - >>> import array, struct - >>> m = memoryview(array.array('H', [32000, 32001, 32002])) - >>> m.itemsize - 2 - >>> m[0] - 32000 - >>> struct.calcsize('H') == m.itemsize - True - - .. attribute:: ndim - - An integer indicating how many dimensions of a multi-dimensional array the - memory represents. - - .. attribute:: shape - - A tuple of integers the length of :attr:`ndim` giving the shape of the - memory as an N-dimensional array. - - .. versionchanged:: 3.3 - An empty tuple instead of ``None`` when ndim = 0. - - .. attribute:: strides - - A tuple of integers the length of :attr:`ndim` giving the size in bytes to - access each element for each dimension of the array. - - .. versionchanged:: 3.3 - An empty tuple instead of ``None`` when ndim = 0. - - .. attribute:: suboffsets - - Used internally for PIL-style arrays. The value is informational only. - - .. attribute:: c_contiguous - - A bool indicating whether the memory is C-:term:`contiguous`. - - .. versionadded:: 3.3 - - .. attribute:: f_contiguous - - A bool indicating whether the memory is Fortran :term:`contiguous`. - - .. versionadded:: 3.3 - - .. attribute:: contiguous - - A bool indicating whether the memory is :term:`contiguous`. - - .. versionadded:: 3.3 - - -.. _types-set: - -Set Types --- :class:`set`, :class:`frozenset` -============================================== - -.. index:: object: set - -A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. -Common uses include membership testing, removing duplicates from a sequence, and -computing mathematical operations such as intersection, union, difference, and -symmetric difference. -(For other containers see the built-in :class:`dict`, :class:`list`, -and :class:`tuple` classes, and the :mod:`collections` module.) - -Like other collections, sets support ``x in set``, ``len(set)``, and ``for x in -set``. Being an unordered collection, sets do not record element position or -order of insertion. Accordingly, sets do not support indexing, slicing, or -other sequence-like behavior. - -There are currently two built-in set types, :class:`set` and :class:`frozenset`. -The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. - -Non-empty sets (not frozensets) can be created by placing a comma-separated list -of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the -:class:`set` constructor. - -The constructors for both classes work the same: - -.. class:: set([iterable]) - frozenset([iterable]) - - Return a new set or frozenset object whose elements are taken from - *iterable*. The elements of a set must be :term:`hashable`. To - represent sets of sets, the inner sets must be :class:`frozenset` - objects. If *iterable* is not specified, a new empty set is - returned. - - Instances of :class:`set` and :class:`frozenset` provide the following - operations: - - .. describe:: len(s) - - Return the number of elements in set *s* (cardinality of *s*). - - .. describe:: x in s - - Test *x* for membership in *s*. - - .. describe:: x not in s - - Test *x* for non-membership in *s*. - - .. method:: isdisjoint(other) - - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. - - .. method:: issubset(other) - set <= other - - Test whether every element in the set is in *other*. - - .. method:: set < other - - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. - - .. method:: issuperset(other) - set >= other - - Test whether every element in *other* is in the set. - - .. method:: set > other - - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. - - .. method:: union(*others) - set | other | ... - - Return a new set with elements from the set and all others. - - .. method:: intersection(*others) - set & other & ... - - Return a new set with elements common to the set and all others. - - .. method:: difference(*others) - set - other - ... - - Return a new set with elements in the set that are not in the others. - - .. method:: symmetric_difference(other) - set ^ other - - Return a new set with elements in either the set or *other* but not both. - - .. method:: copy() - - Return a new set with a shallow copy of *s*. - - - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, and :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. - - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). - - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. - - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. - - Set elements, like dictionary keys, must be :term:`hashable`. - - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. - - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: - - .. method:: update(*others) - set |= other | ... - - Update the set, adding elements from all others. - - .. method:: intersection_update(*others) - set &= other & ... - - Update the set, keeping only elements found in it and all others. - - .. method:: difference_update(*others) - set -= other | ... - - Update the set, removing elements found in others. - - .. method:: symmetric_difference_update(other) - set ^= other - - Update the set, keeping only elements found in either set, but not in both. - - .. method:: add(elem) - - Add element *elem* to the set. - - .. method:: remove(elem) - - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. - - .. method:: discard(elem) - - Remove element *elem* from the set if it is present. - - .. method:: pop() - - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. - - .. method:: clear() - - Remove all elements from the set. - - - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. - - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. - - -.. _typesmapping: - -Mapping Types --- :class:`dict` -=============================== - -.. index:: - object: mapping - object: dictionary - triple: operations on; mapping; types - triple: operations on; dictionary; type - statement: del - builtin: len - -A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in -:class:`list`, :class:`set`, and :class:`tuple` classes, and the -:mod:`collections` module.) - -A dictionary's keys are *almost* arbitrary values. Values that are not -:term:`hashable`, that is, values containing lists, dictionaries or other -mutable types (that are compared by value rather than by object identity) may -not be used as keys. Numeric types used for keys obey the normal rules for -numeric comparison: if two numbers compare equal (such as ``1`` and ``1.0``) -then they can be used interchangeably to index the same dictionary entry. (Note -however, that since computers store floating-point numbers as approximations it -is usually unwise to use them as dictionary keys.) - -Dictionaries can be created by placing a comma-separated list of ``key: value`` -pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: -'jack', 4127: 'sjoerd'}``, or by the :class:`dict` constructor. - -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) - - Return a new dictionary initialized from an optional positional argument - and a possibly empty set of keyword arguments. - - If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. - - If keyword arguments are given, the keyword arguments and their values are - added to the dictionary created from the positional argument. If a key - being added is already present, the value from the keyword argument - replaces the value from the positional argument. - - To illustrate, the following examples all return a dictionary equal to - ``{"one": 1, "two": 2, "three": 3}``:: - - >>> a = dict(one=1, two=2, three=3) - >>> b = {'one': 1, 'two': 2, 'three': 3} - >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) - >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) - >>> e = dict({'three': 3, 'one': 1, 'two': 2}) - >>> a == b == c == d == e - True - - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - - - These are the operations that dictionaries support (and therefore, custom - mapping types should support too): - - .. describe:: len(d) - - Return the number of items in the dictionary *d*. - - .. describe:: d[key] - - Return the item of *d* with key *key*. Raises a :exc:`KeyError` if *key* is - not in the map. - - .. index:: __missing__() - - If a subclass of dict defines a method :meth:`__missing__` and *key* - is not present, the ``d[key]`` operation calls that method with the key *key* - as argument. The ``d[key]`` operation then returns or raises whatever is - returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: - - >>> class Counter(dict): - ... def __missing__(self, key): - ... return 0 - >>> c = Counter() - >>> c['red'] - 0 - >>> c['red'] += 1 - >>> c['red'] - 1 - - The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used - by :class:`collections.defaultdict`. - - .. describe:: d[key] = value - - Set ``d[key]`` to *value*. - - .. describe:: del d[key] - - Remove ``d[key]`` from *d*. Raises a :exc:`KeyError` if *key* is not in the - map. - - .. describe:: key in d - - Return ``True`` if *d* has a key *key*, else ``False``. - - .. describe:: key not in d - - Equivalent to ``not key in d``. - - .. describe:: iter(d) - - Return an iterator over the keys of the dictionary. This is a shortcut - for ``iter(d.keys())``. - - .. method:: clear() - - Remove all items from the dictionary. - - .. method:: copy() - - Return a shallow copy of the dictionary. - - .. classmethod:: fromkeys(seq[, value]) - - Create a new dictionary with keys from *seq* and values set to *value*. - - :meth:`fromkeys` is a class method that returns a new dictionary. *value* - defaults to ``None``. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the dictionary, else *default*. - If *default* is not given, it defaults to ``None``, so that this method - never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the dictionary's items (``(key, value)`` pairs). - See the :ref:`documentation of view objects `. - - .. method:: keys() - - Return a new view of the dictionary's keys. See the :ref:`documentation - of view objects `. - - .. method:: pop(key[, default]) - - If *key* is in the dictionary, remove it and return its value, else return - *default*. If *default* is not given and *key* is not in the dictionary, - a :exc:`KeyError` is raised. - - .. method:: popitem() - - Remove and return an arbitrary ``(key, value)`` pair from the dictionary. - - :meth:`popitem` is useful to destructively iterate over a dictionary, as - often used in set algorithms. If the dictionary is empty, calling - :meth:`popitem` raises a :exc:`KeyError`. - - .. method:: setdefault(key[, default]) - - If *key* is in the dictionary, return its value. If not, insert *key* - with a value of *default* and return *default*. *default* defaults to - ``None``. - - .. method:: update([other]) - - Update the dictionary with the key/value pairs from *other*, overwriting - existing keys. Return ``None``. - - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. - - .. method:: values() - - Return a new view of the dictionary's values. See the - :ref:`documentation of view objects `. - - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - -.. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. - - -.. _dict-views: - -Dictionary view objects ------------------------ - -The objects returned by :meth:`dict.keys`, :meth:`dict.values` and -:meth:`dict.items` are *view objects*. They provide a dynamic view on the -dictionary's entries, which means that when the dictionary changes, the view -reflects these changes. - -Dictionary views can be iterated over to yield their respective data, and -support membership tests: - -.. describe:: len(dictview) - - Return the number of entries in the dictionary. - -.. describe:: iter(dictview) - - Return an iterator over the keys, values or items (represented as tuples of - ``(key, value)``) in the dictionary. - - Keys and values are iterated over in an arbitrary order which is non-random, - varies across Python implementations, and depends on the dictionary's history - of insertions and deletions. If keys, values and items views are iterated - over with no intervening modifications to the dictionary, the order of items - will directly correspond. This allows the creation of ``(value, key)`` pairs - using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to - create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``. - - Iterating views while adding or deleting entries in the dictionary may raise - a :exc:`RuntimeError` or fail to iterate over all entries. - -.. describe:: x in dictview - - Return ``True`` if *x* is in the underlying dictionary's keys, values or - items (in the latter case, *x* should be a ``(key, value)`` tuple). - - -Keys views are set-like since their entries are unique and hashable. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like -since the entries are generally not unique.) For set-like views, all of the -operations defined for the abstract base class :class:`collections.abc.Set` are -available (for example, ``==``, ``<``, or ``^``). - -An example of dictionary view usage:: - - >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} - >>> keys = dishes.keys() - >>> values = dishes.values() - - >>> # iteration - >>> n = 0 - >>> for val in values: - ... n += val - >>> print(n) - 504 - - >>> # keys and values are iterated over in the same order - >>> list(keys) - ['eggs', 'bacon', 'sausage', 'spam'] - >>> list(values) - [2, 1, 1, 500] - - >>> # view objects are dynamic and reflect dict changes - >>> del dishes['eggs'] - >>> del dishes['sausage'] - >>> list(keys) - ['spam', 'bacon'] - - >>> # set operations - >>> keys & {'eggs', 'bacon', 'salad'} - {'bacon'} - >>> keys ^ {'sausage', 'juice'} - {'juice', 'sausage', 'bacon', 'spam'} - - -.. _typecontextmanager: - -Context Manager Types -===================== - -.. index:: - single: context manager - single: context management protocol - single: protocol; context management - -Python's :keyword:`with` statement supports the concept of a runtime context -defined by a context manager. This is implemented using a pair of methods -that allow user-defined classes to define a runtime context that is entered -before the statement body is executed and exited when the statement ends: - - -.. method:: contextmanager.__enter__() - - Enter the runtime context and return either this object or another object - related to the runtime context. The value returned by this method is bound to - the identifier in the :keyword:`as` clause of :keyword:`with` statements using - this context manager. - - An example of a context manager that returns itself is a :term:`file object`. - File objects return themselves from __enter__() to allow :func:`open` to be - used as the context expression in a :keyword:`with` statement. - - An example of a context manager that returns a related object is the one - returned by :func:`decimal.localcontext`. These managers set the active - decimal context to a copy of the original decimal context and then return the - copy. This allows changes to be made to the current decimal context in the body - of the :keyword:`with` statement without affecting code outside the - :keyword:`with` statement. - - -.. method:: contextmanager.__exit__(exc_type, exc_val, exc_tb) - - Exit the runtime context and return a Boolean flag indicating if any exception - that occurred should be suppressed. If an exception occurred while executing the - body of the :keyword:`with` statement, the arguments contain the exception type, - value and traceback information. Otherwise, all three arguments are ``None``. - - Returning a true value from this method will cause the :keyword:`with` statement - to suppress the exception and continue execution with the statement immediately - following the :keyword:`with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`with` statement. - - The exception passed in should never be reraised explicitly - instead, this - method should return a false value to indicate that the method completed - successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`__exit__` - method has actually failed. - -Python defines several context managers to support easy thread synchronisation, -prompt closure of files or other objects, and simpler manipulation of the active -decimal arithmetic context. The specific types are not treated specially beyond -their implementation of the context management protocol. See the -:mod:`contextlib` module for some examples. - -Python's :term:`generator`\s and the :class:`contextlib.contextmanager` decorator -provide a convenient way to implement these protocols. If a generator function is -decorated with the :class:`contextlib.contextmanager` decorator, it will return a -context manager implementing the necessary :meth:`__enter__` and -:meth:`__exit__` methods, rather than the iterator produced by an undecorated -generator function. - -Note that there is no specific slot for any of these methods in the type -structure for Python objects in the Python/C API. Extension types wanting to -define these methods must provide them as a normal Python accessible method. -Compared to the overhead of setting up the runtime context, the overhead of a -single class dictionary lookup is negligible. - - -.. _typesother: - -Other Built-in Types -==================== - -The interpreter supports several other kinds of objects. Most of these support -only one or two operations. - - -.. _typesmodules: - -Modules -------- - -The only special operation on a module is attribute access: ``m.name``, where -*m* is a module and *name* accesses a name defined in *m*'s symbol table. -Module attributes can be assigned to. (Note that the :keyword:`import` -statement is not, strictly speaking, an operation on a module object; ``import -foo`` does not require a module object named *foo* to exist, rather it requires -an (external) *definition* for a module named *foo* somewhere.) - -A special attribute of every module is :attr:`~object.__dict__`. This is the -dictionary containing the module's symbol table. Modifying this dictionary will -actually change the module's symbol table, but direct assignment to the -:attr:`~object.__dict__` attribute is not possible (you can write -``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but you can't write -``m.__dict__ = {}``). Modifying :attr:`~object.__dict__` directly is -not recommended. - -Modules built into the interpreter are written like this: ````. If loaded from a file, they are written as ````. - - -.. _typesobjects: - -Classes and Class Instances ---------------------------- - -See :ref:`objects` and :ref:`class` for these. - - -.. _typesfunctions: - -Functions ---------- - -Function objects are created by function definitions. The only operation on a -function object is to call it: ``func(argument-list)``. - -There are really two flavors of function objects: built-in functions and -user-defined functions. Both support the same operation (to call the function), -but the implementation is different, hence the different object types. - -See :ref:`function` for more information. - - -.. _typesmethods: - -Methods -------- - -.. index:: object: method - -Methods are functions that are called using the attribute notation. There are -two flavors: built-in methods (such as :meth:`append` on lists) and class -instance methods. Built-in methods are described with the types that support -them. - -If you access a method (a function defined in a class namespace) through an -instance, you get a special object: a :dfn:`bound method` (also called -:dfn:`instance method`) object. When called, it will add the ``self`` argument -to the argument list. Bound methods have two special read-only attributes: -``m.__self__`` is the object on which the method operates, and ``m.__func__`` is -the function implementing the method. Calling ``m(arg-1, arg-2, ..., arg-n)`` -is completely equivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ..., -arg-n)``. - -Like function objects, bound method objects support getting arbitrary -attributes. However, since method attributes are actually stored on the -underlying function object (``meth.__func__``), setting method attributes on -bound methods is disallowed. Attempting to set an attribute on a method -results in an :exc:`AttributeError` being raised. In order to set a method -attribute, you need to explicitly set it on the underlying function object:: - - >>> class C: - ... def method(self): - ... pass - ... - >>> c = C() - >>> c.method.whoami = 'my name is method' # can't set on the method - Traceback (most recent call last): - File "", line 1, in - AttributeError: 'method' object has no attribute 'whoami' - >>> c.method.__func__.whoami = 'my name is method' - >>> c.method.whoami - 'my name is method' - -See :ref:`types` for more information. - - -.. index:: object; code, code object - -.. _bltin-code-objects: - -Code Objects ------------- - -.. index:: - builtin: compile - single: __code__ (function object attribute) - -Code objects are used by the implementation to represent "pseudo-compiled" -executable Python code such as a function body. They differ from function -objects because they don't contain a reference to their global execution -environment. Code objects are returned by the built-in :func:`compile` function -and can be extracted from function objects through their :attr:`__code__` -attribute. See also the :mod:`code` module. - -.. index:: - builtin: exec - builtin: eval - -A code object can be executed or evaluated by passing it (instead of a source -string) to the :func:`exec` or :func:`eval` built-in functions. - -See :ref:`types` for more information. - - -.. _bltin-type-objects: - -Type Objects ------------- - -.. index:: - builtin: type - module: types - -Type objects represent the various object types. An object's type is accessed -by the built-in function :func:`type`. There are no special operations on -types. The standard module :mod:`types` defines names for all standard built-in -types. - -Types are written like this: ````. - - -.. _bltin-null-object: - -The Null Object ---------------- - -This object is returned by functions that don't explicitly return a value. It -supports no special operations. There is exactly one null object, named -``None`` (a built-in name). ``type(None)()`` produces the same singleton. - -It is written as ``None``. - - -.. index:: single: ...; ellipsis literal -.. _bltin-ellipsis-object: - -The Ellipsis Object -------------------- - -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named -:const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the -:const:`Ellipsis` singleton. - -It is written as ``Ellipsis`` or ``...``. - - -.. _bltin-notimplemented-object: - -The NotImplemented Object -------------------------- - -This object is returned from comparisons and binary operations when they are -asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. - -It is written as ``NotImplemented``. - - -.. _bltin-boolean-values: - -Boolean Values --------------- - -Boolean values are the two constant objects ``False`` and ``True``. They are -used to represent truth values (although other values can also be considered -false or true). In numeric contexts (for example when used as the argument to -an arithmetic operator), they behave like the integers 0 and 1, respectively. -The built-in function :func:`bool` can be used to convert any value to a -Boolean, if the value can be interpreted as a truth value (see section -:ref:`truth` above). - -.. index:: - single: False - single: True - pair: Boolean; values - -They are written as ``False`` and ``True``, respectively. - - -.. _typesinternal: - -Internal Objects ----------------- - -See :ref:`types` for this information. It describes stack frame objects, -traceback objects, and slice objects. - - -.. _specialattrs: - -Special Attributes -================== - -The implementation adds a few special read-only attributes to several object -types, where they are relevant. Some of these are not reported by the -:func:`dir` built-in function. - - -.. attribute:: object.__dict__ - - A dictionary or other mapping object used to store an object's (writable) - attributes. - - -.. attribute:: instance.__class__ - - The class to which a class instance belongs. - - -.. attribute:: class.__bases__ - - The tuple of base classes of a class object. - - -.. attribute:: definition.__name__ - - The name of the class, function, method, descriptor, or - generator instance. - - -.. attribute:: definition.__qualname__ - - The :term:`qualified name` of the class, function, method, descriptor, - or generator instance. - - .. versionadded:: 3.3 - - -.. attribute:: class.__mro__ - - This attribute is a tuple of classes that are considered when looking for - base classes during method resolution. - - -.. method:: class.mro() - - This method can be overridden by a metaclass to customize the method - resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. - - -.. method:: class.__subclasses__ - - Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. - Example:: - - >>> int.__subclasses__() - [] - - -.. rubric:: Footnotes - -.. [1] Additional information on these special methods may be found in the Python - Reference Manual (:ref:`customization`). - -.. [2] As a consequence, the list ``[1, 2]`` is considered equal to ``[1.0, 2.0]``, and - similarly for tuples. - -.. [3] They must have since the parser can't tell the type of the operands. - -.. [4] Cased characters are those with general category property being one of - "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt" (Letter, titlecase). - -.. [5] To format only a tuple you should therefore provide a singleton tuple whose only - element is the tuple to be formatted. diff --git a/third_party/python/Doc/library/string.rst b/third_party/python/Doc/library/string.rst deleted file mode 100644 index f5c22910c..000000000 --- a/third_party/python/Doc/library/string.rst +++ /dev/null @@ -1,835 +0,0 @@ -:mod:`string` --- Common string operations -========================================== - -.. module:: string - :synopsis: Common string operations. - -**Source code:** :source:`Lib/string.py` - --------------- - -.. seealso:: - - :ref:`textseq` - - :ref:`string-methods` - -String constants ----------------- - -The constants defined in this module are: - - -.. data:: ascii_letters - - The concatenation of the :const:`ascii_lowercase` and :const:`ascii_uppercase` - constants described below. This value is not locale-dependent. - - -.. data:: ascii_lowercase - - The lowercase letters ``'abcdefghijklmnopqrstuvwxyz'``. This value is not - locale-dependent and will not change. - - -.. data:: ascii_uppercase - - The uppercase letters ``'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. This value is not - locale-dependent and will not change. - - -.. data:: digits - - The string ``'0123456789'``. - - -.. data:: hexdigits - - The string ``'0123456789abcdefABCDEF'``. - - -.. data:: octdigits - - The string ``'01234567'``. - - -.. data:: punctuation - - String of ASCII characters which are considered punctuation characters - in the ``C`` locale. - - -.. data:: printable - - String of ASCII characters which are considered printable. This is a - combination of :const:`digits`, :const:`ascii_letters`, :const:`punctuation`, - and :const:`whitespace`. - - -.. data:: whitespace - - A string containing all ASCII characters that are considered whitespace. - This includes the characters space, tab, linefeed, return, formfeed, and - vertical tab. - - -.. _string-formatting: - -Custom String Formatting ------------------------- - -The built-in string class provides the ability to do complex variable -substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows -you to create and customize your own string formatting behaviors using the same -implementation as the built-in :meth:`~str.format` method. - - -.. class:: Formatter - - The :class:`Formatter` class has the following public methods: - - .. method:: format(format_string, *args, **kwargs) - - The primary API method. It takes a format string and - an arbitrary set of positional and keyword arguments. - It is just a wrapper that calls :meth:`vformat`. - - .. deprecated:: 3.5 - Passing a format string as keyword argument *format_string* has been - deprecated. - - .. method:: vformat(format_string, args, kwargs) - - This function does the actual work of formatting. It is exposed as a - separate function for cases where you want to pass in a predefined - dictionary of arguments, rather than unpacking and repacking the - dictionary as individual arguments using the ``*args`` and ``**kwargs`` - syntax. :meth:`vformat` does the work of breaking up the format string - into character data and replacement fields. It calls the various - methods described below. - - In addition, the :class:`Formatter` defines a number of methods that are - intended to be replaced by subclasses: - - .. method:: parse(format_string) - - Loop over the format_string and return an iterable of tuples - (*literal_text*, *field_name*, *format_spec*, *conversion*). This is used - by :meth:`vformat` to break the string into either literal text, or - replacement fields. - - The values in the tuple conceptually represent a span of literal text - followed by a single replacement field. If there is no literal text - (which can happen if two replacement fields occur consecutively), then - *literal_text* will be a zero-length string. If there is no replacement - field, then the values of *field_name*, *format_spec* and *conversion* - will be ``None``. - - .. method:: get_field(field_name, args, kwargs) - - Given *field_name* as returned by :meth:`parse` (see above), convert it to - an object to be formatted. Returns a tuple (obj, used_key). The default - version takes strings of the form defined in :pep:`3101`, such as - "0[name]" or "label.title". *args* and *kwargs* are as passed in to - :meth:`vformat`. The return value *used_key* has the same meaning as the - *key* parameter to :meth:`get_value`. - - .. method:: get_value(key, args, kwargs) - - Retrieve a given field value. The *key* argument will be either an - integer or a string. If it is an integer, it represents the index of the - positional argument in *args*; if it is a string, then it represents a - named argument in *kwargs*. - - The *args* parameter is set to the list of positional arguments to - :meth:`vformat`, and the *kwargs* parameter is set to the dictionary of - keyword arguments. - - For compound field names, these functions are only called for the first - component of the field name; Subsequent components are handled through - normal attribute and indexing operations. - - So for example, the field expression '0.name' would cause - :meth:`get_value` to be called with a *key* argument of 0. The ``name`` - attribute will be looked up after :meth:`get_value` returns by calling the - built-in :func:`getattr` function. - - If the index or keyword refers to an item that does not exist, then an - :exc:`IndexError` or :exc:`KeyError` should be raised. - - .. method:: check_unused_args(used_args, args, kwargs) - - Implement checking for unused arguments if desired. The arguments to this - function is the set of all argument keys that were actually referred to in - the format string (integers for positional arguments, and strings for - named arguments), and a reference to the *args* and *kwargs* that was - passed to vformat. The set of unused args can be calculated from these - parameters. :meth:`check_unused_args` is assumed to raise an exception if - the check fails. - - .. method:: format_field(value, format_spec) - - :meth:`format_field` simply calls the global :func:`format` built-in. The - method is provided so that subclasses can override it. - - .. method:: convert_field(value, conversion) - - Converts the value (returned by :meth:`get_field`) given a conversion type - (as in the tuple returned by the :meth:`parse` method). The default - version understands 's' (str), 'r' (repr) and 'a' (ascii) conversion - types. - - -.. _formatstrings: - -Format String Syntax --------------------- - -The :meth:`str.format` method and the :class:`Formatter` class share the same -syntax for format strings (although in the case of :class:`Formatter`, -subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but -there are differences. - -.. index:: - single: {} (curly brackets); in string formatting - single: . (dot); in string formatting - single: [] (square brackets); in string formatting - single: ! (exclamation); in string formatting - single: : (colon); in string formatting - -Format strings contain "replacement fields" surrounded by curly braces ``{}``. -Anything that is not contained in braces is considered literal text, which is -copied unchanged to the output. If you need to include a brace character in the -literal text, it can be escaped by doubling: ``{{`` and ``}}``. - -The grammar for a replacement field is as follows: - - .. productionlist:: sf - replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" - field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `digit`+] - attribute_name: `identifier` - element_index: `digit`+ | `index_string` - index_string: + - conversion: "r" | "s" | "a" - format_spec: - -In less formal terms, the replacement field can start with a *field_name* that specifies -the object whose value is to be formatted and inserted -into the output instead of the replacement field. -The *field_name* is optionally followed by a *conversion* field, which is -preceded by an exclamation point ``'!'``, and a *format_spec*, which is preceded -by a colon ``':'``. These specify a non-default format for the replacement value. - -See also the :ref:`formatspec` section. - -The *field_name* itself begins with an *arg_name* that is either a number or a -keyword. If it's a number, it refers to a positional argument, and if it's a keyword, -it refers to a named keyword argument. If the numerical arg_names in a format string -are 0, 1, 2, ... in sequence, they can all be omitted (not just some) -and the numbers 0, 1, 2, ... will be automatically inserted in that order. -Because *arg_name* is not quote-delimited, it is not possible to specify arbitrary -dictionary keys (e.g., the strings ``'10'`` or ``':-]'``) within a format string. -The *arg_name* can be followed by any number of index or -attribute expressions. An expression of the form ``'.name'`` selects the named -attribute using :func:`getattr`, while an expression of the form ``'[index]'`` -does an index lookup using :func:`__getitem__`. - -.. versionchanged:: 3.1 - The positional argument specifiers can be omitted for :meth:`str.format`, - so ``'{} {}'.format(a, b)`` is equivalent to ``'{0} {1}'.format(a, b)``. - -.. versionchanged:: 3.4 - The positional argument specifiers can be omitted for :class:`Formatter`. - -Some simple format string examples:: - - "First, thou shalt count to {0}" # References first positional argument - "Bring me a {}" # Implicitly references the first positional argument - "From {} to {}" # Same as "From {0} to {1}" - "My quest is {name}" # References keyword argument 'name' - "Weight in tons {0.weight}" # 'weight' attribute of first positional arg - "Units destroyed: {players[0]}" # First element of keyword argument 'players'. - -The *conversion* field causes a type coercion before formatting. Normally, the -job of formatting a value is done by the :meth:`__format__` method of the value -itself. However, in some cases it is desirable to force a type to be formatted -as a string, overriding its own definition of formatting. By converting the -value to a string before calling :meth:`__format__`, the normal formatting logic -is bypassed. - -Three conversion flags are currently supported: ``'!s'`` which calls :func:`str` -on the value, ``'!r'`` which calls :func:`repr` and ``'!a'`` which calls -:func:`ascii`. - -Some examples:: - - "Harold's a clever {0!s}" # Calls str() on the argument first - "Bring out the holy {name!r}" # Calls repr() on the argument first - "More {!a}" # Calls ascii() on the argument first - -The *format_spec* field contains a specification of how the value should be -presented, including such details as field width, alignment, padding, decimal -precision and so on. Each value type can define its own "formatting -mini-language" or interpretation of the *format_spec*. - -Most built-in types support a common formatting mini-language, which is -described in the next section. - -A *format_spec* field can also include nested replacement fields within it. -These nested replacement fields may contain a field name, conversion flag -and format specification, but deeper nesting is -not allowed. The replacement fields within the -format_spec are substituted before the *format_spec* string is interpreted. -This allows the formatting of a value to be dynamically specified. - -See the :ref:`formatexamples` section for some examples. - - -.. _formatspec: - -Format Specification Mini-Language -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -"Format specifications" are used within replacement fields contained within a -format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). -They can also be passed directly to the built-in -:func:`format` function. Each formattable type may define how the format -specification is to be interpreted. - -Most built-in types implement the following options for format specifications, -although some of the formatting options are only supported by the numeric types. - -A general convention is that an empty format string (``""``) produces -the same result as if you had called :func:`str` on the value. A -non-empty format string typically modifies the result. - -The general form of a *standard format specifier* is: - -.. productionlist:: sf - format_spec: [[`fill`]`align`][`sign`][#][0][`width`][`grouping_option`][.`precision`][`type`] - fill: - align: "<" | ">" | "=" | "^" - sign: "+" | "-" | " " - width: `digit`+ - grouping_option: "_" | "," - precision: `digit`+ - type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" - -If a valid *align* value is specified, it can be preceded by a *fill* -character that can be any character and defaults to a space if omitted. -It is not possible to use a literal curly brace ("``{``" or "``}``") as -the *fill* character in a :ref:`formatted string literal -` or when using the :meth:`str.format` -method. However, it is possible to insert a curly brace -with a nested replacement field. This limitation doesn't -affect the :func:`format` function. - -The meaning of the various alignment options is as follows: - - .. index:: - single: < (less); in string formatting - single: > (greater); in string formatting - single: = (equals); in string formatting - single: ^ (caret); in string formatting - - +---------+----------------------------------------------------------+ - | Option | Meaning | - +=========+==========================================================+ - | ``'<'`` | Forces the field to be left-aligned within the available | - | | space (this is the default for most objects). | - +---------+----------------------------------------------------------+ - | ``'>'`` | Forces the field to be right-aligned within the | - | | available space (this is the default for numbers). | - +---------+----------------------------------------------------------+ - | ``'='`` | Forces the padding to be placed after the sign (if any) | - | | but before the digits. This is used for printing fields | - | | in the form '+000000120'. This alignment option is only | - | | valid for numeric types. It becomes the default when '0'| - | | immediately precedes the field width. | - +---------+----------------------------------------------------------+ - | ``'^'`` | Forces the field to be centered within the available | - | | space. | - +---------+----------------------------------------------------------+ - -Note that unless a minimum field width is defined, the field width will always -be the same size as the data to fill it, so that the alignment option has no -meaning in this case. - -The *sign* option is only valid for number types, and can be one of the -following: - - .. index:: - single: + (plus); in string formatting - single: - (minus); in string formatting - single: space; in string formatting - - +---------+----------------------------------------------------------+ - | Option | Meaning | - +=========+==========================================================+ - | ``'+'`` | indicates that a sign should be used for both | - | | positive as well as negative numbers. | - +---------+----------------------------------------------------------+ - | ``'-'`` | indicates that a sign should be used only for negative | - | | numbers (this is the default behavior). | - +---------+----------------------------------------------------------+ - | space | indicates that a leading space should be used on | - | | positive numbers, and a minus sign on negative numbers. | - +---------+----------------------------------------------------------+ - - -.. index:: single: # (hash); in string formatting - -The ``'#'`` option causes the "alternate form" to be used for the -conversion. The alternate form is defined differently for different -types. This option is only valid for integer, float, complex and -Decimal types. For integers, when binary, octal, or hexadecimal output -is used, this option adds the prefix respective ``'0b'``, ``'0o'``, or -``'0x'`` to the output value. For floats, complex and Decimal the -alternate form causes the result of the conversion to always contain a -decimal-point character, even if no digits follow it. Normally, a -decimal-point character appears in the result of these conversions -only if a digit follows it. In addition, for ``'g'`` and ``'G'`` -conversions, trailing zeros are not removed from the result. - -.. index:: single: , (comma); in string formatting - -The ``','`` option signals the use of a comma for a thousands separator. -For a locale aware separator, use the ``'n'`` integer presentation type -instead. - -.. versionchanged:: 3.1 - Added the ``','`` option (see also :pep:`378`). - -.. index:: single: _ (underscore); in string formatting - -The ``'_'`` option signals the use of an underscore for a thousands -separator for floating point presentation types and for integer -presentation type ``'d'``. For integer presentation types ``'b'``, -``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 -digits. For other presentation types, specifying this option is an -error. - -.. versionchanged:: 3.6 - Added the ``'_'`` option (see also :pep:`515`). - -*width* is a decimal integer defining the minimum field width. If not -specified, then the field width will be determined by the content. - -When no explicit alignment is given, preceding the *width* field by a zero -(``'0'``) character enables -sign-aware zero-padding for numeric types. This is equivalent to a *fill* -character of ``'0'`` with an *alignment* type of ``'='``. - -The *precision* is a decimal number indicating how many digits should be -displayed after the decimal point for a floating point value formatted with -``'f'`` and ``'F'``, or before and after the decimal point for a floating point -value formatted with ``'g'`` or ``'G'``. For non-number types the field -indicates the maximum field size - in other words, how many characters will be -used from the field content. The *precision* is not allowed for integer values. - -Finally, the *type* determines how the data should be presented. - -The available string presentation types are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'s'`` | String format. This is the default type for strings and | - | | may be omitted. | - +---------+----------------------------------------------------------+ - | None | The same as ``'s'``. | - +---------+----------------------------------------------------------+ - -The available integer presentation types are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'b'`` | Binary format. Outputs the number in base 2. | - +---------+----------------------------------------------------------+ - | ``'c'`` | Character. Converts the integer to the corresponding | - | | unicode character before printing. | - +---------+----------------------------------------------------------+ - | ``'d'`` | Decimal Integer. Outputs the number in base 10. | - +---------+----------------------------------------------------------+ - | ``'o'`` | Octal format. Outputs the number in base 8. | - +---------+----------------------------------------------------------+ - | ``'x'`` | Hex format. Outputs the number in base 16, using | - | | lower-case letters for the digits above 9. | - +---------+----------------------------------------------------------+ - | ``'X'`` | Hex format. Outputs the number in base 16, using | - | | upper-case letters for the digits above 9. | - +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | - | | the current locale setting to insert the appropriate | - | | number separator characters. | - +---------+----------------------------------------------------------+ - | None | The same as ``'d'``. | - +---------+----------------------------------------------------------+ - -In addition to the above presentation types, integers can be formatted -with the floating point presentation types listed below (except -``'n'`` and ``None``). When doing so, :func:`float` is used to convert the -integer to a floating point number before formatting. - -The available presentation types for floating point and decimal values are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'e'`` | Exponent notation. Prints the number in scientific | - | | notation using the letter 'e' to indicate the exponent. | - | | The default precision is ``6``. | - +---------+----------------------------------------------------------+ - | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | - | | upper case 'E' as the separator character. | - +---------+----------------------------------------------------------+ - | ``'f'`` | Fixed-point notation. Displays the number as a | - | | fixed-point number. The default precision is ``6``. | - +---------+----------------------------------------------------------+ - | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | - | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | - +---------+----------------------------------------------------------+ - | ``'g'`` | General format. For a given precision ``p >= 1``, | - | | this rounds the number to ``p`` significant digits and | - | | then formats the result in either fixed-point format | - | | or in scientific notation, depending on its magnitude. | - | | | - | | The precise rules are as follows: suppose that the | - | | result formatted with presentation type ``'e'`` and | - | | precision ``p-1`` would have exponent ``exp``. Then | - | | if ``-4 <= exp < p``, the number is formatted | - | | with presentation type ``'f'`` and precision | - | | ``p-1-exp``. Otherwise, the number is formatted | - | | with presentation type ``'e'`` and precision ``p-1``. | - | | In both cases insignificant trailing zeros are removed | - | | from the significand, and the decimal point is also | - | | removed if there are no remaining digits following it. | - | | | - | | Positive and negative infinity, positive and negative | - | | zero, and nans, are formatted as ``inf``, ``-inf``, | - | | ``0``, ``-0`` and ``nan`` respectively, regardless of | - | | the precision. | - | | | - | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. The default precision is ``6``. | - +---------+----------------------------------------------------------+ - | ``'G'`` | General format. Same as ``'g'`` except switches to | - | | ``'E'`` if the number gets too large. The | - | | representations of infinity and NaN are uppercased, too. | - +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | - | | the current locale setting to insert the appropriate | - | | number separator characters. | - +---------+----------------------------------------------------------+ - | ``'%'`` | Percentage. Multiplies the number by 100 and displays | - | | in fixed (``'f'``) format, followed by a percent sign. | - +---------+----------------------------------------------------------+ - | None | Similar to ``'g'``, except that fixed-point notation, | - | | when used, has at least one digit past the decimal point.| - | | The default precision is as high as needed to represent | - | | the particular value. The overall effect is to match the | - | | output of :func:`str` as altered by the other format | - | | modifiers. | - +---------+----------------------------------------------------------+ - - -.. _formatexamples: - -Format examples -^^^^^^^^^^^^^^^ - -This section contains examples of the :meth:`str.format` syntax and -comparison with the old ``%``-formatting. - -In most of the cases the syntax is similar to the old ``%``-formatting, with the -addition of the ``{}`` and with ``:`` used instead of ``%``. -For example, ``'%03.2f'`` can be translated to ``'{:03.2f}'``. - -The new format syntax also supports new and different options, shown in the -following examples. - -Accessing arguments by position:: - - >>> '{0}, {1}, {2}'.format('a', 'b', 'c') - 'a, b, c' - >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only - 'a, b, c' - >>> '{2}, {1}, {0}'.format('a', 'b', 'c') - 'c, b, a' - >>> '{2}, {1}, {0}'.format(*'abc') # unpacking argument sequence - 'c, b, a' - >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' indices can be repeated - 'abracadabra' - -Accessing arguments by name:: - - >>> 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') - 'Coordinates: 37.24N, -115.81W' - >>> coord = {'latitude': '37.24N', 'longitude': '-115.81W'} - >>> 'Coordinates: {latitude}, {longitude}'.format(**coord) - 'Coordinates: 37.24N, -115.81W' - -Accessing arguments' attributes:: - - >>> c = 3-5j - >>> ('The complex number {0} is formed from the real part {0.real} ' - ... 'and the imaginary part {0.imag}.').format(c) - 'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.' - >>> class Point: - ... def __init__(self, x, y): - ... self.x, self.y = x, y - ... def __str__(self): - ... return 'Point({self.x}, {self.y})'.format(self=self) - ... - >>> str(Point(4, 2)) - 'Point(4, 2)' - -Accessing arguments' items:: - - >>> coord = (3, 5) - >>> 'X: {0[0]}; Y: {0[1]}'.format(coord) - 'X: 3; Y: 5' - -Replacing ``%s`` and ``%r``:: - - >>> "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') - "repr() shows quotes: 'test1'; str() doesn't: test2" - -Aligning the text and specifying a width:: - - >>> '{:<30}'.format('left aligned') - 'left aligned ' - >>> '{:>30}'.format('right aligned') - ' right aligned' - >>> '{:^30}'.format('centered') - ' centered ' - >>> '{:*^30}'.format('centered') # use '*' as a fill char - '***********centered***********' - -Replacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:: - - >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it always - '+3.140000; -3.140000' - >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space for positive numbers - ' 3.140000; -3.140000' - >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the minus -- same as '{:f}; {:f}' - '3.140000; -3.140000' - -Replacing ``%x`` and ``%o`` and converting the value to different bases:: - - >>> # format also supports binary numbers - >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42) - 'int: 42; hex: 2a; oct: 52; bin: 101010' - >>> # with 0x, 0o, or 0b as prefix: - >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42) - 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' - -Using the comma as a thousands separator:: - - >>> '{:,}'.format(1234567890) - '1,234,567,890' - -Expressing a percentage:: - - >>> points = 19 - >>> total = 22 - >>> 'Correct answers: {:.2%}'.format(points/total) - 'Correct answers: 86.36%' - -Using type-specific formatting:: - - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) - >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) - '2010-07-04 12:15:58' - -Nesting arguments and more complex examples:: - - >>> for align, text in zip('<^>', ['left', 'center', 'right']): - ... '{0:{fill}{align}16}'.format(text, fill=align, align=align) - ... - 'left<<<<<<<<<<<<' - '^^^^^center^^^^^' - '>>>>>>>>>>>right' - >>> - >>> octets = [192, 168, 0, 1] - >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets) - 'C0A80001' - >>> int(_, 16) - 3232235521 - >>> - >>> width = 5 - >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE - ... for base in 'dXob': - ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') - ... print() - ... - 5 5 5 101 - 6 6 6 110 - 7 7 7 111 - 8 8 10 1000 - 9 9 11 1001 - 10 A 12 1010 - 11 B 13 1011 - - - -.. _template-strings: - -Template strings ----------------- - -Template strings provide simpler string substitutions as described in -:pep:`292`. A primary use case for template strings is for -internationalization (i18n) since in that context, the simpler syntax and -functionality makes it easier to translate than other built-in string -formatting facilities in Python. As an example of a library built on template -strings for i18n, see the -`flufl.i18n `_ package. - -.. index:: single: $ (dollar); in template strings - -Template strings support ``$``-based substitutions, using the following rules: - -* ``$$`` is an escape; it is replaced with a single ``$``. - -* ``$identifier`` names a substitution placeholder matching a mapping key of - ``"identifier"``. By default, ``"identifier"`` is restricted to any - case-insensitive ASCII alphanumeric string (including underscores) that - starts with an underscore or ASCII letter. The first non-identifier - character after the ``$`` character terminates this placeholder - specification. - -* ``${identifier}`` is equivalent to ``$identifier``. It is required when - valid identifier characters follow the placeholder but are not part of the - placeholder, such as ``"${noun}ification"``. - -Any other appearance of ``$`` in the string will result in a :exc:`ValueError` -being raised. - -The :mod:`string` module provides a :class:`Template` class that implements -these rules. The methods of :class:`Template` are: - - -.. class:: Template(template) - - The constructor takes a single argument which is the template string. - - - .. method:: substitute(mapping, **kwds) - - Performs the template substitution, returning a new string. *mapping* is - any dictionary-like object with keys that match the placeholders in the - template. Alternatively, you can provide keyword arguments, where the - keywords are the placeholders. When both *mapping* and *kwds* are given - and there are duplicates, the placeholders from *kwds* take precedence. - - - .. method:: safe_substitute(mapping, **kwds) - - Like :meth:`substitute`, except that if placeholders are missing from - *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the - original placeholder will appear in the resulting string intact. Also, - unlike with :meth:`substitute`, any other appearances of the ``$`` will - simply return ``$`` instead of raising :exc:`ValueError`. - - While other exceptions may still occur, this method is called "safe" - because it always tries to return a usable string instead of - raising an exception. In another sense, :meth:`safe_substitute` may be - anything other than safe, since it will silently ignore malformed - templates containing dangling delimiters, unmatched braces, or - placeholders that are not valid Python identifiers. - - :class:`Template` instances also provide one public data attribute: - - .. attribute:: template - - This is the object passed to the constructor's *template* argument. In - general, you shouldn't change it, but read-only access is not enforced. - -Here is an example of how to use a Template:: - - >>> from string import Template - >>> s = Template('$who likes $what') - >>> s.substitute(who='tim', what='kung pao') - 'tim likes kung pao' - >>> d = dict(who='tim') - >>> Template('Give $who $100').substitute(d) - Traceback (most recent call last): - ... - ValueError: Invalid placeholder in string: line 1, col 11 - >>> Template('$who likes $what').substitute(d) - Traceback (most recent call last): - ... - KeyError: 'what' - >>> Template('$who likes $what').safe_substitute(d) - 'tim likes $what' - -Advanced usage: you can derive subclasses of :class:`Template` to customize the -placeholder syntax, delimiter character, or the entire regular expression used -to parse template strings. To do this, you can override these class attributes: - -* *delimiter* -- This is the literal string describing a placeholder introducing - delimiter. The default value is ``$``. Note that this should *not* be a - regular expression, as the implementation will call :meth:`re.escape` on this - string as needed. - -* *idpattern* -- This is the regular expression describing the pattern for - non-braced placeholders (the braces will be added automatically as - appropriate). The default value is the regular expression - ``(?-i:[_a-zA-Z][_a-zA-Z0-9]*)``. - - .. note:: - - Since default *flags* is ``re.IGNORECASE``, pattern ``[a-z]`` can match - with some non-ASCII characters. That's why we use local ``-i`` flag here. - - While *flags* is kept to ``re.IGNORECASE`` for backward compatibility, - you can override it to ``0`` or ``re.IGNORECASE | re.ASCII`` when - subclassing. - - -* *flags* -- The regular expression flags that will be applied when compiling - the regular expression used for recognizing substitutions. The default value - is ``re.IGNORECASE``. Note that ``re.VERBOSE`` will always be added to the - flags, so custom *idpattern*\ s must follow conventions for verbose regular - expressions. - - .. versionadded:: 3.2 - -Alternatively, you can provide the entire regular expression pattern by -overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing -groups correspond to the rules given above, along with the invalid placeholder -rule: - -* *escaped* -- This group matches the escape sequence, e.g. ``$$``, in the - default pattern. - -* *named* -- This group matches the unbraced placeholder name; it should not - include the delimiter in capturing group. - -* *braced* -- This group matches the brace enclosed placeholder name; it should - not include either the delimiter or braces in the capturing group. - -* *invalid* -- This group matches any other delimiter pattern (usually a single - delimiter), and it should appear last in the regular expression. - - -Helper functions ----------------- - -.. function:: capwords(s, sep=None) - - Split the argument into words using :meth:`str.split`, capitalize each word - using :meth:`str.capitalize`, and join the capitalized words using - :meth:`str.join`. If the optional second argument *sep* is absent - or ``None``, runs of whitespace characters are replaced by a single space - and leading and trailing whitespace are removed, otherwise *sep* is used to - split and join the words. - diff --git a/third_party/python/Doc/library/stringprep.rst b/third_party/python/Doc/library/stringprep.rst deleted file mode 100644 index 330032ba1..000000000 --- a/third_party/python/Doc/library/stringprep.rst +++ /dev/null @@ -1,143 +0,0 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= - -.. module:: stringprep - :synopsis: String preparation, as per RFC 3453 - -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/stringprep.py` - --------------- - -When identifying things (such as host names) in the internet, it is often -necessary to compare such identifications for "equality". Exactly how this -comparison is executed may depend on the application domain, e.g. whether it -should be case-insensitive or not. It may be also necessary to restrict the -possible identifications, to allow only identifications consisting of -"printable" characters. - -:rfc:`3454` defines a procedure for "preparing" Unicode strings in internet -protocols. Before passing strings onto the wire, they are processed with the -preparation procedure, after which they have a certain normalized form. The RFC -defines a set of tables, which can be combined into profiles. Each profile must -define which tables it uses, and what other optional parts of the ``stringprep`` -procedure are part of the profile. One example of a ``stringprep`` profile is -``nameprep``, which is used for internationalized domain names. - -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these -tables would be very large to represent them as dictionaries or lists, the -module uses the Unicode character database internally. The module source code -itself was generated using the ``mkstringprep.py`` utility. - -As a result, these tables are exposed as functions, not as data structures. -There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that -returns true if the parameter is part of the set. For mappings, it provides the -mapping function: given the key, it returns the associated value. Below is a -list of all functions available in the module. - - -.. function:: in_table_a1(code) - - Determine whether *code* is in tableA.1 (Unassigned code points in Unicode 3.2). - - -.. function:: in_table_b1(code) - - Determine whether *code* is in tableB.1 (Commonly mapped to nothing). - - -.. function:: map_table_b2(code) - - Return the mapped value for *code* according to tableB.2 (Mapping for - case-folding used with NFKC). - - -.. function:: map_table_b3(code) - - Return the mapped value for *code* according to tableB.3 (Mapping for - case-folding used with no normalization). - - -.. function:: in_table_c11(code) - - Determine whether *code* is in tableC.1.1 (ASCII space characters). - - -.. function:: in_table_c12(code) - - Determine whether *code* is in tableC.1.2 (Non-ASCII space characters). - - -.. function:: in_table_c11_c12(code) - - Determine whether *code* is in tableC.1 (Space characters, union of C.1.1 and - C.1.2). - - -.. function:: in_table_c21(code) - - Determine whether *code* is in tableC.2.1 (ASCII control characters). - - -.. function:: in_table_c22(code) - - Determine whether *code* is in tableC.2.2 (Non-ASCII control characters). - - -.. function:: in_table_c21_c22(code) - - Determine whether *code* is in tableC.2 (Control characters, union of C.2.1 and - C.2.2). - - -.. function:: in_table_c3(code) - - Determine whether *code* is in tableC.3 (Private use). - - -.. function:: in_table_c4(code) - - Determine whether *code* is in tableC.4 (Non-character code points). - - -.. function:: in_table_c5(code) - - Determine whether *code* is in tableC.5 (Surrogate codes). - - -.. function:: in_table_c6(code) - - Determine whether *code* is in tableC.6 (Inappropriate for plain text). - - -.. function:: in_table_c7(code) - - Determine whether *code* is in tableC.7 (Inappropriate for canonical - representation). - - -.. function:: in_table_c8(code) - - Determine whether *code* is in tableC.8 (Change display properties or are - deprecated). - - -.. function:: in_table_c9(code) - - Determine whether *code* is in tableC.9 (Tagging characters). - - -.. function:: in_table_d1(code) - - Determine whether *code* is in tableD.1 (Characters with bidirectional property - "R" or "AL"). - - -.. function:: in_table_d2(code) - - Determine whether *code* is in tableD.2 (Characters with bidirectional property - "L"). - diff --git a/third_party/python/Doc/library/struct.rst b/third_party/python/Doc/library/struct.rst deleted file mode 100644 index 32bc71f2d..000000000 --- a/third_party/python/Doc/library/struct.rst +++ /dev/null @@ -1,464 +0,0 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= - -.. module:: struct - :synopsis: Interpret bytes as packed binary data. - -**Source code:** :source:`Lib/struct.py` - -.. index:: - pair: C; structures - triple: packing; binary; data - --------------- - -This module performs conversions between Python values and C structs represented -as Python :class:`bytes` objects. This can be used in handling binary data -stored in files or from network connections, among other sources. It uses -:ref:`struct-format-strings` as compact descriptions of the layout of the C -structs and the intended conversion to/from Python values. - -.. note:: - - By default, the result of packing a given C struct includes pad bytes in - order to maintain proper alignment for the C types involved; similarly, - alignment is taken into account when unpacking. This behavior is chosen so - that the bytes of a packed struct correspond exactly to the layout in memory - of the corresponding C struct. To handle platform-independent data formats - or omit implicit pad bytes, use ``standard`` size and alignment instead of - ``native`` size and alignment: see :ref:`struct-alignment` for details. - -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* -argument. This refers to objects that implement the :ref:`bufferobjects` and -provide either a readable or read-writable buffer. The most common types used -for that purpose are :class:`bytes` and :class:`bytearray`, but many other types -that can be viewed as an array of bytes implement the buffer protocol, so that -they can be read/filled without additional copying from a :class:`bytes` object. - - -Functions and Exceptions ------------------------- - -The module defines the following exception and functions: - - -.. exception:: error - - Exception raised on various occasions; argument is a string describing what - is wrong. - - -.. function:: pack(fmt, v1, v2, ...) - - Return a bytes object containing the values *v1*, *v2*, ... packed according - to the format string *fmt*. The arguments must match the values required by - the format exactly. - - -.. function:: pack_into(fmt, buffer, offset, v1, v2, ...) - - Pack the values *v1*, *v2*, ... according to the format string *fmt* and - write the packed bytes into the writable buffer *buffer* starting at - position *offset*. Note that *offset* is a required argument. - - -.. function:: unpack(fmt, buffer) - - Unpack from the buffer *buffer* (presumably packed by ``pack(fmt, ...)``) - according to the format string *fmt*. The result is a tuple even if it - contains exactly one item. The buffer's size in bytes must match the - size required by the format, as reflected by :func:`calcsize`. - - -.. function:: unpack_from(fmt, buffer, offset=0) - - Unpack from *buffer* starting at position *offset*, according to the format - string *fmt*. The result is a tuple even if it contains exactly one - item. The buffer's size in bytes, minus *offset*, must be at least - the size required by the format, as reflected by :func:`calcsize`. - - -.. function:: iter_unpack(fmt, buffer) - - Iteratively unpack from the buffer *buffer* according to the format - string *fmt*. This function returns an iterator which will read - equally-sized chunks from the buffer until all its contents have been - consumed. The buffer's size in bytes must be a multiple of the size - required by the format, as reflected by :func:`calcsize`. - - Each iteration yields a tuple as specified by the format string. - - .. versionadded:: 3.4 - - -.. function:: calcsize(fmt) - - Return the size of the struct (and hence of the bytes object produced by - ``pack(fmt, ...)``) corresponding to the format string *fmt*. - -.. _struct-format-strings: - -Format Strings --------------- - -Format strings are the mechanism used to specify the expected layout when -packing and unpacking data. They are built up from :ref:`format-characters`, -which specify the type of data being packed/unpacked. In addition, there are -special characters for controlling the :ref:`struct-alignment`. - - -.. _struct-alignment: - -Byte Order, Size, and Alignment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, C types are represented in the machine's native format and byte -order, and properly aligned by skipping pad bytes if necessary (according to the -rules used by the C compiler). - -.. index:: - single: @ (at); in struct format strings - single: = (equals); in struct format strings - single: < (less); in struct format strings - single: > (greater); in struct format strings - single: ! (exclamation); in struct format strings - -Alternatively, the first character of the format string can be used to indicate -the byte order, size and alignment of the packed data, according to the -following table: - -+-----------+------------------------+----------+-----------+ -| Character | Byte order | Size | Alignment | -+===========+========================+==========+===========+ -| ``@`` | native | native | native | -+-----------+------------------------+----------+-----------+ -| ``=`` | native | standard | none | -+-----------+------------------------+----------+-----------+ -| ``<`` | little-endian | standard | none | -+-----------+------------------------+----------+-----------+ -| ``>`` | big-endian | standard | none | -+-----------+------------------------+----------+-----------+ -| ``!`` | network (= big-endian) | standard | none | -+-----------+------------------------+----------+-----------+ - -If the first character is not one of these, ``'@'`` is assumed. - -Native byte order is big-endian or little-endian, depending on the host -system. For example, Intel x86 and AMD64 (x86-64) are little-endian; -Motorola 68000 and PowerPC G5 are big-endian; ARM and Intel Itanium feature -switchable endianness (bi-endian). Use ``sys.byteorder`` to check the -endianness of your system. - -Native size and alignment are determined using the C compiler's -``sizeof`` expression. This is always combined with native byte order. - -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. - -Note the difference between ``'@'`` and ``'='``: both use native byte order, but -the size and alignment of the latter is standardized. - -The form ``'!'`` is available for those poor souls who claim they can't remember -whether network byte order is big-endian or little-endian. - -There is no way to indicate non-native byte order (force byte-swapping); use the -appropriate choice of ``'<'`` or ``'>'``. - -Notes: - -(1) Padding is only automatically added between successive structure members. - No padding is added at the beginning or the end of the encoded struct. - -(2) No padding is added when using non-native size and alignment, e.g. - with '<', '>', '=', and '!'. - -(3) To align the end of a structure to the alignment requirement of a - particular type, end the format with the code for that type with a repeat - count of zero. See :ref:`struct-examples`. - - -.. _format-characters: - -Format Characters -^^^^^^^^^^^^^^^^^ - -Format characters have the following meaning; the conversion between C and -Python values should be obvious given their types. The 'Standard size' column -refers to the size of the packed value in bytes when using standard size; that -is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or -``'='``. When using native size, the size of the packed value is -platform-dependent. - -+--------+--------------------------+--------------------+----------------+------------+ -| Format | C Type | Python type | Standard size | Notes | -+========+==========================+====================+================+============+ -| ``x`` | pad byte | no value | | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``c`` | :c:type:`char` | bytes of length 1 | 1 | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:type:`signed char` | integer | 1 | \(1),\(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:type:`unsigned char` | integer | 1 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``?`` | :c:type:`_Bool` | bool | 1 | \(1) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:type:`short` | integer | 2 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:type:`unsigned short` | integer | 2 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:type:`int` | integer | 4 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:type:`unsigned int` | integer | 4 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:type:`long` | integer | 4 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:type:`unsigned long` | integer | 4 | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:type:`long long` | integer | 8 | \(2), \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:type:`unsigned long | integer | 8 | \(2), \(3) | -| | long` | | | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(4) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(4) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(7) | float | 2 | \(5) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``f`` | :c:type:`float` | float | 4 | \(5) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``d`` | :c:type:`double` | float | 8 | \(5) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``s`` | :c:type:`char[]` | bytes | | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``p`` | :c:type:`char[]` | bytes | | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:type:`void \*` | integer | | \(6) | -+--------+--------------------------+--------------------+----------------+------------+ - -.. versionchanged:: 3.3 - Added support for the ``'n'`` and ``'N'`` formats. - -.. versionchanged:: 3.6 - Added support for the ``'e'`` format. - - -Notes: - -(1) - .. index:: single: ? (question mark); in struct format strings - - The ``'?'`` conversion code corresponds to the :c:type:`_Bool` type defined by - C99. If this type is not available, it is simulated using a :c:type:`char`. In - standard mode, it is always represented by one byte. - -(2) - The ``'q'`` and ``'Q'`` conversion codes are available in native mode only if - the platform C compiler supports C :c:type:`long long`, or, on Windows, - :c:type:`__int64`. They are always available in standard modes. - -(3) - When attempting to pack a non-integer using any of the integer conversion - codes, if the non-integer has a :meth:`__index__` method then that method is - called to convert the argument to an integer before packing. - - .. versionchanged:: 3.2 - Use of the :meth:`__index__` method for non-integers is new in 3.2. - -(4) - The ``'n'`` and ``'N'`` conversion codes are only available for the native - size (selected as the default or with the ``'@'`` byte order character). - For the standard size, you can use whichever of the other integer formats - fits your application. - -(5) - For the ``'f'``, ``'d'`` and ``'e'`` conversion codes, the packed - representation uses the IEEE 754 binary32, binary64 or binary16 format (for - ``'f'``, ``'d'`` or ``'e'`` respectively), regardless of the floating-point - format used by the platform. - -(6) - The ``'P'`` format character is only available for the native byte ordering - (selected as the default or with the ``'@'`` byte order character). The byte - order character ``'='`` chooses to use little- or big-endian ordering based - on the host system. The struct module does not interpret this as native - ordering, so the ``'P'`` format is not available. - -(7) - The IEEE 754 binary16 "half precision" type was introduced in the 2008 - revision of the `IEEE 754 standard `_. It has a sign - bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), - and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a - typical machine, an unsigned short can be used for storage, but not for math - operations. See the Wikipedia page on the `half-precision floating-point - format `_ for more information. - - -A format character may be preceded by an integral repeat count. For example, -the format string ``'4h'`` means exactly the same as ``'hhhh'``. - -Whitespace characters between formats are ignored; a count and its format must -not contain whitespace though. - -For the ``'s'`` format character, the count is interpreted as the length of the -bytes, not a repeat count like for the other format characters; for example, -``'10s'`` means a single 10-byte string, while ``'10c'`` means 10 characters. -If a count is not given, it defaults to 1. For packing, the string is -truncated or padded with null bytes as appropriate to make it fit. For -unpacking, the resulting bytes object always has exactly the specified number -of bytes. As a special case, ``'0s'`` means a single, empty string (while -``'0c'`` means 0 characters). - -When packing a value ``x`` using one of the integer formats (``'b'``, -``'B'``, ``'h'``, ``'H'``, ``'i'``, ``'I'``, ``'l'``, ``'L'``, -``'q'``, ``'Q'``), if ``x`` is outside the valid range for that format -then :exc:`struct.error` is raised. - -.. versionchanged:: 3.1 - In 3.0, some of the integer formats wrapped out-of-range values and - raised :exc:`DeprecationWarning` instead of :exc:`struct.error`. - -The ``'p'`` format character encodes a "Pascal string", meaning a short -variable-length string stored in a *fixed number of bytes*, given by the count. -The first byte stored is the length of the string, or 255, whichever is -smaller. The bytes of the string follow. If the string passed in to -:func:`pack` is too long (longer than the count minus 1), only the leading -``count-1`` bytes of the string are stored. If the string is shorter than -``count-1``, it is padded with null bytes so that exactly count bytes in all -are used. Note that for :func:`unpack`, the ``'p'`` format character consumes -``count`` bytes, but that the string returned can never contain more than 255 -bytes. - -.. index:: single: ? (question mark); in struct format strings - -For the ``'?'`` format character, the return value is either :const:`True` or -:const:`False`. When packing, the truth value of the argument object is used. -Either 0 or 1 in the native or standard bool representation will be packed, and -any non-zero value will be ``True`` when unpacking. - - - -.. _struct-examples: - -Examples -^^^^^^^^ - -.. note:: - All examples assume a native byte order, size, and alignment with a - big-endian machine. - -A basic example of packing/unpacking three integers:: - - >>> from struct import * - >>> pack('hhl', 1, 2, 3) - b'\x00\x01\x00\x02\x00\x00\x00\x03' - >>> unpack('hhl', b'\x00\x01\x00\x02\x00\x00\x00\x03') - (1, 2, 3) - >>> calcsize('hhl') - 8 - -Unpacked fields can be named by assigning them to variables or by wrapping -the result in a named tuple:: - - >>> record = b'raymond \x32\x12\x08\x01\x08' - >>> name, serialnum, school, gradelevel = unpack('<10sHHb', record) - - >>> from collections import namedtuple - >>> Student = namedtuple('Student', 'name serialnum school gradelevel') - >>> Student._make(unpack('<10sHHb', record)) - Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) - -The ordering of format characters may have an impact on size since the padding -needed to satisfy alignment requirements is different:: - - >>> pack('ci', b'*', 0x12131415) - b'*\x00\x00\x00\x12\x13\x14\x15' - >>> pack('ic', 0x12131415, b'*') - b'\x12\x13\x14\x15*' - >>> calcsize('ci') - 8 - >>> calcsize('ic') - 5 - -The following format ``'llh0l'`` specifies two pad bytes at the end, assuming -longs are aligned on 4-byte boundaries:: - - >>> pack('llh0l', 1, 2, 3) - b'\x00\x00\x00\x01\x00\x00\x00\x02\x00\x03\x00\x00' - -This only works when native size and alignment are in effect; standard size and -alignment does not enforce any alignment. - - -.. seealso:: - - Module :mod:`array` - Packed binary storage of homogeneous data. - - Module :mod:`xdrlib` - Packing and unpacking of XDR data. - - -.. _struct-objects: - -Classes -------- - -The :mod:`struct` module also defines the following type: - - -.. class:: Struct(format) - - Return a new Struct object which writes and reads binary data according to - the format string *format*. Creating a Struct object once and calling its - methods is more efficient than calling the :mod:`struct` functions with the - same format since the format string only needs to be compiled once. - - - Compiled Struct objects support the following methods and attributes: - - .. method:: pack(v1, v2, ...) - - Identical to the :func:`pack` function, using the compiled format. - (``len(result)`` will equal :attr:`size`.) - - - .. method:: pack_into(buffer, offset, v1, v2, ...) - - Identical to the :func:`pack_into` function, using the compiled format. - - - .. method:: unpack(buffer) - - Identical to the :func:`unpack` function, using the compiled format. - The buffer's size in bytes must equal :attr:`size`. - - - .. method:: unpack_from(buffer, offset=0) - - Identical to the :func:`unpack_from` function, using the compiled format. - The buffer's size in bytes, minus *offset*, must be at least - :attr:`size`. - - - .. method:: iter_unpack(buffer) - - Identical to the :func:`iter_unpack` function, using the compiled format. - The buffer's size in bytes must be a multiple of :attr:`size`. - - .. versionadded:: 3.4 - - .. attribute:: format - - The format string used to construct this Struct object. - - .. attribute:: size - - The calculated size of the struct (and hence of the bytes object produced - by the :meth:`pack` method) corresponding to :attr:`format`. - - -.. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format - -.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_floating_point#IEEE_754-2008 diff --git a/third_party/python/Doc/library/subprocess.rst b/third_party/python/Doc/library/subprocess.rst deleted file mode 100644 index dfb183aba..000000000 --- a/third_party/python/Doc/library/subprocess.rst +++ /dev/null @@ -1,1258 +0,0 @@ -:mod:`subprocess` --- Subprocess management -=========================================== - -.. module:: subprocess - :synopsis: Subprocess management. - -.. moduleauthor:: Peter Åstrand -.. sectionauthor:: Peter Åstrand - -**Source code:** :source:`Lib/subprocess.py` - --------------- - -The :mod:`subprocess` module allows you to spawn new processes, connect to their -input/output/error pipes, and obtain their return codes. This module intends to -replace several older modules and functions:: - - os.system - os.spawn* - -Information about how the :mod:`subprocess` module can be used to replace these -modules and functions can be found in the following sections. - -.. seealso:: - - :pep:`324` -- PEP proposing the subprocess module - - -Using the :mod:`subprocess` Module ----------------------------------- - -The recommended approach to invoking subprocesses is to use the :func:`run` -function for all use cases it can handle. For more advanced use cases, the -underlying :class:`Popen` interface can be used directly. - -The :func:`run` function was added in Python 3.5; if you need to retain -compatibility with older versions, see the :ref:`call-function-trio` section. - - -.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - shell=False, cwd=None, timeout=None, check=False, \ - encoding=None, errors=None, env=None) - - Run the command described by *args*. Wait for command to complete, then - return a :class:`CompletedProcess` instance. - - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - apart from *timeout*, - *input* and *check*, all the arguments to this function are passed through to - that interface. - - This does not capture stdout or stderr by default. To do so, pass - :data:`PIPE` for the *stdout* and/or *stderr* arguments. - - The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout - expires, the child process will be killed and waited for. The - :exc:`TimeoutExpired` exception will be re-raised after the child process - has terminated. - - The *input* argument is passed to :meth:`Popen.communicate` and thus to the - subprocess's stdin. If used it must be a byte sequence, or a string if - *encoding* or *errors* is specified or *universal_newlines* is true. When - used, the internal :class:`Popen` object is automatically created with - ``stdin=PIPE``, and the *stdin* argument may not be used as well. - - If *check* is true, and the process exits with a non-zero exit code, a - :exc:`CalledProcessError` exception will be raised. Attributes of that - exception hold the arguments, the exit code, and stdout and stderr if they - were captured. - - If *encoding* or *errors* are specified, or *universal_newlines* is true, - file objects for stdin, stdout and stderr are opened in text mode using the - specified *encoding* and *errors* or the :class:`io.TextIOWrapper` default. - Otherwise, file objects are opened in binary mode. - - If *env* is not ``None``, it must be a mapping that defines the environment - variables for the new process; these are used instead of the default - behavior of inheriting the current process' environment. It is passed directly - to :class:`Popen`. - - Examples:: - - >>> subprocess.run(["ls", "-l"]) # doesn't capture output - CompletedProcess(args=['ls', '-l'], returncode=0) - - >>> subprocess.run("exit 1", shell=True, check=True) - Traceback (most recent call last): - ... - subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - - >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) - CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, - stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - - Added *encoding* and *errors* parameters - -.. class:: CompletedProcess - - The return value from :func:`run`, representing a process that has finished. - - .. attribute:: args - - The arguments used to launch the process. This may be a list or a string. - - .. attribute:: returncode - - Exit status of the child process. Typically, an exit status of 0 indicates - that it ran successfully. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). - - .. attribute:: stdout - - Captured stdout from the child process. A bytes sequence, or a string if - :func:`run` was called with an encoding or errors. ``None`` if stdout was not - captured. - - If you ran the process with ``stderr=subprocess.STDOUT``, stdout and - stderr will be combined in this attribute, and :attr:`stderr` will be - ``None``. - - .. attribute:: stderr - - Captured stderr from the child process. A bytes sequence, or a string if - :func:`run` was called with an encoding or errors. ``None`` if stderr was not - captured. - - .. method:: check_returncode() - - If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`. - - .. versionadded:: 3.5 - -.. data:: DEVNULL - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :class:`Popen` and indicates that the special file :data:`os.devnull` - will be used. - - .. versionadded:: 3.3 - - -.. data:: PIPE - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :class:`Popen` and indicates that a pipe to the standard stream should be - opened. Most useful with :meth:`Popen.communicate`. - - -.. data:: STDOUT - - Special value that can be used as the *stderr* argument to :class:`Popen` and - indicates that standard error should go into the same handle as standard - output. - - -.. exception:: SubprocessError - - Base class for all other exceptions from this module. - - .. versionadded:: 3.3 - - -.. exception:: TimeoutExpired - - Subclass of :exc:`SubprocessError`, raised when a timeout expires - while waiting for a child process. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: timeout - - Timeout in seconds. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - -.. exception:: CalledProcessError - - Subclass of :exc:`SubprocessError`, raised when a process run by - :func:`check_call` or :func:`check_output` returns a non-zero exit status. - - .. attribute:: returncode - - Exit status of the child process. If the process exited due to a - signal, this will be the negative signal number. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - - -.. _frequently-used-arguments: - -Frequently Used Arguments -^^^^^^^^^^^^^^^^^^^^^^^^^ - -To support a wide variety of use cases, the :class:`Popen` constructor (and -the convenience functions) accept a large number of optional arguments. For -most typical use cases, many of these arguments can be safely left at their -default values. The arguments that are most commonly needed are: - - *args* is required for all calls and should be a string, or a sequence of - program arguments. Providing a sequence of arguments is generally - preferred, as it allows the module to take care of any required escaping - and quoting of arguments (e.g. to permit spaces in file names). If passing - a single string, either *shell* must be :const:`True` (see below) or else - the string must simply name the program to be executed without specifying - any arguments. - - *stdin*, *stdout* and *stderr* specify the executed program's standard input, - standard output and standard error file handles, respectively. Valid values - are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive - integer), an existing file object, and ``None``. :data:`PIPE` indicates - that a new pipe to the child should be created. :data:`DEVNULL` indicates - that the special file :data:`os.devnull` will be used. With the default - settings of ``None``, no redirection will occur; the child's file handles - will be inherited from the parent. Additionally, *stderr* can be - :data:`STDOUT`, which indicates that the stderr data from the child - process should be captured into the same file handle as for *stdout*. - - .. index:: - single: universal newlines; subprocess module - - If *encoding* or *errors* are specified, or *universal_newlines* is true, - the file objects *stdin*, *stdout* and *stderr* will be opened in text - mode using the *encoding* and *errors* specified in the call or the - defaults for :class:`io.TextIOWrapper`. - - For *stdin*, line ending characters ``'\n'`` in the input will be converted - to the default line separator :data:`os.linesep`. For *stdout* and *stderr*, - all line endings in the output will be converted to ``'\n'``. For more - information see the documentation of the :class:`io.TextIOWrapper` class - when the *newline* argument to its constructor is ``None``. - - If text mode is not used, *stdin*, *stdout* and *stderr* will be opened as - binary streams. No encoding or line ending conversion is performed. - - .. versionadded:: 3.6 - Added *encoding* and *errors* parameters. - - .. note:: - - The newlines attribute of the file objects :attr:`Popen.stdin`, - :attr:`Popen.stdout` and :attr:`Popen.stderr` are not updated by - the :meth:`Popen.communicate` method. - - If *shell* is ``True``, the specified command will be executed through - the shell. This can be useful if you are using Python primarily for the - enhanced control flow it offers over most system shells and still want - convenient access to other shell features such as shell pipes, filename - wildcards, environment variable expansion, and expansion of ``~`` to a - user's home directory. However, note that Python itself offers - implementations of many shell-like features (in particular, :mod:`glob`, - :mod:`fnmatch`, :func:`os.walk`, :func:`os.path.expandvars`, - :func:`os.path.expanduser`, and :mod:`shutil`). - - .. versionchanged:: 3.3 - When *universal_newlines* is ``True``, the class uses the encoding - :func:`locale.getpreferredencoding(False) ` - instead of ``locale.getpreferredencoding()``. See the - :class:`io.TextIOWrapper` class for more information on this change. - - .. note:: - - Read the `Security Considerations`_ section before using ``shell=True``. - -These options, along with all of the other options, are described in more -detail in the :class:`Popen` constructor documentation. - - -Popen Constructor -^^^^^^^^^^^^^^^^^ - -The underlying process creation and management in this module is handled by -the :class:`Popen` class. It offers a lot of flexibility so that developers -are able to handle the less common cases not covered by the convenience -functions. - - -.. class:: Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, \ - stderr=None, preexec_fn=None, close_fds=True, shell=False, \ - cwd=None, env=None, universal_newlines=False, \ - startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=(), *, \ - encoding=None, errors=None) - - Execute a child program in a new process. On POSIX, the class uses - :meth:`os.execvp`-like behavior to execute the child program. On Windows, - the class uses the Windows ``CreateProcess()`` function. The arguments to - :class:`Popen` are as follows. - - *args* should be a sequence of program arguments or else a single string. - By default, the program to execute is the first item in *args* if *args* is - a sequence. If *args* is a string, the interpretation is - platform-dependent and described below. See the *shell* and *executable* - arguments for additional differences from the default behavior. Unless - otherwise stated, it is recommended to pass *args* as a sequence. - - On POSIX, if *args* is a string, the string is interpreted as the name or - path of the program to execute. However, this can only be done if not - passing arguments to the program. - - .. note:: - - :meth:`shlex.split` can be useful when determining the correct - tokenization for *args*, especially in complex cases:: - - >>> import shlex, subprocess - >>> command_line = input() - /bin/vikings -input eggs.txt -output "spam spam.txt" -cmd "echo '$MONEY'" - >>> args = shlex.split(command_line) - >>> print(args) - ['/bin/vikings', '-input', 'eggs.txt', '-output', 'spam spam.txt', '-cmd', "echo '$MONEY'"] - >>> p = subprocess.Popen(args) # Success! - - Note in particular that options (such as *-input*) and arguments (such - as *eggs.txt*) that are separated by whitespace in the shell go in separate - list elements, while arguments that need quoting or backslash escaping when - used in the shell (such as filenames containing spaces or the *echo* command - shown above) are single list elements. - - On Windows, if *args* is a sequence, it will be converted to a string in a - manner described in :ref:`converting-argument-sequence`. This is because - the underlying ``CreateProcess()`` operates on strings. - - The *shell* argument (which defaults to ``False``) specifies whether to use - the shell as the program to execute. If *shell* is ``True``, it is - recommended to pass *args* as a string rather than as a sequence. - - On POSIX with ``shell=True``, the shell defaults to :file:`/bin/sh`. If - *args* is a string, the string specifies the command - to execute through the shell. This means that the string must be - formatted exactly as it would be when typed at the shell prompt. This - includes, for example, quoting or backslash escaping filenames with spaces in - them. If *args* is a sequence, the first item specifies the command string, and - any additional items will be treated as additional arguments to the shell - itself. That is to say, :class:`Popen` does the equivalent of:: - - Popen(['/bin/sh', '-c', args[0], args[1], ...]) - - On Windows with ``shell=True``, the :envvar:`COMSPEC` environment variable - specifies the default shell. The only time you need to specify - ``shell=True`` on Windows is when the command you wish to execute is built - into the shell (e.g. :command:`dir` or :command:`copy`). You do not need - ``shell=True`` to run a batch file or console-based executable. - - .. note:: - - Read the `Security Considerations`_ section before using ``shell=True``. - - *bufsize* will be supplied as the corresponding argument to the - :func:`open` function when creating the stdin/stdout/stderr pipe - file objects: - - - :const:`0` means unbuffered (read and write are one - system call and can return short) - - :const:`1` means line buffered - (only usable if ``universal_newlines=True`` i.e., in a text mode) - - any other positive value means use a buffer of approximately that - size - - negative bufsize (the default) means the system default of - io.DEFAULT_BUFFER_SIZE will be used. - - .. versionchanged:: 3.3.1 - *bufsize* now defaults to -1 to enable buffering by default to match the - behavior that most code expects. In versions prior to Python 3.2.4 and - 3.3.1 it incorrectly defaulted to :const:`0` which was unbuffered - and allowed short reads. This was unintentional and did not match the - behavior of Python 2 as most code expected. - - The *executable* argument specifies a replacement program to execute. It - is very seldom needed. When ``shell=False``, *executable* replaces the - program to execute specified by *args*. However, the original *args* is - still passed to the program. Most programs treat the program specified - by *args* as the command name, which can then be different from the program - actually executed. On POSIX, the *args* name - becomes the display name for the executable in utilities such as - :program:`ps`. If ``shell=True``, on POSIX the *executable* argument - specifies a replacement shell for the default :file:`/bin/sh`. - - *stdin*, *stdout* and *stderr* specify the executed program's standard input, - standard output and standard error file handles, respectively. Valid values - are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive - integer), an existing :term:`file object`, and ``None``. :data:`PIPE` - indicates that a new pipe to the child should be created. :data:`DEVNULL` - indicates that the special file :data:`os.devnull` will be used. With the - default settings of ``None``, no redirection will occur; the child's file - handles will be inherited from the parent. Additionally, *stderr* can be - :data:`STDOUT`, which indicates that the stderr data from the applications - should be captured into the same file handle as for stdout. - - If *preexec_fn* is set to a callable object, this object will be called in the - child process just before the child is executed. - (POSIX only) - - .. warning:: - - The *preexec_fn* parameter is not safe to use in the presence of threads - in your application. The child process could deadlock before exec is - called. - If you must use it, keep it trivial! Minimize the number of libraries - you call into. - - .. note:: - - If you need to modify the environment for the child use the *env* - parameter rather than doing it in a *preexec_fn*. - The *start_new_session* parameter can take the place of a previously - common use of *preexec_fn* to call os.setsid() in the child. - - If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. (POSIX only). - The default varies by platform: Always true on POSIX. On Windows it is - true when *stdin*/*stdout*/*stderr* are :const:`None`, false otherwise. - On Windows, if *close_fds* is true then no handles will be inherited by the - child process. Note that on Windows, you cannot set *close_fds* to true and - also redirect the standard handles by setting *stdin*, *stdout* or *stderr*. - - .. versionchanged:: 3.2 - The default for *close_fds* was changed from :const:`False` to - what is described above. - - *pass_fds* is an optional sequence of file descriptors to keep open - between the parent and child. Providing any *pass_fds* forces - *close_fds* to be :const:`True`. (POSIX only) - - .. versionadded:: 3.2 - The *pass_fds* parameter was added. - - If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. *cwd* can be a :class:`str` and - :term:`path-like ` object. In particular, the function - looks for *executable* (or for the first item in *args*) relative to *cwd* - if the executable path is a relative path. - - .. versionchanged:: 3.6 - *cwd* parameter accepts a :term:`path-like object`. - - If *restore_signals* is true (the default) all signals that Python has set to - SIG_IGN are restored to SIG_DFL in the child process before the exec. - Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. - (POSIX only) - - .. versionchanged:: 3.2 - *restore_signals* was added. - - If *start_new_session* is true the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - - .. versionchanged:: 3.2 - *start_new_session* was added. - - If *env* is not ``None``, it must be a mapping that defines the environment - variables for the new process; these are used instead of the default - behavior of inheriting the current process' environment. - - .. note:: - - If specified, *env* must provide any variables required for the program to - execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. - - .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly - - If *encoding* or *errors* are specified, the file objects *stdin*, *stdout* - and *stderr* are opened in text mode with the specified encoding and - *errors*, as described above in :ref:`frequently-used-arguments`. If - *universal_newlines* is ``True``, they are opened in text mode with default - encoding. Otherwise, they are opened as binary streams. - - .. versionadded:: 3.6 - *encoding* and *errors* were added. - - If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is - passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be :data:`CREATE_NEW_CONSOLE` or - :data:`CREATE_NEW_PROCESS_GROUP`. (Windows only) - - Popen objects are supported as context managers via the :keyword:`with` statement: - on exit, standard file descriptors are closed, and the process is waited for. - :: - - with Popen(["ifconfig"], stdout=PIPE) as proc: - log.write(proc.stdout.read()) - - .. versionchanged:: 3.2 - Added context manager support. - - .. versionchanged:: 3.6 - Popen destructor now emits a :exc:`ResourceWarning` warning if the child - process is still running. - - -Exceptions -^^^^^^^^^^ - -Exceptions raised in the child process, before the new program has started to -execute, will be re-raised in the parent. Additionally, the exception object -will have one extra attribute called :attr:`child_traceback`, which is a string -containing traceback information from the child's point of view. - -The most common exception raised is :exc:`OSError`. This occurs, for example, -when trying to execute a non-existent file. Applications should prepare for -:exc:`OSError` exceptions. - -A :exc:`ValueError` will be raised if :class:`Popen` is called with invalid -arguments. - -:func:`check_call` and :func:`check_output` will raise -:exc:`CalledProcessError` if the called process returns a non-zero return -code. - -All of the functions and methods that accept a *timeout* parameter, such as -:func:`call` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if -the timeout expires before the process exits. - -Exceptions defined in this module all inherit from :exc:`SubprocessError`. - - .. versionadded:: 3.3 - The :exc:`SubprocessError` base class was added. - - -Security Considerations ------------------------ - -Unlike some other popen functions, this implementation will never -implicitly call a system shell. This means that all characters, -including shell metacharacters, can safely be passed to child processes. -If the shell is invoked explicitly, via ``shell=True``, it is the application's -responsibility to ensure that all whitespace and metacharacters are -quoted appropriately to avoid -`shell injection `_ -vulnerabilities. - -When using ``shell=True``, the :func:`shlex.quote` function can be -used to properly escape whitespace and shell metacharacters in strings -that are going to be used to construct shell commands. - - -Popen Objects -------------- - -Instances of the :class:`Popen` class have the following methods: - - -.. method:: Popen.poll() - - Check if child process has terminated. Set and return - :attr:`~Popen.returncode` attribute. Otherwise, returns ``None``. - - -.. method:: Popen.wait(timeout=None) - - Wait for child process to terminate. Set and return - :attr:`~Popen.returncode` attribute. - - If the process does not terminate after *timeout* seconds, raise a - :exc:`TimeoutExpired` exception. It is safe to catch this exception and - retry the wait. - - .. note:: - - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` - and the child process generates enough output to a pipe such that - it blocks waiting for the OS pipe buffer to accept more data. - Use :meth:`Popen.communicate` when using pipes to avoid that. - - .. note:: - - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. - - .. versionchanged:: 3.3 - *timeout* was added. - - .. deprecated:: 3.4 - - Do not use the *endtime* parameter. It is was unintentionally - exposed in 3.3 but was left undocumented as it was intended to be - private for internal use. Use *timeout* instead. - -.. method:: Popen.communicate(input=None, timeout=None) - - Interact with process: Send data to stdin. Read data from stdout and stderr, - until end-of-file is reached. Wait for process to terminate. The optional - *input* argument should be data to be sent to the child process, or - ``None``, if no data should be sent to the child. If streams were opened in - text mode, *input* must be a string. Otherwise, it must be bytes. - - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - The data will be strings if streams were opened in text mode; otherwise, - bytes. - - Note that if you want to send data to the process's stdin, you need to create - the Popen object with ``stdin=PIPE``. Similarly, to get anything other than - ``None`` in the result tuple, you need to give ``stdout=PIPE`` and/or - ``stderr=PIPE`` too. - - If the process does not terminate after *timeout* seconds, a - :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. - - The child process is not killed if the timeout expires, so in order to - cleanup properly a well-behaved application should kill the child process and - finish communication:: - - proc = subprocess.Popen(...) - try: - outs, errs = proc.communicate(timeout=15) - except TimeoutExpired: - proc.kill() - outs, errs = proc.communicate() - - .. note:: - - The data read is buffered in memory, so do not use this method if the data - size is large or unlimited. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. method:: Popen.send_signal(signal) - - Sends the signal *signal* to the child. - - .. note:: - - On Windows, SIGTERM is an alias for :meth:`terminate`. CTRL_C_EVENT and - CTRL_BREAK_EVENT can be sent to processes started with a *creationflags* - parameter which includes `CREATE_NEW_PROCESS_GROUP`. - - -.. method:: Popen.terminate() - - Stop the child. On Posix OSs the method sends SIGTERM to the - child. On Windows the Win32 API function :c:func:`TerminateProcess` is called - to stop the child. - - -.. method:: Popen.kill() - - Kills the child. On Posix OSs the function sends SIGKILL to the child. - On Windows :meth:`kill` is an alias for :meth:`terminate`. - - -The following attributes are also available: - -.. attribute:: Popen.args - - The *args* argument as it was passed to :class:`Popen` -- a - sequence of program arguments or else a single string. - - .. versionadded:: 3.3 - -.. attribute:: Popen.stdin - - If the *stdin* argument was :data:`PIPE`, this attribute is a writeable - stream object as returned by :func:`open`. If the *encoding* or *errors* - arguments were specified or the *universal_newlines* argument was ``True``, - the stream is a text stream, otherwise it is a byte stream. If the *stdin* - argument was not :data:`PIPE`, this attribute is ``None``. - - -.. attribute:: Popen.stdout - - If the *stdout* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - output from the child process. If the *encoding* or *errors* arguments were - specified or the *universal_newlines* argument was ``True``, the stream is a - text stream, otherwise it is a byte stream. If the *stdout* argument was not - :data:`PIPE`, this attribute is ``None``. - - -.. attribute:: Popen.stderr - - If the *stderr* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - error output from the child process. If the *encoding* or *errors* arguments - were specified or the *universal_newlines* argument was ``True``, the stream - is a text stream, otherwise it is a byte stream. If the *stderr* argument was - not :data:`PIPE`, this attribute is ``None``. - -.. warning:: - - Use :meth:`~Popen.communicate` rather than :attr:`.stdin.write `, - :attr:`.stdout.read ` or :attr:`.stderr.read ` to avoid - deadlocks due to any of the other OS pipe buffers filling up and blocking the - child process. - - -.. attribute:: Popen.pid - - The process ID of the child process. - - Note that if you set the *shell* argument to ``True``, this is the process ID - of the spawned shell. - - -.. attribute:: Popen.returncode - - The child return code, set by :meth:`poll` and :meth:`wait` (and indirectly - by :meth:`communicate`). A ``None`` value indicates that the process - hasn't terminated yet. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). - - -Windows Popen Helpers ---------------------- - -The :class:`STARTUPINFO` class and following constants are only available -on Windows. - -.. class:: STARTUPINFO() - - Partial support of the Windows - `STARTUPINFO `__ - structure is used for :class:`Popen` creation. - - .. attribute:: dwFlags - - A bit field that determines whether certain :class:`STARTUPINFO` - attributes are used when the process creates a window. :: - - si = subprocess.STARTUPINFO() - si.dwFlags = subprocess.STARTF_USESTDHANDLES | subprocess.STARTF_USESHOWWINDOW - - .. attribute:: hStdInput - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard input handle for the process. If - :data:`STARTF_USESTDHANDLES` is not specified, the default for standard - input is the keyboard buffer. - - .. attribute:: hStdOutput - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard output handle for the process. Otherwise, this attribute - is ignored and the default for standard output is the console window's - buffer. - - .. attribute:: hStdError - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard error handle for the process. Otherwise, this attribute is - ignored and the default for standard error is the console window's buffer. - - .. attribute:: wShowWindow - - If :attr:`dwFlags` specifies :data:`STARTF_USESHOWWINDOW`, this attribute - can be any of the values that can be specified in the ``nCmdShow`` - parameter for the - `ShowWindow `__ - function, except for ``SW_SHOWDEFAULT``. Otherwise, this attribute is - ignored. - - :data:`SW_HIDE` is provided for this attribute. It is used when - :class:`Popen` is called with ``shell=True``. - - -Constants -^^^^^^^^^ - -The :mod:`subprocess` module exposes the following constants. - -.. data:: STD_INPUT_HANDLE - - The standard input device. Initially, this is the console input buffer, - ``CONIN$``. - -.. data:: STD_OUTPUT_HANDLE - - The standard output device. Initially, this is the active console screen - buffer, ``CONOUT$``. - -.. data:: STD_ERROR_HANDLE - - The standard error device. Initially, this is the active console screen - buffer, ``CONOUT$``. - -.. data:: SW_HIDE - - Hides the window. Another window will be activated. - -.. data:: STARTF_USESTDHANDLES - - Specifies that the :attr:`STARTUPINFO.hStdInput`, - :attr:`STARTUPINFO.hStdOutput`, and :attr:`STARTUPINFO.hStdError` attributes - contain additional information. - -.. data:: STARTF_USESHOWWINDOW - - Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains - additional information. - -.. data:: CREATE_NEW_CONSOLE - - The new process has a new console, instead of inheriting its parent's - console (the default). - -.. data:: CREATE_NEW_PROCESS_GROUP - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - group will be created. This flag is necessary for using :func:`os.kill` - on the subprocess. - - This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified. - -.. _call-function-trio: - -Older high-level API --------------------- - -Prior to Python 3.5, these three functions comprised the high level API to -subprocess. You can now use :func:`run` in many cases, but lots of existing code -calls these functions. - -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) - - Run the command described by *args*. Wait for command to complete, then - return the :attr:`~Popen.returncode` attribute. - - This is equivalent to:: - - run(...).returncode - - (except that the *input* and *check* parameters are not supported) - - The arguments shown above are merely the most - common ones. The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, cwd=None, timeout=None) - - Run command with arguments. Wait for command to complete. If the return - code was zero then return, otherwise raise :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute. - - This is equivalent to:: - - run(..., check=True) - - (except that the *input* parameter is not supported) - - The arguments shown above are merely the most - common ones. The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ - cwd=None, encoding=None, errors=None, \ - universal_newlines=False, timeout=None) - - Run command with arguments and return its output. - - If the return code was non-zero it raises a :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute and any output in the - :attr:`~CalledProcessError.output` attribute. - - This is equivalent to:: - - run(..., check=True, stdout=PIPE).stdout - - The arguments shown above are merely the most common ones. - The full function signature is largely the same as that of :func:`run` - - most arguments are passed directly through to that interface. - However, explicitly passing ``input=None`` to inherit the parent's - standard input file handle is not supported. - - By default, this function will return the data as encoded bytes. The actual - encoding of the output data may depend on the command being invoked, so the - decoding to text will often need to be handled at the application level. - - This behaviour may be overridden by setting *universal_newlines* to - ``True`` as described above in :ref:`frequently-used-arguments`. - - To also capture standard error in the result, use - ``stderr=subprocess.STDOUT``:: - - >>> subprocess.check_output( - ... "ls non_existent_file; exit 0", - ... stderr=subprocess.STDOUT, - ... shell=True) - 'ls: non_existent_file: No such file or directory\n' - - .. versionadded:: 3.1 - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.4 - Support for the *input* keyword argument was added. - - .. versionchanged:: 3.6 - *encoding* and *errors* were added. See :func:`run` for details. - -.. _subprocess-replacements: - -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ - -In this section, "a becomes b" means that b can be used as a replacement for a. - -.. note:: - - All "a" functions in this section fail (more or less) silently if the - executed program cannot be found; the "b" replacements raise :exc:`OSError` - instead. - - In addition, the replacements using :func:`check_output` will fail with a - :exc:`CalledProcessError` if the requested operation produces a non-zero - return code. The output is still available as the - :attr:`~CalledProcessError.output` attribute of the raised exception. - -In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. - - -Replacing /bin/sh shell backquote -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - output=`mycmd myarg` - -becomes:: - - output = check_output(["mycmd", "myarg"]) - -Replacing shell pipeline -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - output=`dmesg | grep hda` - -becomes:: - - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. - output = p2.communicate()[0] - -The p1.stdout.close() call after starting the p2 is important in order for p1 -to receive a SIGPIPE if p2 exits before p1. - -Alternatively, for trusted input, the shell's own pipeline support may still -be used directly: - -.. code-block:: bash - - output=`dmesg | grep hda` - -becomes:: - - output=check_output("dmesg | grep hda", shell=True) - - -Replacing :func:`os.system` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - sts = os.system("mycmd" + " myarg") - # becomes - sts = call("mycmd" + " myarg", shell=True) - -Notes: - -* Calling the program through the shell is usually not required. - -A more realistic example would look like this:: - - try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print("Child was terminated by signal", -retcode, file=sys.stderr) - else: - print("Child returned", retcode, file=sys.stderr) - except OSError as e: - print("Execution failed:", e, file=sys.stderr) - - -Replacing the :func:`os.spawn ` family -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -P_NOWAIT example:: - - pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") - ==> - pid = Popen(["/bin/mycmd", "myarg"]).pid - -P_WAIT example:: - - retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") - ==> - retcode = call(["/bin/mycmd", "myarg"]) - -Vector example:: - - os.spawnvp(os.P_NOWAIT, path, args) - ==> - Popen([path] + args[1:]) - -Environment example:: - - os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) - ==> - Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - - -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -Return code handling translates as follows:: - - pipe = os.popen(cmd, 'w') - ... - rc = pipe.close() - if rc is not None and rc >> 8: - print("There were some errors") - ==> - process = Popen(cmd, stdin=PIPE) - ... - process.stdin.close() - if process.wait() != 0: - print("There were some errors") - - -Replacing functions from the :mod:`popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* the *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - -Legacy Shell Invocation Functions ---------------------------------- - -This module also provides the following legacy functions from the 2.x -``commands`` module. These operations implicitly invoke the system shell and -none of the guarantees described above regarding security and exception -handling consistency are valid for these functions. - -.. function:: getstatusoutput(cmd) - - Return ``(exitcode, output)`` of executing *cmd* in a shell. - - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and - return a 2-tuple ``(exitcode, output)``. The locale encoding is used; - see the notes on :ref:`frequently-used-arguments` for more details. - - A trailing newline is stripped from the output. - The exit code for the command can be interpreted as the return code - of subprocess. Example:: - - >>> subprocess.getstatusoutput('ls /bin/ls') - (0, '/bin/ls') - >>> subprocess.getstatusoutput('cat /bin/junk') - (1, 'cat: /bin/junk: No such file or directory') - >>> subprocess.getstatusoutput('/bin/junk') - (127, 'sh: /bin/junk: not found') - >>> subprocess.getstatusoutput('/bin/kill $$') - (-15, '') - - Availability: POSIX & Windows - - .. versionchanged:: 3.3.4 - Windows support was added. - - The function now returns (exitcode, output) instead of (status, output) - as it did in Python 3.3.3 and earlier. exitcode has the same value as - :attr:`~Popen.returncode`. - - -.. function:: getoutput(cmd) - - Return output (stdout and stderr) of executing *cmd* in a shell. - - Like :func:`getstatusoutput`, except the exit code is ignored and the return - value is a string containing the command's output. Example:: - - >>> subprocess.getoutput('ls /bin/ls') - '/bin/ls' - - Availability: POSIX & Windows - - .. versionchanged:: 3.3.4 - Windows support added - - -Notes ------ - -.. _converting-argument-sequence: - -Converting an argument sequence to a string on Windows -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On Windows, an *args* sequence is converted to a string that can be parsed -using the following rules (which correspond to the rules used by the MS C -runtime): - -1. Arguments are delimited by white space, which is either a - space or a tab. - -2. A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - -3. A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - -4. Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - -5. If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - - -.. seealso:: - - :mod:`shlex` - Module which provides function to parse and escape command lines. diff --git a/third_party/python/Doc/library/sunau.rst b/third_party/python/Doc/library/sunau.rst deleted file mode 100644 index c8357e4fc..000000000 --- a/third_party/python/Doc/library/sunau.rst +++ /dev/null @@ -1,274 +0,0 @@ -:mod:`sunau` --- Read and write Sun AU files -============================================ - -.. module:: sunau - :synopsis: Provide an interface to the Sun AU sound format. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/sunau.py` - --------------- - -The :mod:`sunau` module provides a convenient interface to the Sun AU sound -format. Note that this module is interface-compatible with the modules -:mod:`aifc` and :mod:`wave`. - -An audio file consists of a header followed by the data. The fields of the -header are: - -+---------------+-----------------------------------------------+ -| Field | Contents | -+===============+===============================================+ -| magic word | The four bytes ``.snd``. | -+---------------+-----------------------------------------------+ -| header size | Size of the header, including info, in bytes. | -+---------------+-----------------------------------------------+ -| data size | Physical size of the data, in bytes. | -+---------------+-----------------------------------------------+ -| encoding | Indicates how the audio samples are encoded. | -+---------------+-----------------------------------------------+ -| sample rate | The sampling rate. | -+---------------+-----------------------------------------------+ -| # of channels | The number of channels in the samples. | -+---------------+-----------------------------------------------+ -| info | ASCII string giving a description of the | -| | audio file (padded with null bytes). | -+---------------+-----------------------------------------------+ - -Apart from the info field, all header fields are 4 bytes in size. They are all -32-bit unsigned integers encoded in big-endian byte order. - -The :mod:`sunau` module defines the following functions: - - -.. function:: open(file, mode) - - If *file* is a string, open the file by that name, otherwise treat it as a - seekable file-like object. *mode* can be any of - - ``'r'`` - Read only mode. - - ``'w'`` - Write only mode. - - Note that it does not allow read/write files. - - A *mode* of ``'r'`` returns an :class:`AU_read` object, while a *mode* of ``'w'`` - or ``'wb'`` returns an :class:`AU_write` object. - - -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - -The :mod:`sunau` module defines the following exception: - -.. exception:: Error - - An error raised when something is impossible because of Sun AU specs or - implementation deficiency. - - -The :mod:`sunau` module defines the following data items: - -.. data:: AUDIO_FILE_MAGIC - - An integer every valid Sun AU file begins with, stored in big-endian form. This - is the string ``.snd`` interpreted as an integer. - - -.. data:: AUDIO_FILE_ENCODING_MULAW_8 - AUDIO_FILE_ENCODING_LINEAR_8 - AUDIO_FILE_ENCODING_LINEAR_16 - AUDIO_FILE_ENCODING_LINEAR_24 - AUDIO_FILE_ENCODING_LINEAR_32 - AUDIO_FILE_ENCODING_ALAW_8 - - Values of the encoding field from the AU header which are supported by this - module. - - -.. data:: AUDIO_FILE_ENCODING_FLOAT - AUDIO_FILE_ENCODING_DOUBLE - AUDIO_FILE_ENCODING_ADPCM_G721 - AUDIO_FILE_ENCODING_ADPCM_G722 - AUDIO_FILE_ENCODING_ADPCM_G723_3 - AUDIO_FILE_ENCODING_ADPCM_G723_5 - - Additional known values of the encoding field from the AU header, but which are - not supported by this module. - - -.. _au-read-objects: - -AU_read Objects ---------------- - -AU_read objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_read.close() - - Close the stream, and make the instance unusable. (This is called automatically - on deletion.) - - -.. method:: AU_read.getnchannels() - - Returns number of audio channels (1 for mono, 2 for stereo). - - -.. method:: AU_read.getsampwidth() - - Returns sample width in bytes. - - -.. method:: AU_read.getframerate() - - Returns sampling frequency. - - -.. method:: AU_read.getnframes() - - Returns number of audio frames. - - -.. method:: AU_read.getcomptype() - - Returns compression type. Supported compression types are ``'ULAW'``, ``'ALAW'`` - and ``'NONE'``. - - -.. method:: AU_read.getcompname() - - Human-readable version of :meth:`getcomptype`. The supported types have the - respective names ``'CCITT G.711 u-law'``, ``'CCITT G.711 A-law'`` and ``'not - compressed'``. - - -.. method:: AU_read.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: AU_read.readframes(n) - - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. The data - will be returned in linear format. If the original data is in u-LAW format, it - will be converted. - - -.. method:: AU_read.rewind() - - Rewind the file pointer to the beginning of the audio stream. - -The following two methods define a term "position" which is compatible between -them, and is otherwise implementation dependent. - - -.. method:: AU_read.setpos(pos) - - Set the file pointer to the specified position. Only values returned from - :meth:`tell` should be used for *pos*. - - -.. method:: AU_read.tell() - - Return current file pointer position. Note that the returned value has nothing - to do with the actual position in the file. - -The following two functions are defined for compatibility with the :mod:`aifc`, -and don't do anything interesting. - - -.. method:: AU_read.getmarkers() - - Returns ``None``. - - -.. method:: AU_read.getmark(id) - - Raise an error. - - -.. _au-write-objects: - -AU_write Objects ----------------- - -AU_write objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_write.setnchannels(n) - - Set the number of channels. - - -.. method:: AU_write.setsampwidth(n) - - Set the sample width (in bytes.) - - .. versionchanged:: 3.4 - Added support for 24-bit samples. - - -.. method:: AU_write.setframerate(n) - - Set the frame rate. - - -.. method:: AU_write.setnframes(n) - - Set the number of frames. This can be later changed, when and if more frames - are written. - - -.. method:: AU_write.setcomptype(type, name) - - Set the compression type and description. Only ``'NONE'`` and ``'ULAW'`` are - supported on output. - - -.. method:: AU_write.setparams(tuple) - - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the :meth:`set\*` methods. Set all - parameters. - - -.. method:: AU_write.tell() - - Return current position in the file, with the same disclaimer for the - :meth:`AU_read.tell` and :meth:`AU_read.setpos` methods. - - -.. method:: AU_write.writeframesraw(data) - - Write audio frames, without correcting *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.writeframes(data) - - Write audio frames and make sure *nframes* is correct. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.close() - - Make sure *nframes* is correct, and close the file. - - This method is called upon deletion. - -Note that it is invalid to set any parameters after calling :meth:`writeframes` -or :meth:`writeframesraw`. - diff --git a/third_party/python/Doc/library/superseded.rst b/third_party/python/Doc/library/superseded.rst deleted file mode 100644 index 50a598323..000000000 --- a/third_party/python/Doc/library/superseded.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _superseded: - -****************** -Superseded Modules -****************** - -The modules described in this chapter are deprecated and only kept for -backwards compatibility. They have been superseded by other modules. - - -.. toctree:: - - optparse.rst - imp.rst diff --git a/third_party/python/Doc/library/symbol.rst b/third_party/python/Doc/library/symbol.rst deleted file mode 100644 index 44996936e..000000000 --- a/third_party/python/Doc/library/symbol.rst +++ /dev/null @@ -1,27 +0,0 @@ -:mod:`symbol` --- Constants used with Python parse trees -======================================================== - -.. module:: symbol - :synopsis: Constants representing internal nodes of the parse tree. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/symbol.py` - --------------- - -This module provides constants which represent the numeric values of internal -nodes of the parse tree. Unlike most Python constants, these use lower-case -names. Refer to the file :file:`Grammar/Grammar` in the Python distribution for -the definitions of the names in the context of the language grammar. The -specific numeric values which the names map to may change between Python -versions. - -This module also provides one additional data object: - - -.. data:: sym_name - - Dictionary mapping the numeric values of the constants defined in this module - back to name strings, allowing more human-readable representation of parse trees - to be generated. diff --git a/third_party/python/Doc/library/symtable.rst b/third_party/python/Doc/library/symtable.rst deleted file mode 100644 index ba2caff58..000000000 --- a/third_party/python/Doc/library/symtable.rst +++ /dev/null @@ -1,188 +0,0 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== - -.. module:: symtable - :synopsis: Interface to the compiler's internal symbol tables. - -**Source code:** :source:`Lib/symtable.py` - --------------- - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Benjamin Peterson - - -Symbol tables are generated by the compiler from AST just before bytecode is -generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these -tables. - - -Generating Symbol Tables ------------------------- - -.. function:: symtable(code, filename, compile_type) - - Return the toplevel :class:`SymbolTable` for the Python source *code*. - *filename* is the name of the file containing the code. *compile_type* is - like the *mode* argument to :func:`compile`. - - -Examining Symbol Tables ------------------------ - -.. class:: SymbolTable - - A namespace table for a block. The constructor is not public. - - .. method:: get_type() - - Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, and ``'function'``. - - .. method:: get_id() - - Return the table's identifier. - - .. method:: get_name() - - Return the table's name. This is the name of the class if the table is - for a class, the name of the function if the table is for a function, or - ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``). - - .. method:: get_lineno() - - Return the number of the first line in the block this table represents. - - .. method:: is_optimized() - - Return ``True`` if the locals in this table can be optimized. - - .. method:: is_nested() - - Return ``True`` if the block is a nested class or function. - - .. method:: has_children() - - Return ``True`` if the block has nested namespaces within it. These can - be obtained with :meth:`get_children`. - - .. method:: has_exec() - - Return ``True`` if the block uses ``exec``. - - .. method:: get_identifiers() - - Return a list of names of symbols in this table. - - .. method:: lookup(name) - - Lookup *name* in the table and return a :class:`Symbol` instance. - - .. method:: get_symbols() - - Return a list of :class:`Symbol` instances for names in the table. - - .. method:: get_children() - - Return a list of the nested symbol tables. - - -.. class:: Function - - A namespace for a function or method. This class inherits - :class:`SymbolTable`. - - .. method:: get_parameters() - - Return a tuple containing names of parameters to this function. - - .. method:: get_locals() - - Return a tuple containing names of locals in this function. - - .. method:: get_globals() - - Return a tuple containing names of globals in this function. - - .. method:: get_frees() - - Return a tuple containing names of free variables in this function. - - -.. class:: Class - - A namespace of a class. This class inherits :class:`SymbolTable`. - - .. method:: get_methods() - - Return a tuple containing the names of methods declared in the class. - - -.. class:: Symbol - - An entry in a :class:`SymbolTable` corresponding to an identifier in the - source. The constructor is not public. - - .. method:: get_name() - - Return the symbol's name. - - .. method:: is_referenced() - - Return ``True`` if the symbol is used in its block. - - .. method:: is_imported() - - Return ``True`` if the symbol is created from an import statement. - - .. method:: is_parameter() - - Return ``True`` if the symbol is a parameter. - - .. method:: is_global() - - Return ``True`` if the symbol is global. - - .. method:: is_declared_global() - - Return ``True`` if the symbol is declared global with a global statement. - - .. method:: is_local() - - Return ``True`` if the symbol is local to its block. - - .. method:: is_free() - - Return ``True`` if the symbol is referenced in its block, but not assigned - to. - - .. method:: is_assigned() - - Return ``True`` if the symbol is assigned to in its block. - - .. method:: is_namespace() - - Return ``True`` if name binding introduces new namespace. - - If the name is used as the target of a function or class statement, this - will be true. - - For example:: - - >>> table = symtable.symtable("def some_func(): pass", "string", "exec") - >>> table.lookup("some_func").is_namespace() - True - - Note that a single name can be bound to multiple objects. If the result - is ``True``, the name may also be bound to other objects, like an int or - list, that does not introduce a new namespace. - - .. method:: get_namespaces() - - Return a list of namespaces bound to this name. - - .. method:: get_namespace() - - Return the namespace bound to this name. If more than one namespace is - bound, :exc:`ValueError` is raised. diff --git a/third_party/python/Doc/library/sys.rst b/third_party/python/Doc/library/sys.rst deleted file mode 100644 index 91984667b..000000000 --- a/third_party/python/Doc/library/sys.rst +++ /dev/null @@ -1,1381 +0,0 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= - -.. module:: sys - :synopsis: Access system-specific parameters and functions. - --------------- - -This module provides access to some variables used or maintained by the -interpreter and to functions that interact strongly with the interpreter. It is -always available. - - -.. data:: abiflags - - On POSIX systems where Python was built with the standard ``configure`` - script, this contains the ABI flags as specified by :pep:`3149`. - - .. versionadded:: 3.2 - - -.. data:: argv - - The list of command line arguments passed to a Python script. ``argv[0]`` is the - script name (it is operating system dependent whether this is a full pathname or - not). If the command was executed using the :option:`-c` command line option to - the interpreter, ``argv[0]`` is set to the string ``'-c'``. If no script name - was passed to the Python interpreter, ``argv[0]`` is the empty string. - - To loop over the standard input, or the list of files given on the - command line, see the :mod:`fileinput` module. - - -.. data:: base_exec_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - -.. data:: base_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - -.. data:: byteorder - - An indicator of the native byte order. This will have the value ``'big'`` on - big-endian (most-significant byte first) platforms, and ``'little'`` on - little-endian (least-significant byte first) platforms. - - -.. data:: builtin_module_names - - A tuple of strings giving the names of all modules that are compiled into this - Python interpreter. (This information is not available in any other way --- - ``modules.keys()`` only lists the imported modules.) - - -.. function:: call_tracing(func, args) - - Call ``func(*args)``, while tracing is enabled. The tracing state is saved, - and restored afterwards. This is intended to be called from a debugger from - a checkpoint, to recursively debug some other code. - - -.. data:: copyright - - A string containing the copyright pertaining to the Python interpreter. - - -.. function:: _clear_type_cache() - - Clear the internal type cache. The type cache is used to speed up attribute - and method lookups. Use the function *only* to drop unnecessary references - during reference leak debugging. - - This function should be used for internal and specialized purposes only. - - -.. function:: _current_frames() - - Return a dictionary mapping each thread's identifier to the topmost stack frame - currently active in that thread at the time the function is called. Note that - functions in the :mod:`traceback` module can build the call stack given such a - frame. - - This is most useful for debugging deadlock: this function does not require the - deadlocked threads' cooperation, and such threads' call stacks are frozen for as - long as they remain deadlocked. The frame returned for a non-deadlocked thread - may bear no relationship to that thread's current activity by the time calling - code examines the frame. - - This function should be used for internal and specialized purposes only. - - -.. function:: _debugmallocstats() - - Print low-level information to stderr about the state of CPython's memory - allocator. - - If Python is configured --with-pydebug, it also performs some expensive - internal consistency checks. - - .. versionadded:: 3.3 - - .. impl-detail:: - - This function is specific to CPython. The exact output format is not - defined here, and may change. - - -.. data:: dllhandle - - Integer specifying the handle of the Python DLL. Availability: Windows. - - -.. function:: displayhook(value) - - If *value* is not ``None``, this function prints ``repr(value)`` to - ``sys.stdout``, and saves *value* in ``builtins._``. If ``repr(value)`` is - not encodable to ``sys.stdout.encoding`` with ``sys.stdout.errors`` error - handler (which is probably ``'strict'``), encode it to - ``sys.stdout.encoding`` with ``'backslashreplace'`` error handler. - - ``sys.displayhook`` is called on the result of evaluating an :term:`expression` - entered in an interactive Python session. The display of these values can be - customized by assigning another one-argument function to ``sys.displayhook``. - - Pseudo-code:: - - def displayhook(value): - if value is None: - return - # Set '_' to None to avoid recursion - builtins._ = None - text = repr(value) - try: - sys.stdout.write(text) - except UnicodeEncodeError: - bytes = text.encode(sys.stdout.encoding, 'backslashreplace') - if hasattr(sys.stdout, 'buffer'): - sys.stdout.buffer.write(bytes) - else: - text = bytes.decode(sys.stdout.encoding, 'strict') - sys.stdout.write(text) - sys.stdout.write("\n") - builtins._ = value - - .. versionchanged:: 3.2 - Use ``'backslashreplace'`` error handler on :exc:`UnicodeEncodeError`. - - -.. data:: dont_write_bytecode - - If this is true, Python won't try to write ``.pyc`` files on the - import of source modules. This value is initially set to ``True`` or - ``False`` depending on the :option:`-B` command line option and the - :envvar:`PYTHONDONTWRITEBYTECODE` environment variable, but you can set it - yourself to control bytecode file generation. - - -.. function:: excepthook(type, value, traceback) - - This function prints out a given traceback and exception to ``sys.stderr``. - - When an exception is raised and uncaught, the interpreter calls - ``sys.excepthook`` with three arguments, the exception class, exception - instance, and a traceback object. In an interactive session this happens just - before control is returned to the prompt; in a Python program this happens just - before the program exits. The handling of such top-level exceptions can be - customized by assigning another three-argument function to ``sys.excepthook``. - - -.. data:: __displayhook__ - __excepthook__ - - These objects contain the original values of ``displayhook`` and ``excepthook`` - at the start of the program. They are saved so that ``displayhook`` and - ``excepthook`` can be restored in case they happen to get replaced with broken - objects. - - -.. function:: exc_info() - - This function returns a tuple of three values that give information about the - exception that is currently being handled. The information returned is specific - both to the current thread and to the current stack frame. If the current stack - frame is not handling an exception, the information is taken from the calling - stack frame, or its caller, and so on until a stack frame is found that is - handling an exception. Here, "handling an exception" is defined as "executing - an except clause." For any stack frame, only information about the exception - being currently handled is accessible. - - .. index:: object: traceback - - If no exception is being handled anywhere on the stack, a tuple containing - three ``None`` values is returned. Otherwise, the values returned are - ``(type, value, traceback)``. Their meaning is: *type* gets the type of the - exception being handled (a subclass of :exc:`BaseException`); *value* gets - the exception instance (an instance of the exception type); *traceback* gets - a traceback object (see the Reference Manual) which encapsulates the call - stack at the point where the exception originally occurred. - - -.. data:: exec_prefix - - A string giving the site-specific directory prefix where the platform-dependent - Python files are installed; by default, this is also ``'/usr/local'``. This can - be set at build time with the ``--exec-prefix`` argument to the - :program:`configure` script. Specifically, all configuration files (e.g. the - :file:`pyconfig.h` header file) are installed in the directory - :file:`{exec_prefix}/lib/python{X.Y}/config`, and shared library modules are - installed in :file:`{exec_prefix}/lib/python{X.Y}/lib-dynload`, where *X.Y* - is the version number of Python, for example ``3.2``. - - .. note:: - - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. - - -.. data:: executable - - A string giving the absolute path of the executable binary for the Python - interpreter, on systems where this makes sense. If Python is unable to retrieve - the real path to its executable, :data:`sys.executable` will be an empty string - or ``None``. - - -.. function:: exit([arg]) - - Exit from Python. This is implemented by raising the :exc:`SystemExit` - exception, so cleanup actions specified by finally clauses of :keyword:`try` - statements are honored, and it is possible to intercept the exit attempt at - an outer level. - - The optional argument *arg* can be an integer giving the exit status - (defaulting to zero), or another type of object. If it is an integer, zero - is considered "successful termination" and any nonzero value is considered - "abnormal termination" by shells and the like. Most systems require it to be - in the range 0--127, and produce undefined results otherwise. Some systems - have a convention for assigning specific meanings to specific exit codes, but - these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of - object is passed, ``None`` is equivalent to passing zero, and any other - object is printed to :data:`stderr` and results in an exit code of 1. In - particular, ``sys.exit("some error message")`` is a quick way to exit a - program when an error occurs. - - Since :func:`exit` ultimately "only" raises an exception, it will only exit - the process when called from the main thread, and the exception is not - intercepted. - - .. versionchanged:: 3.6 - If an error occurs in the cleanup after the Python interpreter - has caught :exc:`SystemExit` (such as an error flushing buffered data - in the standard streams), the exit status is changed to 120. - - -.. data:: flags - - The :term:`struct sequence` *flags* exposes the status of command line - flags. The attributes are read only. - - ============================= ============================= - attribute flag - ============================= ============================= - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - ============================= ============================= - - .. versionchanged:: 3.2 - Added ``quiet`` attribute for the new :option:`-q` flag. - - .. versionadded:: 3.2.3 - The ``hash_randomization`` attribute. - - .. versionchanged:: 3.3 - Removed obsolete ``division_warning`` attribute. - - .. versionchanged:: 3.4 - Added ``isolated`` attribute for :option:`-I` ``isolated`` flag. - -.. data:: float_info - - A :term:`struct sequence` holding information about the float type. It - contains low level information about the precision and internal - representation. The values correspond to the various floating-point - constants defined in the standard header file :file:`float.h` for the 'C' - programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard - [C99]_, 'Characteristics of floating types', for details. - - .. tabularcolumns:: |l|l|L| - - +---------------------+----------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+================+==================================================+ - | :const:`epsilon` | DBL_EPSILON | difference between 1 and the least value greater | - | | | than 1 that is representable as a float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`dig` | DBL_DIG | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+----------------+--------------------------------------------------+ - | :const:`mant_dig` | DBL_MANT_DIG | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max` | DBL_MAX | maximum representable finite float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max_exp` | DBL_MAX_EXP | maximum integer e such that ``radix**(e-1)`` is | - | | | a representable finite float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max_10_exp` | DBL_MAX_10_EXP | maximum integer e such that ``10**e`` is in the | - | | | range of representable finite floats | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min` | DBL_MIN | minimum positive normalized float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min_exp` | DBL_MIN_EXP | minimum integer e such that ``radix**(e-1)`` is | - | | | a normalized float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min_10_exp` | DBL_MIN_10_EXP | minimum integer e such that ``10**e`` is a | - | | | normalized float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`radix` | FLT_RADIX | radix of exponent representation | - +---------------------+----------------+--------------------------------------------------+ - | :const:`rounds` | FLT_ROUNDS | integer constant representing the rounding mode | - | | | used for arithmetic operations. This reflects | - | | | the value of the system FLT_ROUNDS macro at | - | | | interpreter startup time. See section 5.2.4.2.2 | - | | | of the C99 standard for an explanation of the | - | | | possible values and their meanings. | - +---------------------+----------------+--------------------------------------------------+ - - The attribute :attr:`sys.float_info.dig` needs further explanation. If - ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a - float and back again will recover a string representing the same decimal - value:: - - >>> import sys - >>> sys.float_info.dig - 15 - >>> s = '3.14159265358979' # decimal string with 15 significant digits - >>> format(float(s), '.15g') # convert to float and back -> same value - '3.14159265358979' - - But for strings with more than :attr:`sys.float_info.dig` significant digits, - this isn't always true:: - - >>> s = '9876543211234567' # 16 significant digits is too many! - >>> format(float(s), '.16g') # conversion changes value - '9876543211234568' - -.. data:: float_repr_style - - A string indicating how the :func:`repr` function behaves for - floats. If the string has value ``'short'`` then for a finite - float ``x``, ``repr(x)`` aims to produce a short string with the - property that ``float(repr(x)) == x``. This is the usual behaviour - in Python 3.1 and later. Otherwise, ``float_repr_style`` has value - ``'legacy'`` and ``repr(x)`` behaves in the same way as it did in - versions of Python prior to 3.1. - - .. versionadded:: 3.1 - - -.. function:: getallocatedblocks() - - Return the number of memory blocks currently allocated by the interpreter, - regardless of their size. This function is mainly useful for tracking - and debugging memory leaks. Because of the interpreter's internal - caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more - predictable results. - - If a Python build or implementation cannot reasonably compute this - information, :func:`getallocatedblocks()` is allowed to return 0 instead. - - .. versionadded:: 3.4 - - -.. function:: getcheckinterval() - - Return the interpreter's "check interval"; see :func:`setcheckinterval`. - - .. deprecated:: 3.2 - Use :func:`getswitchinterval` instead. - - -.. function:: getdefaultencoding() - - Return the name of the current default string encoding used by the Unicode - implementation. - - -.. function:: getdlopenflags() - - Return the current value of the flags that are used for - :c:func:`dlopen` calls. Symbolic names for the flag values can be - found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. - :data:`os.RTLD_LAZY`). Availability: Unix. - - -.. function:: getfilesystemencoding() - - Return the name of the encoding used to convert between Unicode - filenames and bytes filenames. For best compatibility, str should be - used for filenames in all cases, although representing filenames as bytes - is also supported. Functions accepting or returning filenames should support - either str or bytes and internally convert to the system's preferred - representation. - - This encoding is always ASCII-compatible. - - :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that - the correct encoding and errors mode are used. - - * On Mac OS X, the encoding is ``'utf-8'``. - - * On Unix, the encoding is the locale encoding. - - * On Windows, the encoding may be ``'utf-8'`` or ``'mbcs'``, depending - on user configuration. - - .. versionchanged:: 3.2 - :func:`getfilesystemencoding` result cannot be ``None`` anymore. - - .. versionchanged:: 3.6 - Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. - -.. function:: getfilesystemencodeerrors() - - Return the name of the error mode used to convert between Unicode filenames - and bytes filenames. The encoding name is returned from - :func:`getfilesystemencoding`. - - :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that - the correct encoding and errors mode are used. - - .. versionadded:: 3.6 - -.. function:: getrefcount(object) - - Return the reference count of the *object*. The count returned is generally one - higher than you might expect, because it includes the (temporary) reference as - an argument to :func:`getrefcount`. - - -.. function:: getrecursionlimit() - - Return the current value of the recursion limit, the maximum depth of the Python - interpreter stack. This limit prevents infinite recursion from causing an - overflow of the C stack and crashing Python. It can be set by - :func:`setrecursionlimit`. - - -.. function:: getsizeof(object[, default]) - - Return the size of an object in bytes. The object can be any type of - object. All built-in objects will return correct results, but this - does not have to hold true for third-party extensions as it is implementation - specific. - - Only the memory consumption directly attributed to the object is - accounted for, not the memory consumption of objects it refers to. - - If given, *default* will be returned if the object does not provide means to - retrieve the size. Otherwise a :exc:`TypeError` will be raised. - - :func:`getsizeof` calls the object's ``__sizeof__`` method and adds an - additional garbage collector overhead if the object is managed by the garbage - collector. - - See `recursive sizeof recipe `_ - for an example of using :func:`getsizeof` recursively to find the size of - containers and all their contents. - -.. function:: getswitchinterval() - - Return the interpreter's "thread switch interval"; see - :func:`setswitchinterval`. - - .. versionadded:: 3.2 - - -.. function:: _getframe([depth]) - - Return a frame object from the call stack. If optional integer *depth* is - given, return the frame object that many calls below the top of the stack. If - that is deeper than the call stack, :exc:`ValueError` is raised. The default - for *depth* is zero, returning the frame at the top of the call stack. - - .. impl-detail:: - - This function should be used for internal and specialized purposes only. - It is not guaranteed to exist in all implementations of Python. - - -.. function:: getprofile() - - .. index:: - single: profile function - single: profiler - - Get the profiler function as set by :func:`setprofile`. - - -.. function:: gettrace() - - .. index:: - single: trace function - single: debugger - - Get the trace function as set by :func:`settrace`. - - .. impl-detail:: - - The :func:`gettrace` function is intended only for implementing debuggers, - profilers, coverage tools and the like. Its behavior is part of the - implementation platform, rather than part of the language definition, and - thus may not be available in all Python implementations. - - -.. function:: getwindowsversion() - - Return a named tuple describing the Windows version - currently running. The named elements are *major*, *minor*, - *build*, *platform*, *service_pack*, *service_pack_minor*, - *service_pack_major*, *suite_mask*, *product_type* and - *platform_version*. *service_pack* contains a string, - *platform_version* a 3-tuple and all other values are - integers. The components can also be accessed by name, so - ``sys.getwindowsversion()[0]`` is equivalent to - ``sys.getwindowsversion().major``. For compatibility with prior - versions, only the first 5 elements are retrievable by indexing. - - *platform* will be :const:`2 (VER_PLATFORM_WIN32_NT)`. - - *product_type* may be one of the following values: - - +---------------------------------------+---------------------------------+ - | Constant | Meaning | - +=======================================+=================================+ - | :const:`1 (VER_NT_WORKSTATION)` | The system is a workstation. | - +---------------------------------------+---------------------------------+ - | :const:`2 (VER_NT_DOMAIN_CONTROLLER)` | The system is a domain | - | | controller. | - +---------------------------------------+---------------------------------+ - | :const:`3 (VER_NT_SERVER)` | The system is a server, but not | - | | a domain controller. | - +---------------------------------------+---------------------------------+ - - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information - about these fields. - - *platform_version* returns the accurate major version, minor version and - build number of the current operating system, rather than the version that - is being emulated for the process. It is intended for use in logging rather - than for feature detection. - - Availability: Windows. - - .. versionchanged:: 3.2 - Changed to a named tuple and added *service_pack_minor*, - *service_pack_major*, *suite_mask*, and *product_type*. - - .. versionchanged:: 3.6 - Added *platform_version* - - -.. function:: get_asyncgen_hooks() - - Returns an *asyncgen_hooks* object, which is similar to a - :class:`~collections.namedtuple` of the form `(firstiter, finalizer)`, - where *firstiter* and *finalizer* are expected to be either ``None`` or - functions which take an :term:`asynchronous generator iterator` as an - argument, and are used to schedule finalization of an asychronous - generator by an event loop. - - .. versionadded:: 3.6 - See :pep:`525` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) - - -.. function:: get_coroutine_wrapper() - - Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - -.. data:: hash_info - - A :term:`struct sequence` giving parameters of the numeric hash - implementation. For more details about hashing of numeric types, see - :ref:`numeric-hash`. - - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | hash value returned for a nan | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ - - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* - - -.. data:: hexversion - - The version number encoded as a single integer. This is guaranteed to increase - with each version, including proper support for non-production releases. For - example, to test that the Python interpreter is at least version 1.5.2, use:: - - if sys.hexversion >= 0x010502F0: - # use some advanced feature - ... - else: - # use an alternative implementation or warn the user - ... - - This is called ``hexversion`` since it only really looks meaningful when viewed - as the result of passing it to the built-in :func:`hex` function. The - :term:`struct sequence` :data:`sys.version_info` may be used for a more - human-friendly encoding of the same information. - - More details of ``hexversion`` can be found at :ref:`apiabiversion`. - - -.. data:: implementation - - An object containing information about the implementation of the - currently running Python interpreter. The following attributes are - required to exist in all Python implementations. - - *name* is the implementation's identifier, e.g. ``'cpython'``. The actual - string is defined by the Python implementation, but it is guaranteed to be - lower case. - - *version* is a named tuple, in the same format as - :data:`sys.version_info`. It represents the version of the Python - *implementation*. This has a distinct meaning from the specific - version of the Python *language* to which the currently running - interpreter conforms, which ``sys.version_info`` represents. For - example, for PyPy 1.8 ``sys.implementation.version`` might be - ``sys.version_info(1, 8, 0, 'final', 0)``, whereas ``sys.version_info`` - would be ``sys.version_info(2, 7, 2, 'final', 0)``. For CPython they - are the same value, since it is the reference implementation. - - *hexversion* is the implementation version in hexadecimal format, like - :data:`sys.hexversion`. - - *cache_tag* is the tag used by the import machinery in the filenames of - cached modules. By convention, it would be a composite of the - implementation's name and version, like ``'cpython-33'``. However, a - Python implementation may use some other value if appropriate. If - ``cache_tag`` is set to ``None``, it indicates that module caching should - be disabled. - - :data:`sys.implementation` may contain additional attributes specific to - the Python implementation. These non-standard attributes must start with - an underscore, and are not described here. Regardless of its contents, - :data:`sys.implementation` will not change during a run of the interpreter, - nor between implementation versions. (It may change between Python - language versions, however.) See :pep:`421` for more information. - - .. versionadded:: 3.3 - - -.. data:: int_info - - A :term:`struct sequence` that holds information about Python's internal - representation of integers. The attributes are read only. - - .. tabularcolumns:: |l|L| - - +-------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=========================+==============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +-------------------------+----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +-------------------------+----------------------------------------------+ - - .. versionadded:: 3.1 - - -.. data:: __interactivehook__ - - When this attribute exists, its value is automatically called (with no - arguments) when the interpreter is launched in :ref:`interactive mode - `. This is done after the :envvar:`PYTHONSTARTUP` file is - read, so that you can set this hook there. The :mod:`site` module - :ref:`sets this `. - - .. versionadded:: 3.4 - - -.. function:: intern(string) - - Enter *string* in the table of "interned" strings and return the interned string - -- which is *string* itself or a copy. Interning strings is useful to gain a - little performance on dictionary lookup -- if the keys in a dictionary are - interned, and the lookup key is interned, the key comparisons (after hashing) - can be done by a pointer compare instead of a string compare. Normally, the - names used in Python programs are automatically interned, and the dictionaries - used to hold module, class or instance attributes have interned keys. - - Interned strings are not immortal; you must keep a reference to the return - value of :func:`intern` around to benefit from it. - - -.. function:: is_finalizing() - - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. - - .. versionadded:: 3.5 - - -.. data:: last_type - last_value - last_traceback - - These three variables are not always defined; they are set when an exception is - not handled and the interpreter prints an error message and a stack traceback. - Their intended use is to allow an interactive user to import a debugger module - and engage in post-mortem debugging without having to re-execute the command - that caused the error. (Typical use is ``import pdb; pdb.pm()`` to enter the - post-mortem debugger; see :mod:`pdb` module for - more information.) - - The meaning of the variables is the same as that of the return values from - :func:`exc_info` above. - - -.. data:: maxsize - - An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can - take. It's usually ``2**31 - 1`` on a 32-bit platform and ``2**63 - 1`` on a - 64-bit platform. - - -.. data:: maxunicode - - An integer giving the value of the largest Unicode code point, - i.e. ``1114111`` (``0x10FFFF`` in hexadecimal). - - .. versionchanged:: 3.3 - Before :pep:`393`, ``sys.maxunicode`` used to be either ``0xFFFF`` - or ``0x10FFFF``, depending on the configuration option that specified - whether Unicode characters were stored as UCS-2 or UCS-4. - - -.. data:: meta_path - - A list of :term:`meta path finder` objects that have their - :meth:`~importlib.abc.MetaPathFinder.find_spec` methods called to see if one - of the objects can find the module to be imported. The - :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at - least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` - attribute is passed in as a second argument. The method returns a - :term:`module spec`, or ``None`` if the module cannot be found. - - .. seealso:: - - :class:`importlib.abc.MetaPathFinder` - The abstract base class defining the interface of finder objects on - :data:`meta_path`. - :class:`importlib.machinery.ModuleSpec` - The concrete class which - :meth:`~importlib.abc.MetaPathFinder.find_spec` should return - instances of. - - .. versionchanged:: 3.4 - - :term:`Module specs ` were introduced in Python 3.4, by - :pep:`451`. Earlier versions of Python looked for a method called - :meth:`~importlib.abc.MetaPathFinder.find_module`. - This is still called as a fallback if a :data:`meta_path` entry doesn't - have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. - -.. data:: modules - - This is a dictionary that maps module names to modules which have already been - loaded. This can be manipulated to force reloading of modules and other tricks. - However, replacing the dictionary will not necessarily work as expected and - deleting essential items from the dictionary may cause Python to fail. - - -.. data:: path - - .. index:: triple: module; search; path - - A list of strings that specifies the search path for modules. Initialized from - the environment variable :envvar:`PYTHONPATH`, plus an installation-dependent - default. - - As initialized upon program startup, the first item of this list, ``path[0]``, - is the directory containing the script that was used to invoke the Python - interpreter. If the script directory is not available (e.g. if the interpreter - is invoked interactively or if the script is read from standard input), - ``path[0]`` is the empty string, which directs Python to search modules in the - current directory first. Notice that the script directory is inserted *before* - the entries inserted as a result of :envvar:`PYTHONPATH`. - - A program is free to modify this list for its own purposes. Only strings - and bytes should be added to :data:`sys.path`; all other data types are - ignored during import. - - - .. seealso:: - Module :mod:`site` This describes how to use .pth files to extend - :data:`sys.path`. - - -.. data:: path_hooks - - A list of callables that take a path argument to try to create a - :term:`finder` for the path. If a finder can be created, it is to be - returned by the callable, else raise :exc:`ImportError`. - - Originally specified in :pep:`302`. - - -.. data:: path_importer_cache - - A dictionary acting as a cache for :term:`finder` objects. The keys are - paths that have been passed to :data:`sys.path_hooks` and the values are - the finders that are found. If a path is a valid file system path but no - finder is found on :data:`sys.path_hooks` then ``None`` is - stored. - - Originally specified in :pep:`302`. - - .. versionchanged:: 3.3 - ``None`` is stored instead of :class:`imp.NullImporter` when no finder - is found. - - -.. data:: platform - - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - - For other systems, the values are: - - ================ =========================== - System ``platform`` value - ================ =========================== - Linux ``'linux'`` - Windows ``'win32'`` - Windows/Cygwin ``'cygwin'`` - Mac OS X ``'darwin'`` - ================ =========================== - - .. versionchanged:: 3.3 - On Linux, :attr:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. - - .. seealso:: - - :attr:`os.name` has a coarser granularity. :func:`os.uname` gives - system-dependent version information. - - The :mod:`platform` module provides detailed checks for the - system's identity. - - -.. data:: prefix - - A string giving the site-specific directory prefix where the platform - independent Python files are installed; by default, this is the string - ``'/usr/local'``. This can be set at build time with the ``--prefix`` - argument to the :program:`configure` script. The main collection of Python - library modules is installed in the directory :file:`{prefix}/lib/python{X.Y}` - while the platform independent header files (all except :file:`pyconfig.h`) are - stored in :file:`{prefix}/include/python{X.Y}`, where *X.Y* is the version - number of Python, for example ``3.2``. - - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. - - -.. data:: ps1 - ps2 - - .. index:: - single: interpreter prompts - single: prompts, interpreter - single: >>>; interpreter prompt - single: ...; interpreter prompt - - Strings specifying the primary and secondary prompt of the interpreter. These - are only defined if the interpreter is in interactive mode. Their initial - values in this case are ``'>>> '`` and ``'... '``. If a non-string object is - assigned to either variable, its :func:`str` is re-evaluated each time the - interpreter prepares to read a new interactive command; this can be used to - implement a dynamic prompt. - - -.. function:: setcheckinterval(interval) - - Set the interpreter's "check interval". This integer value determines how often - the interpreter checks for periodic things such as thread switches and signal - handlers. The default is ``100``, meaning the check is performed every 100 - Python virtual instructions. Setting it to a larger value may increase - performance for programs using threads. Setting it to a value ``<=`` 0 checks - every virtual instruction, maximizing responsiveness as well as overhead. - - .. deprecated:: 3.2 - This function doesn't have an effect anymore, as the internal logic for - thread switching and asynchronous tasks has been rewritten. Use - :func:`setswitchinterval` instead. - - -.. function:: setdlopenflags(n) - - Set the flags used by the interpreter for :c:func:`dlopen` calls, such as when - the interpreter loads extension modules. Among other things, this will enable a - lazy resolving of symbols when importing a module, if called as - ``sys.setdlopenflags(0)``. To share symbols across extension modules, call as - ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values - can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. - :data:`os.RTLD_LAZY`). - - Availability: Unix. - -.. function:: setprofile(profilefunc) - - .. index:: - single: profile function - single: profiler - - Set the system's profile function, which allows you to implement a Python source - code profiler in Python. See chapter :ref:`profile` for more information on the - Python profiler. The system's profile function is called similarly to the - system's trace function (see :func:`settrace`), but it is called with different events, - for example it isn't called for each executed line of code (only on call and return, - but the return event is reported even when an exception has been set). The function is - thread-specific, but there is no way for the profiler to know about context switches between - threads, so it does not make sense to use this in the presence of multiple threads. Also, - its return value is not used, so it can simply return ``None``. - - Profile functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, - ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends - on the event type. - - The events have the following meaning: - - ``'call'`` - A function is called (or some other code block entered). The - profile function is called; *arg* is ``None``. - - ``'return'`` - A function (or other code block) is about to return. The profile - function is called; *arg* is the value that will be returned, or ``None`` - if the event is caused by an exception being raised. - - ``'c_call'`` - A C function is about to be called. This may be an extension function or - a built-in. *arg* is the C function object. - - ``'c_return'`` - A C function has returned. *arg* is the C function object. - - ``'c_exception'`` - A C function has raised an exception. *arg* is the C function object. - -.. function:: setrecursionlimit(limit) - - Set the maximum depth of the Python interpreter stack to *limit*. This limit - prevents infinite recursion from causing an overflow of the C stack and crashing - Python. - - The highest possible limit is platform-dependent. A user may need to set the - limit higher when they have a program that requires deep recursion and a platform - that supports a higher limit. This should be done with care, because a too-high - limit can lead to a crash. - - If the new limit is too low at the current recursion depth, a - :exc:`RecursionError` exception is raised. - - .. versionchanged:: 3.5.1 - A :exc:`RecursionError` exception is now raised if the new limit is too - low at the current recursion depth. - - -.. function:: setswitchinterval(interval) - - Set the interpreter's thread switch interval (in seconds). This floating-point - value determines the ideal duration of the "timeslices" allocated to - concurrently running Python threads. Please note that the actual value - can be higher, especially if long-running internal functions or methods - are used. Also, which thread becomes scheduled at the end of the interval - is the operating system's decision. The interpreter doesn't have its - own scheduler. - - .. versionadded:: 3.2 - - -.. function:: settrace(tracefunc) - - .. index:: - single: trace function - single: debugger - - Set the system's trace function, which allows you to implement a Python - source code debugger in Python. The function is thread-specific; for a - debugger to support multiple threads, it must be registered using - :func:`settrace` for each thread being debugged. - - Trace functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, - ``'line'``, ``'return'`` or ``'exception'``. *arg* depends on - the event type. - - The trace function is invoked (with *event* set to ``'call'``) whenever a new - local scope is entered; it should return a reference to a local trace - function to be used that scope, or ``None`` if the scope shouldn't be traced. - - The local trace function should return a reference to itself (or to another - function for further tracing in that scope), or ``None`` to turn off tracing - in that scope. - - The events have the following meaning: - - ``'call'`` - A function is called (or some other code block entered). The - global trace function is called; *arg* is ``None``; the return value - specifies the local trace function. - - ``'line'`` - The interpreter is about to execute a new line of code or re-execute the - condition of a loop. The local trace function is called; *arg* is - ``None``; the return value specifies the new local trace function. See - :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this - works. - - ``'return'`` - A function (or other code block) is about to return. The local trace - function is called; *arg* is the value that will be returned, or ``None`` - if the event is caused by an exception being raised. The trace function's - return value is ignored. - - ``'exception'`` - An exception has occurred. The local trace function is called; *arg* is a - tuple ``(exception, value, traceback)``; the return value specifies the - new local trace function. - - Note that as an exception is propagated down the chain of callers, an - ``'exception'`` event is generated at each level. - - For more information on code and frame objects, refer to :ref:`types`. - - .. impl-detail:: - - The :func:`settrace` function is intended only for implementing debuggers, - profilers, coverage tools and the like. Its behavior is part of the - implementation platform, rather than part of the language definition, and - thus may not be available in all Python implementations. - -.. function:: set_asyncgen_hooks(firstiter, finalizer) - - Accepts two optional keyword arguments which are callables that accept an - :term:`asynchronous generator iterator` as an argument. The *firstiter* - callable will be called when an asynchronous generator is iterated for the - first time. The *finalizer* will be called when an asynchronous generator - is about to be garbage collected. - - .. versionadded:: 3.6 - See :pep:`525` for more details, and for a reference example of a - *finalizer* method see the implementation of - ``asyncio.Loop.shutdown_asyncgens`` in - :source:`Lib/asyncio/base_events.py` - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) - - -.. function:: set_coroutine_wrapper(wrapper) - - Allows intercepting creation of :term:`coroutine` objects (only ones that - are created by an :keyword:`async def` function; generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine` will not be - intercepted). - - The *wrapper* argument must be either: - - * a callable that accepts one argument (a coroutine object); - * ``None``, to reset the wrapper. - - If called twice, the new wrapper replaces the previous one. The function - is thread-specific. - - The *wrapper* callable cannot define new coroutines directly or indirectly:: - - def wrapper(coro): - async def wrap(coro): - return await coro - return wrap(coro) - sys.set_coroutine_wrapper(wrapper) - - async def foo(): - pass - - # The following line will fail with a RuntimeError, because - # ``wrapper`` creates a ``wrap(coro)`` coroutine: - foo() - - See also :func:`get_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - -.. function:: _enablelegacywindowsfsencoding() - - Changes the default filesystem encoding and errors mode to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - Availability: Windows - - .. versionadded:: 3.6 - See :pep:`529` for more details. - -.. data:: stdin - stdout - stderr - - :term:`File objects ` used by the interpreter for standard - input, output and errors: - - * ``stdin`` is used for all interactive input (including calls to - :func:`input`); - * ``stdout`` is used for the output of :func:`print` and :term:`expression` - statements and for the prompts of :func:`input`; - * The interpreter's own prompts and its error messages go to ``stderr``. - - These streams are regular :term:`text files ` like those - returned by the :func:`open` function. Their parameters are chosen as - follows: - - * The character encoding is platform-dependent. Under Windows, if the stream - is interactive (that is, if its :meth:`isatty` method returns ``True``), the - console codepage is used, otherwise the ANSI code page. Under other - platforms, the locale encoding is used (see :meth:`locale.getpreferredencoding`). - - Under all platforms though, you can override this value by setting the - :envvar:`PYTHONIOENCODING` environment variable before starting Python. - - * When interactive, standard streams are line-buffered. Otherwise, they - are block-buffered like regular text files. You can override this - value with the :option:`-u` command-line option. - - .. note:: - - To write or read binary data from/to the standard streams, use the - underlying binary :data:`~io.TextIOBase.buffer` object. For example, to - write bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. - - However, if you are writing a library (and do not control in which - context its code will be executed), be aware that the standard streams - may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. - - -.. data:: __stdin__ - __stdout__ - __stderr__ - - These objects contain the original values of ``stdin``, ``stderr`` and - ``stdout`` at the start of the program. They are used during finalization, - and could be useful to print to the actual standard stream no matter if the - ``sys.std*`` object has been redirected. - - It can also be used to restore the actual files to known working file objects - in case they have been overwritten with a broken object. However, the - preferred way to do this is to explicitly save the previous stream before - replacing it, and restore the saved object. - - .. note:: - Under some conditions ``stdin``, ``stdout`` and ``stderr`` as well as the - original values ``__stdin__``, ``__stdout__`` and ``__stderr__`` can be - ``None``. It is usually the case for Windows GUI apps that aren't connected - to a console and Python apps started with :program:`pythonw`. - - -.. data:: thread_info - - A :term:`struct sequence` holding information about the thread - implementation. - - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if these informations are unknown. | - +------------------+---------------------------------------------------------+ - - .. versionadded:: 3.3 - - -.. data:: tracebacklimit - - When this variable is set to an integer value, it determines the maximum number - of levels of traceback information printed when an unhandled exception occurs. - The default is ``1000``. When set to ``0`` or less, all traceback information - is suppressed and only the exception type and value are printed. - - -.. data:: version - - A string containing the version number of the Python interpreter plus additional - information on the build number and compiler used. This string is displayed - when the interactive interpreter is started. Do not extract version information - out of it, rather, use :data:`version_info` and the functions provided by the - :mod:`platform` module. - - -.. data:: api_version - - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. - - -.. data:: version_info - - A tuple containing the five components of the version number: *major*, *minor*, - *micro*, *releaselevel*, and *serial*. All values except *releaselevel* are - integers; the release level is ``'alpha'``, ``'beta'``, ``'candidate'``, or - ``'final'``. The ``version_info`` value corresponding to the Python version 2.0 - is ``(2, 0, 0, 'final', 0)``. The components can also be accessed by name, - so ``sys.version_info[0]`` is equivalent to ``sys.version_info.major`` - and so on. - - .. versionchanged:: 3.1 - Added named component attributes. - -.. data:: warnoptions - - This is an implementation detail of the warnings framework; do not modify this - value. Refer to the :mod:`warnings` module for more information on the warnings - framework. - - -.. data:: winver - - The version number used to form registry keys on Windows platforms. This is - stored as string resource 1000 in the Python DLL. The value is normally the - first three characters of :const:`version`. It is provided in the :mod:`sys` - module for informational purposes; modifying this value has no effect on the - registry keys used by Python. Availability: Windows. - - -.. data:: _xoptions - - A dictionary of the various implementation-specific flags passed through - the :option:`-X` command-line option. Option names are either mapped to - their values, if given explicitly, or to :const:`True`. Example: - - .. code-block:: shell-session - - $ ./python -Xa=b -Xc - Python 3.2a3+ (py3k, Oct 16 2010, 20:14:50) - [GCC 4.4.3] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys._xoptions - {'a': 'b', 'c': True} - - .. impl-detail:: - - This is a CPython-specific way of accessing options passed through - :option:`-X`. Other implementations may export them through other - means, or not at all. - - .. versionadded:: 3.2 - - -.. rubric:: Citations - -.. [C99] ISO/IEC 9899:1999. "Programming languages -- C." A public draft of this standard is available at http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf\ . diff --git a/third_party/python/Doc/library/sysconfig.rst b/third_party/python/Doc/library/sysconfig.rst deleted file mode 100644 index d29396e01..000000000 --- a/third_party/python/Doc/library/sysconfig.rst +++ /dev/null @@ -1,262 +0,0 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= - -.. module:: sysconfig - :synopsis: Python's configuration information - -.. moduleauthor:: Tarek Ziadé -.. sectionauthor:: Tarek Ziadé - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/sysconfig.py` - -.. index:: - single: configuration information - --------------- - -The :mod:`sysconfig` module provides access to Python's configuration -information like the list of installation paths and the configuration variables -relevant for the current platform. - -Configuration variables ------------------------ - -A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` -header file that are necessary to build both the Python binary itself and -third-party C extensions compiled using :mod:`distutils`. - -:mod:`sysconfig` puts all variables found in these files in a dictionary that -can be accessed using :func:`get_config_vars` or :func:`get_config_var`. - -Notice that on Windows, it's a much smaller set. - -.. function:: get_config_vars(\*args) - - With no arguments, return a dictionary of all configuration variables - relevant for the current platform. - - With arguments, return a list of values that result from looking up each - argument in the configuration variable dictionary. - - For each argument, if the value is not found, return ``None``. - - -.. function:: get_config_var(name) - - Return the value of a single variable *name*. Equivalent to - ``get_config_vars().get(name)``. - - If *name* is not found, return ``None``. - -Example of usage:: - - >>> import sysconfig - >>> sysconfig.get_config_var('Py_ENABLE_SHARED') - 0 - >>> sysconfig.get_config_var('LIBDIR') - '/usr/local/lib' - >>> sysconfig.get_config_vars('AR', 'CXX') - ['ar', 'g++'] - - -Installation paths ------------------- - -Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under -unique identifiers based on the value returned by :const:`os.name`. - -Every new component that is installed using :mod:`distutils` or a -Distutils-based system will follow the same scheme to copy its file in the right -places. - -Python currently supports seven schemes: - -- *posix_prefix*: scheme for Posix platforms like Linux or Mac OS X. This is - the default scheme used when Python or a component is installed. -- *posix_home*: scheme for Posix platforms used when a *home* option is used - upon installation. This scheme is used when a component is installed through - Distutils with a specific home prefix. -- *posix_user*: scheme for Posix platforms used when a component is installed - through Distutils and the *user* option is used. This scheme defines paths - located under the user home directory. -- *nt*: scheme for NT platforms like Windows. -- *nt_user*: scheme for NT platforms, when the *user* option is used. - -Each scheme is itself composed of a series of paths and each path has a unique -identifier. Python currently uses eight paths: - -- *stdlib*: directory containing the standard Python library files that are not - platform-specific. -- *platstdlib*: directory containing the standard Python library files that are - platform-specific. -- *platlib*: directory for site-specific, platform-specific files. -- *purelib*: directory for site-specific, non-platform-specific files. -- *include*: directory for non-platform-specific header files. -- *platinclude*: directory for platform-specific header files. -- *scripts*: directory for script files. -- *data*: directory for data files. - -:mod:`sysconfig` provides some functions to determine these paths. - -.. function:: get_scheme_names() - - Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. - - -.. function:: get_path_names() - - Return a tuple containing all path names currently supported in - :mod:`sysconfig`. - - -.. function:: get_path(name, [scheme, [vars, [expand]]]) - - Return an installation path corresponding to the path *name*, from the - install scheme named *scheme*. - - *name* has to be a value from the list returned by :func:`get_path_names`. - - :mod:`sysconfig` stores installation paths corresponding to each path name, - for each platform, with variables to be expanded. For instance the *stdlib* - path for the *nt* scheme is: ``{base}/Lib``. - - :func:`get_path` will use the variables returned by :func:`get_config_vars` - to expand the path. All variables have default values for each platform so - one may call this function and get the default value. - - If *scheme* is provided, it must be a value from the list returned by - :func:`get_scheme_names`. Otherwise, the default scheme for the current - platform is used. - - If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. - - If *expand* is set to ``False``, the path will not be expanded using the - variables. - - If *name* is not found, return ``None``. - - -.. function:: get_paths([scheme, [vars, [expand]]]) - - Return a dictionary containing all installation paths corresponding to an - installation scheme. See :func:`get_path` for more information. - - If *scheme* is not provided, will use the default scheme for the current - platform. - - If *vars* is provided, it must be a dictionary of variables that will - update the dictionary used to expand the paths. - - If *expand* is set to false, the paths will not be expanded. - - If *scheme* is not an existing scheme, :func:`get_paths` will raise a - :exc:`KeyError`. - - -Other functions ---------------- - -.. function:: get_python_version() - - Return the ``MAJOR.MINOR`` Python version number as a string. Similar to - ``'%d.%d' % sys.version_info[:2]``. - - -.. function:: get_platform() - - Return a string that identifies the current platform. - - This is used mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by :func:`os.uname`), although the - exact information included depends on the OS; e.g. for IRIX the architecture - isn't particularly important (IRIX only runs on SGI hardware), but for Linux - the kernel version isn't particularly important. - - Examples of returned values: - - - linux-i586 - - linux-alpha (?) - - solaris-2.6-sun4u - - irix-5.3 - - irix64-6.2 - - Windows will return one of: - - - win-amd64 (64bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - - win-ia64 (64bit Windows on Itanium) - - win32 (all others - specifically, sys.platform is returned) - - Mac OS X can return: - - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat - - For other non-POSIX platforms, currently just returns :data:`sys.platform`. - - -.. function:: is_python_build() - - Return ``True`` if the running Python interpreter was built from source and - is being run from its built location, and not from a location resulting from - e.g. running ``make install`` or installing via a binary installer. - - -.. function:: parse_config_h(fp[, vars]) - - Parse a :file:`config.h`\-style file. - - *fp* is a file-like object pointing to the :file:`config.h`\-like file. - - A dictionary containing name/value pairs is returned. If an optional - dictionary is passed in as the second argument, it is used instead of a new - dictionary, and updated with the values read in the file. - - -.. function:: get_config_h_filename() - - Return the path of :file:`pyconfig.h`. - -.. function:: get_makefile_filename() - - Return the path of :file:`Makefile`. - -Using :mod:`sysconfig` as a script ----------------------------------- - -You can use :mod:`sysconfig` as a script with Python's *-m* option: - -.. code-block:: shell-session - - $ python -m sysconfig - Platform: "macosx-10.4-i386" - Python version: "3.2" - Current installation scheme: "posix_prefix" - - Paths: - data = "/usr/local" - include = "/Users/tarek/Dev/svn.python.org/py3k/Include" - platinclude = "." - platlib = "/usr/local/lib/python3.2/site-packages" - platstdlib = "/usr/local/lib/python3.2" - purelib = "/usr/local/lib/python3.2/site-packages" - scripts = "/usr/local/bin" - stdlib = "/usr/local/lib/python3.2" - - Variables: - AC_APPLE_UNIVERSAL_BUILD = "0" - AIX_GENUINE_CPLUSPLUS = "0" - AR = "ar" - ARFLAGS = "rc" - ... - -This call will print in the standard output the information returned by -:func:`get_platform`, :func:`get_python_version`, :func:`get_path` and -:func:`get_config_vars`. diff --git a/third_party/python/Doc/library/syslog.rst b/third_party/python/Doc/library/syslog.rst deleted file mode 100644 index af3fb9b57..000000000 --- a/third_party/python/Doc/library/syslog.rst +++ /dev/null @@ -1,112 +0,0 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== - -.. module:: syslog - :platform: Unix - :synopsis: An interface to the Unix syslog library routines. - --------------- - -This module provides an interface to the Unix ``syslog`` library routines. -Refer to the Unix manual pages for a detailed description of the ``syslog`` -facility. - -This module wraps the system ``syslog`` family of routines. A pure Python -library that can speak to a syslog server is available in the -:mod:`logging.handlers` module as :class:`SysLogHandler`. - -The module defines the following functions: - - -.. function:: syslog(message) - syslog(priority, message) - - Send the string *message* to the system logger. A trailing newline is added - if necessary. Each message is tagged with a priority composed of a - *facility* and a *level*. The optional *priority* argument, which defaults - to :const:`LOG_INFO`, determines the message priority. If the facility is - not encoded in *priority* using logical-or (``LOG_INFO | LOG_USER``), the - value given in the :func:`openlog` call is used. - - If :func:`openlog` has not been called prior to the call to :func:`syslog`, - ``openlog()`` will be called with no arguments. - - -.. function:: openlog([ident[, logoption[, facility]]]) - - Logging options of subsequent :func:`syslog` calls can be set by calling - :func:`openlog`. :func:`syslog` will call :func:`openlog` with no arguments - if the log is not currently open. - - The optional *ident* keyword argument is a string which is prepended to every - message, and defaults to ``sys.argv[0]`` with leading path components - stripped. The optional *logoption* keyword argument (default is 0) is a bit - field -- see below for possible values to combine. The optional *facility* - keyword argument (default is :const:`LOG_USER`) sets the default facility for - messages which do not have a facility explicitly encoded. - - .. versionchanged:: 3.2 - In previous versions, keyword arguments were not allowed, and *ident* was - required. The default for *ident* was dependent on the system libraries, - and often was ``python`` instead of the name of the python program file. - - -.. function:: closelog() - - Reset the syslog module values and call the system library ``closelog()``. - - This causes the module to behave as it does when initially imported. For - example, :func:`openlog` will be called on the first :func:`syslog` call (if - :func:`openlog` hasn't already been called), and *ident* and other - :func:`openlog` parameters are reset to defaults. - - -.. function:: setlogmask(maskpri) - - Set the priority mask to *maskpri* and return the previous mask value. Calls - to :func:`syslog` with a priority level not set in *maskpri* are ignored. - The default is to log all priorities. The function ``LOG_MASK(pri)`` - calculates the mask for the individual priority *pri*. The function - ``LOG_UPTO(pri)`` calculates the mask for all priorities up to and including - *pri*. - -The module defines the following constants: - -Priority levels (high to low): - :const:`LOG_EMERG`, :const:`LOG_ALERT`, :const:`LOG_CRIT`, :const:`LOG_ERR`, - :const:`LOG_WARNING`, :const:`LOG_NOTICE`, :const:`LOG_INFO`, - :const:`LOG_DEBUG`. - -Facilities: - :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, - :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ````, - :const:`LOG_AUTHPRIV`. - -Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ````, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. - - -Examples --------- - -Simple example -~~~~~~~~~~~~~~ - -A simple set of examples:: - - import syslog - - syslog.syslog('Processing started') - if error: - syslog.syslog(syslog.LOG_ERR, 'Processing started') - -An example of setting some log options, these would include the process ID in -logged messages, and write the messages to the destination facility used for -mail logging:: - - syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL) - syslog.syslog('E-mail processing initiated...') diff --git a/third_party/python/Doc/library/tabnanny.rst b/third_party/python/Doc/library/tabnanny.rst deleted file mode 100644 index dfe688a2f..000000000 --- a/third_party/python/Doc/library/tabnanny.rst +++ /dev/null @@ -1,67 +0,0 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== - -.. module:: tabnanny - :synopsis: Tool for detecting white space related problems in Python - source files in a directory tree. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Peter Funk - -.. rudimentary documentation based on module comments - -**Source code:** :source:`Lib/tabnanny.py` - --------------- - -For the time being this module is intended to be called as a script. However it -is possible to import it into an IDE and use the function :func:`check` -described below. - -.. note:: - - The API provided by this module is likely to change in future releases; such - changes may not be backward compatible. - - -.. function:: check(file_or_dir) - - If *file_or_dir* is a directory and not a symbolic link, then recursively - descend the directory tree named by *file_or_dir*, checking all :file:`.py` - files along the way. If *file_or_dir* is an ordinary Python source file, it - is checked for whitespace related problems. The diagnostic messages are - written to standard output using the :func:`print` function. - - -.. data:: verbose - - Flag indicating whether to print verbose messages. This is incremented by the - ``-v`` option if called as a script. - - -.. data:: filename_only - - Flag indicating whether to print only the filenames of files containing - whitespace related problems. This is set to true by the ``-q`` option if called - as a script. - - -.. exception:: NannyNag - - Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and - handled in :func:`check`. - - -.. function:: process_tokens(tokens) - - This function is used by :func:`check` to process tokens generated by the - :mod:`tokenize` module. - -.. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, - reset_globals - - -.. seealso:: - - Module :mod:`tokenize` - Lexical scanner for Python source code. diff --git a/third_party/python/Doc/library/tarfile.rst b/third_party/python/Doc/library/tarfile.rst deleted file mode 100644 index 337c06110..000000000 --- a/third_party/python/Doc/library/tarfile.rst +++ /dev/null @@ -1,880 +0,0 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== - -.. module:: tarfile - :synopsis: Read and write tar-format archive files. - -.. moduleauthor:: Lars Gustäbel -.. sectionauthor:: Lars Gustäbel - -**Source code:** :source:`Lib/tarfile.py` - --------------- - -The :mod:`tarfile` module makes it possible to read and write tar -archives, including those using gzip, bz2 and lzma compression. -Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the -higher-level functions in :ref:`shutil `. - -Some facts and figures: - -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. - -* read/write support for the POSIX.1-1988 (ustar) format. - -* read/write support for the GNU tar format including *longname* and *longlink* - extensions, read-only support for all variants of the *sparse* extension - including restoration of sparse files. - -* read/write support for the POSIX.1-2001 (pax) format. - -* handles directories, regular files, hardlinks, symbolic links, fifos, - character devices and block devices and is able to acquire and restore file - information like timestamp, access permissions and owner. - -.. versionchanged:: 3.3 - Added support for :mod:`lzma` compression. - - -.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, \*\*kwargs) - - Return a :class:`TarFile` object for the pathname *name*. For detailed - information on :class:`TarFile` objects and the keyword arguments that are - allowed, see :ref:`tarfile-objects`. - - *mode* has to be a string of the form ``'filemode[:compression]'``, it defaults - to ``'r'``. Here is a full list of mode combinations: - - +------------------+---------------------------------------------+ - | mode | action | - +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | - +------------------+---------------------------------------------+ - | ``'r:'`` | Open for reading exclusively without | - | | compression. | - +------------------+---------------------------------------------+ - | ``'r:gz'`` | Open for reading with gzip compression. | - +------------------+---------------------------------------------+ - | ``'r:bz2'`` | Open for reading with bzip2 compression. | - +------------------+---------------------------------------------+ - | ``'r:xz'`` | Open for reading with lzma compression. | - +------------------+---------------------------------------------+ - | ``'x'`` or | Create a tarfile exclusively without | - | ``'x:'`` | compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:gz'`` | Create a tarfile with gzip compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:bz2'`` | Create a tarfile with bzip2 compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:xz'`` | Create a tarfile with lzma compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | - +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | - +------------------+---------------------------------------------+ - | ``'w:gz'`` | Open for gzip compressed writing. | - +------------------+---------------------------------------------+ - | ``'w:bz2'`` | Open for bzip2 compressed writing. | - +------------------+---------------------------------------------+ - | ``'w:xz'`` | Open for lzma compressed writing. | - +------------------+---------------------------------------------+ - - Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* - is not suitable to open a certain (compressed) file for reading, - :exc:`ReadError` is raised. Use *mode* ``'r'`` to avoid this. If a - compression method is not supported, :exc:`CompressionError` is raised. - - If *fileobj* is specified, it is used as an alternative to a :term:`file object` - opened in binary mode for *name*. It is supposed to be at position 0. - - For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, ``'x:gz'``, - ``'x:bz2'``, :func:`tarfile.open` accepts the keyword argument - *compresslevel* (default ``9``) to specify the compression level of the file. - - For special purposes, there is a second format for *mode*: - ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` - object that processes its data as a stream of blocks. No random seeking will - be done on the file. If given, *fileobj* may be any object that has a - :meth:`read` or :meth:`write` method (depending on the *mode*). *bufsize* - specifies the blocksize and defaults to ``20 * 512`` bytes. Use this variant - in combination with e.g. ``sys.stdin``, a socket :term:`file object` or a tape - device. However, such a :class:`TarFile` object is limited in that it does - not allow random access, see :ref:`tar-examples`. The currently - possible modes: - - +-------------+--------------------------------------------+ - | Mode | Action | - +=============+============================================+ - | ``'r|*'`` | Open a *stream* of tar blocks for reading | - | | with transparent compression. | - +-------------+--------------------------------------------+ - | ``'r|'`` | Open a *stream* of uncompressed tar blocks | - | | for reading. | - +-------------+--------------------------------------------+ - | ``'r|gz'`` | Open a gzip compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'r|bz2'`` | Open a bzip2 compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'r|xz'`` | Open an lzma compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'w|'`` | Open an uncompressed *stream* for writing. | - +-------------+--------------------------------------------+ - | ``'w|gz'`` | Open a gzip compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - | ``'w|bz2'`` | Open a bzip2 compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - | ``'w|xz'`` | Open an lzma compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. class:: TarFile - - Class for reading and writing tar archives. Do not use this class directly: - use :func:`tarfile.open` instead. See :ref:`tarfile-objects`. - - -.. function:: is_tarfile(name) - - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` - module can read. - - -The :mod:`tarfile` module defines the following exceptions: - - -.. exception:: TarError - - Base class for all :mod:`tarfile` exceptions. - - -.. exception:: ReadError - - Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. - - -.. exception:: CompressionError - - Is raised when a compression method is not supported or when the data cannot be - decoded properly. - - -.. exception:: StreamError - - Is raised for the limitations that are typical for stream-like :class:`TarFile` - objects. - - -.. exception:: ExtractError - - Is raised for *non-fatal* errors when using :meth:`TarFile.extract`, but only if - :attr:`TarFile.errorlevel`\ ``== 2``. - - -.. exception:: HeaderError - - Is raised by :meth:`TarInfo.frombuf` if the buffer it gets is invalid. - - -The following constants are available at the module level: - -.. data:: ENCODING - - The default character encoding: ``'utf-8'`` on Windows, the value returned by - :func:`sys.getfilesystemencoding` otherwise. - - -Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for -details. - - -.. data:: USTAR_FORMAT - - POSIX.1-1988 (ustar) format. - - -.. data:: GNU_FORMAT - - GNU tar format. - - -.. data:: PAX_FORMAT - - POSIX.1-2001 (pax) format. - - -.. data:: DEFAULT_FORMAT - - The default format for creating archives. This is currently :const:`GNU_FORMAT`. - - -.. seealso:: - - Module :mod:`zipfile` - Documentation of the :mod:`zipfile` standard module. - - :ref:`archiving-operations` - Documentation of the higher-level archiving facilities provided by the - standard :mod:`shutil` module. - - `GNU tar manual, Basic Tar Format `_ - Documentation for tar archive files, including GNU tar extensions. - - -.. _tarfile-objects: - -TarFile Objects ---------------- - -The :class:`TarFile` object provides an interface to a tar archive. A tar -archive is a sequence of blocks. An archive member (a stored file) is made up of -a header block followed by data blocks. It is possible to store a file in a tar -archive several times. Each archive member is represented by a :class:`TarInfo` -object, see :ref:`tarinfo-objects` for details. - -A :class:`TarFile` object can be used as a context manager in a :keyword:`with` -statement. It will automatically be closed when the block is completed. Please -note that in the event of an exception an archive opened for writing will not -be finalized; only the internally used file object will be closed. See the -:ref:`tar-examples` section for a use case. - -.. versionadded:: 3.2 - Added support for the context management protocol. - -.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=0) - - All following arguments are optional and can be accessed as instance attributes - as well. - - *name* is the pathname of the archive. *name* may be a :term:`path-like object`. - It can be omitted if *fileobj* is given. - In this case, the file object's :attr:`name` attribute is used if it exists. - - *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append - data to an existing file, ``'w'`` to create a new file overwriting an existing - one, or ``'x'`` to create a new file only if it does not already exist. - - If *fileobj* is given, it is used for reading or writing data. If it can be - determined, *mode* is overridden by *fileobj*'s mode. *fileobj* will be used - from position 0. - - .. note:: - - *fileobj* is not closed, when :class:`TarFile` is closed. - - *format* controls the archive format. It must be one of the constants - :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are - defined at module level. - - The *tarinfo* argument can be used to replace the default :class:`TarInfo` class - with a different one. - - If *dereference* is :const:`False`, add symbolic and hard links to the archive. If it - is :const:`True`, add the content of the target files to the archive. This has no - effect on systems that do not support symbolic links. - - If *ignore_zeros* is :const:`False`, treat an empty block as the end of the archive. - If it is :const:`True`, skip empty (and invalid) blocks and try to get as many members - as possible. This is only useful for reading concatenated or damaged archives. - - *debug* can be set from ``0`` (no debug messages) up to ``3`` (all debug - messages). The messages are written to ``sys.stderr``. - - If *errorlevel* is ``0``, all errors are ignored when using :meth:`TarFile.extract`. - Nevertheless, they appear as error messages in the debug output, when debugging - is enabled. If ``1``, all *fatal* errors are raised as :exc:`OSError` - exceptions. If ``2``, all *non-fatal* errors are raised as :exc:`TarError` - exceptions as well. - - The *encoding* and *errors* arguments define the character encoding to be - used for reading or writing the archive and how conversion errors are going - to be handled. The default settings will work for most users. - See section :ref:`tar-unicode` for in-depth information. - - The *pax_headers* argument is an optional dictionary of strings which - will be added as a pax global header if *format* is :const:`PAX_FORMAT`. - - .. versionchanged:: 3.2 - Use ``'surrogateescape'`` as the default for the *errors* argument. - - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. classmethod:: TarFile.open(...) - - Alternative constructor. The :func:`tarfile.open` function is actually a - shortcut to this classmethod. - - -.. method:: TarFile.getmember(name) - - Return a :class:`TarInfo` object for member *name*. If *name* can not be found - in the archive, :exc:`KeyError` is raised. - - .. note:: - - If a member occurs more than once in the archive, its last occurrence is assumed - to be the most up-to-date version. - - -.. method:: TarFile.getmembers() - - Return the members of the archive as a list of :class:`TarInfo` objects. The - list has the same order as the members in the archive. - - -.. method:: TarFile.getnames() - - Return the members as a list of their names. It has the same order as the list - returned by :meth:`getmembers`. - - -.. method:: TarFile.list(verbose=True, *, members=None) - - Print a table of contents to ``sys.stdout``. If *verbose* is :const:`False`, - only the names of the members are printed. If it is :const:`True`, output - similar to that of :program:`ls -l` is produced. If optional *members* is - given, it must be a subset of the list returned by :meth:`getmembers`. - - .. versionchanged:: 3.5 - Added the *members* parameter. - - -.. method:: TarFile.next() - - Return the next member of the archive as a :class:`TarInfo` object, when - :class:`TarFile` is opened for reading. Return :const:`None` if there is no more - available. - - -.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False) - - Extract all members from the archive to the current working directory or - directory *path*. If optional *members* is given, it must be a subset of the - list returned by :meth:`getmembers`. Directory information like owner, - modification time and permissions are set after all members have been extracted. - This is done to work around two problems: A directory's modification time is - reset each time a file is created in it. And, if a directory's permissions do - not allow writing, extracting files to it will fail. - - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. - - .. warning:: - - Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - - .. versionchanged:: 3.5 - Added the *numeric_owner* parameter. - - .. versionchanged:: 3.6 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) - - Extract a member from the archive to the current working directory, using its - full name. Its file information is extracted as accurately as possible. *member* - may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. *path* may be a :term:`path-like object`. - File attributes (owner, mtime, mode) are set unless *set_attrs* is false. - - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. - - .. note:: - - The :meth:`extract` method does not take care of several extraction issues. - In most cases you should consider using the :meth:`extractall` method. - - .. warning:: - - See the warning for :meth:`extractall`. - - .. versionchanged:: 3.2 - Added the *set_attrs* parameter. - - .. versionchanged:: 3.5 - Added the *numeric_owner* parameter. - - .. versionchanged:: 3.6 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: TarFile.extractfile(member) - - Extract a member from the archive as a file object. *member* may be a filename - or a :class:`TarInfo` object. If *member* is a regular file or a link, an - :class:`io.BufferedReader` object is returned. Otherwise, :const:`None` is - returned. - - .. versionchanged:: 3.3 - Return an :class:`io.BufferedReader` object. - - -.. method:: TarFile.add(name, arcname=None, recursive=True, exclude=None, *, filter=None) - - Add the file *name* to the archive. *name* may be any type of file - (directory, fifo, symbolic link, etc.). If given, *arcname* specifies an - alternative name for the file in the archive. Directories are added - recursively by default. This can be avoided by setting *recursive* to - :const:`False`. If *exclude* is given, it must be a function that takes one - filename argument and returns a boolean value. Depending on this value the - respective file is either excluded (:const:`True`) or added - (:const:`False`). If *filter* is specified it must be a keyword argument. It - should be a function that takes a :class:`TarInfo` object argument and - returns the changed :class:`TarInfo` object. If it instead returns - :const:`None` the :class:`TarInfo` object will be excluded from the - archive. See :ref:`tar-examples` for an example. - - .. versionchanged:: 3.2 - Added the *filter* parameter. - - .. deprecated:: 3.2 - The *exclude* parameter is deprecated, please use the *filter* parameter - instead. - - -.. method:: TarFile.addfile(tarinfo, fileobj=None) - - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can - create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. - - -.. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) - - Create a :class:`TarInfo` object from the result of :func:`os.stat` or - equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. - *name* may be a :term:`path-like object`. If - given, *arcname* specifies an alternative name for the file in the - archive, otherwise, the name is taken from *fileobj*’s - :attr:`~io.FileIO.name` attribute, or the *name* argument. The name - should be a text string. - - You can modify - some of the :class:`TarInfo`’s attributes before you add it using :meth:`addfile`. - If the file object is not an ordinary file object positioned at the - beginning of the file, attributes such as :attr:`~TarInfo.size` may need - modifying. This is the case for objects such as :class:`~gzip.GzipFile`. - The :attr:`~TarInfo.name` may also be modified, in which case *arcname* - could be a dummy string. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. method:: TarFile.close() - - Close the :class:`TarFile`. In write mode, two finishing zero blocks are - appended to the archive. - - -.. attribute:: TarFile.pax_headers - - A dictionary containing key-value pairs of pax global headers. - - - -.. _tarinfo-objects: - -TarInfo Objects ---------------- - -A :class:`TarInfo` object represents one member in a :class:`TarFile`. Aside -from storing all required attributes of a file (like file type, size, time, -permissions, owner etc.), it provides some useful methods to determine its type. -It does *not* contain the file's data itself. - -:class:`TarInfo` objects are returned by :class:`TarFile`'s methods -:meth:`getmember`, :meth:`getmembers` and :meth:`gettarinfo`. - - -.. class:: TarInfo(name="") - - Create a :class:`TarInfo` object. - - -.. classmethod:: TarInfo.frombuf(buf, encoding, errors) - - Create and return a :class:`TarInfo` object from string buffer *buf*. - - Raises :exc:`HeaderError` if the buffer is invalid. - - -.. classmethod:: TarInfo.fromtarfile(tarfile) - - Read the next member from the :class:`TarFile` object *tarfile* and return it as - a :class:`TarInfo` object. - - -.. method:: TarInfo.tobuf(format=DEFAULT_FORMAT, encoding=ENCODING, errors='surrogateescape') - - Create a string buffer from a :class:`TarInfo` object. For information on the - arguments see the constructor of the :class:`TarFile` class. - - .. versionchanged:: 3.2 - Use ``'surrogateescape'`` as the default for the *errors* argument. - - -A ``TarInfo`` object has the following public data attributes: - - -.. attribute:: TarInfo.name - - Name of the archive member. - - -.. attribute:: TarInfo.size - - Size in bytes. - - -.. attribute:: TarInfo.mtime - - Time of last modification. - - -.. attribute:: TarInfo.mode - - Permission bits. - - -.. attribute:: TarInfo.type - - File type. *type* is usually one of these constants: :const:`REGTYPE`, - :const:`AREGTYPE`, :const:`LNKTYPE`, :const:`SYMTYPE`, :const:`DIRTYPE`, - :const:`FIFOTYPE`, :const:`CONTTYPE`, :const:`CHRTYPE`, :const:`BLKTYPE`, - :const:`GNUTYPE_SPARSE`. To determine the type of a :class:`TarInfo` object - more conveniently, use the ``is*()`` methods below. - - -.. attribute:: TarInfo.linkname - - Name of the target file name, which is only present in :class:`TarInfo` objects - of type :const:`LNKTYPE` and :const:`SYMTYPE`. - - -.. attribute:: TarInfo.uid - - User ID of the user who originally stored this member. - - -.. attribute:: TarInfo.gid - - Group ID of the user who originally stored this member. - - -.. attribute:: TarInfo.uname - - User name. - - -.. attribute:: TarInfo.gname - - Group name. - - -.. attribute:: TarInfo.pax_headers - - A dictionary containing key-value pairs of an associated pax extended header. - - -A :class:`TarInfo` object also provides some convenient query methods: - - -.. method:: TarInfo.isfile() - - Return :const:`True` if the :class:`Tarinfo` object is a regular file. - - -.. method:: TarInfo.isreg() - - Same as :meth:`isfile`. - - -.. method:: TarInfo.isdir() - - Return :const:`True` if it is a directory. - - -.. method:: TarInfo.issym() - - Return :const:`True` if it is a symbolic link. - - -.. method:: TarInfo.islnk() - - Return :const:`True` if it is a hard link. - - -.. method:: TarInfo.ischr() - - Return :const:`True` if it is a character device. - - -.. method:: TarInfo.isblk() - - Return :const:`True` if it is a block device. - - -.. method:: TarInfo.isfifo() - - Return :const:`True` if it is a FIFO. - - -.. method:: TarInfo.isdev() - - Return :const:`True` if it is one of character device, block device or FIFO. - - -.. _tarfile-commandline: -.. program:: tarfile - -Command-Line Interface ----------------------- - -.. versionadded:: 3.4 - -The :mod:`tarfile` module provides a simple command-line interface to interact -with tar archives. - -If you want to create a new tar archive, specify its name after the :option:`-c` -option and then list the filename(s) that should be included: - -.. code-block:: shell-session - - $ python -m tarfile -c monty.tar spam.txt eggs.txt - -Passing a directory is also acceptable: - -.. code-block:: shell-session - - $ python -m tarfile -c monty.tar life-of-brian_1979/ - -If you want to extract a tar archive into the current directory, use -the :option:`-e` option: - -.. code-block:: shell-session - - $ python -m tarfile -e monty.tar - -You can also extract a tar archive into a different directory by passing the -directory's name: - -.. code-block:: shell-session - - $ python -m tarfile -e monty.tar other-dir/ - -For a list of the files in a tar archive, use the :option:`-l` option: - -.. code-block:: shell-session - - $ python -m tarfile -l monty.tar - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -.. cmdoption:: -l - --list - - List files in a tarfile. - -.. cmdoption:: -c ... - --create ... - - Create tarfile from source files. - -.. cmdoption:: -e [] - --extract [] - - Extract tarfile into the current directory if *output_dir* is not specified. - -.. cmdoption:: -t - --test - - Test whether the tarfile is valid or not. - -.. cmdoption:: -v, --verbose - - Verbose output. - -.. _tar-examples: - -Examples --------- - -How to extract an entire tar archive to the current working directory:: - - import tarfile - tar = tarfile.open("sample.tar.gz") - tar.extractall() - tar.close() - -How to extract a subset of a tar archive with :meth:`TarFile.extractall` using -a generator function instead of a list:: - - import os - import tarfile - - def py_files(members): - for tarinfo in members: - if os.path.splitext(tarinfo.name)[1] == ".py": - yield tarinfo - - tar = tarfile.open("sample.tar.gz") - tar.extractall(members=py_files(tar)) - tar.close() - -How to create an uncompressed tar archive from a list of filenames:: - - import tarfile - tar = tarfile.open("sample.tar", "w") - for name in ["foo", "bar", "quux"]: - tar.add(name) - tar.close() - -The same example using the :keyword:`with` statement:: - - import tarfile - with tarfile.open("sample.tar", "w") as tar: - for name in ["foo", "bar", "quux"]: - tar.add(name) - -How to read a gzip compressed tar archive and display some member information:: - - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() - -How to create an archive and reset the user information using the *filter* -parameter in :meth:`TarFile.add`:: - - import tarfile - def reset(tarinfo): - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - return tarinfo - tar = tarfile.open("sample.tar.gz", "w:gz") - tar.add("foo", filter=reset) - tar.close() - - -.. _tar-formats: - -Supported tar formats ---------------------- - -There are three tar formats that can be created with the :mod:`tarfile` module: - -* The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames - up to a length of at best 256 characters and linknames up to 100 characters. The - maximum file size is 8 GiB. This is an old and limited but widely - supported format. - -* The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and - linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar - extensions for long names, sparse file support is read-only. - -* The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible - format with virtually no limits. It supports long filenames and linknames, large - files and stores pathnames in a portable way. However, not all tar - implementations today are able to handle pax archives properly. - - The *pax* format is an extension to the existing *ustar* format. It uses extra - headers for information that cannot be stored otherwise. There are two flavours - of pax headers: Extended headers only affect the subsequent file header, global - headers are valid for the complete archive and affect all following files. All - the data in a pax header is encoded in *UTF-8* for portability reasons. - -There are some more variants of the tar format which can be read, but not -created: - -* The ancient V7 format. This is the first tar format from Unix Seventh Edition, - storing only regular files and directories. Names must not be longer than 100 - characters, there is no user/group name information. Some archives have - miscalculated header checksums in case of fields with non-ASCII characters. - -* The SunOS tar extended format. This format is a variant of the POSIX.1-2001 - pax format, but is not compatible. - -.. _tar-unicode: - -Unicode issues --------------- - -The tar format was originally conceived to make backups on tape drives with the -main focus on preserving file system information. Nowadays tar archives are -commonly used for file distribution and exchanging archives over networks. One -problem of the original format (which is the basis of all other formats) is -that there is no concept of supporting different character encodings. For -example, an ordinary tar archive created on a *UTF-8* system cannot be read -correctly on a *Latin-1* system if it contains non-*ASCII* characters. Textual -metadata (like filenames, linknames, user/group names) will appear damaged. -Unfortunately, there is no way to autodetect the encoding of an archive. The -pax format was designed to solve this problem. It stores non-ASCII metadata -using the universal character encoding *UTF-8*. - -The details of character conversion in :mod:`tarfile` are controlled by the -*encoding* and *errors* keyword arguments of the :class:`TarFile` class. - -*encoding* defines the character encoding to use for the metadata in the -archive. The default value is :func:`sys.getfilesystemencoding` or ``'ascii'`` -as a fallback. Depending on whether the archive is read or written, the -metadata must be either decoded or encoded. If *encoding* is not set -appropriately, this conversion may fail. - -The *errors* argument defines how characters are treated that cannot be -converted. Possible values are listed in section :ref:`error-handlers`. -The default scheme is ``'surrogateescape'`` which Python also uses for its -file system calls, see :ref:`os-filenames`. - -In case of :const:`PAX_FORMAT` archives, *encoding* is generally not needed -because all the metadata is stored using *UTF-8*. *encoding* is only used in -the rare cases when binary pax headers are decoded or when strings with -surrogate characters are stored. diff --git a/third_party/python/Doc/library/telnetlib.rst b/third_party/python/Doc/library/telnetlib.rst deleted file mode 100644 index f9c5153e3..000000000 --- a/third_party/python/Doc/library/telnetlib.rst +++ /dev/null @@ -1,252 +0,0 @@ -:mod:`telnetlib` --- Telnet client -================================== - -.. module:: telnetlib - :synopsis: Telnet client class. - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/telnetlib.py` - -.. index:: single: protocol; Telnet - --------------- - -The :mod:`telnetlib` module provides a :class:`Telnet` class that implements the -Telnet protocol. See :rfc:`854` for details about the protocol. In addition, it -provides symbolic constants for the protocol characters (see below), and for the -telnet options. The symbolic names of the telnet options follow the definitions -in ``arpa/telnet.h``, with the leading ``TELOPT_`` removed. For symbolic names -of options which are traditionally not included in ``arpa/telnet.h``, see the -module source itself. - -The symbolic constants for the telnet commands are: IAC, DONT, DO, WONT, WILL, -SE (Subnegotiation End), NOP (No Operation), DM (Data Mark), BRK (Break), IP -(Interrupt process), AO (Abort output), AYT (Are You There), EC (Erase -Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). - - -.. class:: Telnet(host=None, port=0[, timeout]) - - :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`open` method must be used to - establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to - the server will be established before the constructor returns. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). - - Do not reopen an already connected instance. - - This class has many :meth:`read_\*` methods. Note that some of them raise - :exc:`EOFError` when the end of the connection is read, because they can return - an empty string for other reasons. See the individual descriptions below. - - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - - -.. seealso:: - - :rfc:`854` - Telnet Protocol Specification - Definition of the Telnet protocol. - - -.. _telnet-objects: - -Telnet Objects --------------- - -:class:`Telnet` instances have the following methods: - - -.. method:: Telnet.read_until(expected, timeout=None) - - Read until a given byte string, *expected*, is encountered or until *timeout* - seconds have passed. - - When no match is found, return whatever is available instead, possibly empty - bytes. Raise :exc:`EOFError` if the connection is closed and no cooked data - is available. - - -.. method:: Telnet.read_all() - - Read all data until EOF as bytes; block until connection closed. - - -.. method:: Telnet.read_some() - - Read at least one byte of cooked data unless EOF is hit. Return ``b''`` if - EOF is hit. Block if no data is immediately available. - - -.. method:: Telnet.read_very_eager() - - Read everything that can be without blocking in I/O (eager). - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_eager() - - Read readily available data. - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_lazy() - - Process and return data already in the queues (lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. Do not block unless in the - midst of an IAC sequence. - - -.. method:: Telnet.read_very_lazy() - - Return any data available in the cooked queue (very lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. This method never blocks. - - -.. method:: Telnet.read_sb_data() - - Return the data collected between a SB/SE pair (suboption begin/end). The - callback should access these data when it was invoked with a ``SE`` command. - This method never blocks. - - -.. method:: Telnet.open(host, port=0[, timeout]) - - Connect to a host. The optional second argument is the port number, which - defaults to the standard Telnet port (23). The optional *timeout* parameter - specifies a timeout in seconds for blocking operations like the connection - attempt (if not specified, the global default timeout setting will be used). - - Do not try to reopen an already connected instance. - - -.. method:: Telnet.msg(msg, *args) - - Print a debug message when the debug level is ``>`` 0. If extra arguments are - present, they are substituted in the message using the standard string - formatting operator. - - -.. method:: Telnet.set_debuglevel(debuglevel) - - Set the debug level. The higher the value of *debuglevel*, the more debug - output you get (on ``sys.stdout``). - - -.. method:: Telnet.close() - - Close the connection. - - -.. method:: Telnet.get_socket() - - Return the socket object used internally. - - -.. method:: Telnet.fileno() - - Return the file descriptor of the socket object used internally. - - -.. method:: Telnet.write(buffer) - - Write a byte string to the socket, doubling any IAC characters. This can - block if the connection is blocked. May raise :exc:`OSError` if the - connection is closed. - - .. versionchanged:: 3.3 - This method used to raise :exc:`socket.error`, which is now an alias - of :exc:`OSError`. - - -.. method:: Telnet.interact() - - Interaction function, emulates a very dumb Telnet client. - - -.. method:: Telnet.mt_interact() - - Multithreaded version of :meth:`interact`. - - -.. method:: Telnet.expect(list, timeout=None) - - Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The - optional second argument is a timeout, in seconds; the default is to block - indefinitely. - - Return a tuple of three items: the index in the list of the first regular - expression that matches; the match object returned; and the bytes read up - till and including the match. - - If end of file is found and no bytes were read, raise :exc:`EOFError`. - Otherwise, when nothing matches, return ``(-1, None, data)`` where *data* is - the bytes received so far (may be empty bytes if a timeout happened). - - If a regular expression ends with a greedy match (such as ``.*``) or if more - than one expression can match the same input, the results are - non-deterministic, and may depend on the I/O timing. - - -.. method:: Telnet.set_option_negotiation_callback(callback) - - Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command - (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. - - -.. _telnet-example: - -Telnet Example --------------- - -.. sectionauthor:: Peter Funk - - -A simple example illustrating typical use:: - - import getpass - import telnetlib - - HOST = "localhost" - user = input("Enter your remote account: ") - password = getpass.getpass() - - tn = telnetlib.Telnet(HOST) - - tn.read_until(b"login: ") - tn.write(user.encode('ascii') + b"\n") - if password: - tn.read_until(b"Password: ") - tn.write(password.encode('ascii') + b"\n") - - tn.write(b"ls\n") - tn.write(b"exit\n") - - print(tn.read_all().decode('ascii')) - diff --git a/third_party/python/Doc/library/tempfile.rst b/third_party/python/Doc/library/tempfile.rst deleted file mode 100644 index 0d0da4d62..000000000 --- a/third_party/python/Doc/library/tempfile.rst +++ /dev/null @@ -1,333 +0,0 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ - -.. module:: tempfile - :synopsis: Generate temporary files and directories. - -.. sectionauthor:: Zack Weinberg - -**Source code:** :source:`Lib/tempfile.py` - -.. index:: - pair: temporary; file name - pair: temporary; file - --------------- - -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. - -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. -To maintain backward compatibility, the argument order is somewhat odd; it -is recommended to use keyword arguments for clarity. - -The module defines the following user-callable items: - -.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None) - - Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon - as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed - immediately after the file is created. Other platforms do not support - this; your code should not rely on a temporary file created using this - function having or not having a visible name in the file system. - - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - - The *mode* parameter defaults to ``'w+b'`` so that the file created can - be read and written without being closed. Binary mode is used so that it - behaves consistently on all platforms without regard for the data that is - stored. *buffering*, *encoding* and *newline* are interpreted as for - :func:`open`. - - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. - - The returned object is a true file object on POSIX platforms. On other - platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. - - The :py:data:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - - .. versionchanged:: 3.5 - - The :py:data:`os.O_TMPFILE` flag is now used if available. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True) - - This function operates exactly as :func:`TemporaryFile` does, except that - the file is guaranteed to have a visible name in the file system (on - Unix, the directory entry is not unlinked). That name can be retrieved - from the :attr:`name` attribute of the returned - file-like object. Whether the name can be - used to open the file a second time, while the named temporary file is - still open, varies across platforms (it can be so used on Unix; it cannot - on Windows NT or later). If *delete* is true (the default), the file is - deleted as soon as it is closed. - The returned object is always a file-like object whose :attr:`!file` - attribute is the underlying true file object. This file-like object can - be used in a :keyword:`with` statement, just like a normal file. - - -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the - contents are written to disk and operation proceeds as with - :func:`TemporaryFile`. - - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. - - The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.StringIO` object (depending on - whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. - - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. - - -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) - - This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory - and all its contents are removed from the filesystem. - - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. - - .. versionadded:: 3.2 - - -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) - - Creates a temporary file in the most secure manner possible. There are - no race conditions in the file's creation, assuming that the platform - properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The - file is readable and writable only by the creating user ID. If the - platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. - - Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. - - If *suffix* is not ``None``, the file name will end with that suffix, - otherwise there will be no suffix. :func:`mkstemp` does not put a dot - between the file name and the suffix; if you need one, put it at the - beginning of *suffix*. - - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. - - If *dir* is not ``None``, the file will be created in that directory; - otherwise, a default directory is used. The default directory is chosen - from a platform-dependent list, but the user of the application can - control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* - environment variables. There is thus no guarantee that the generated - filename will have any nice properties, such as not requiring quoting - when passed to external commands via ``os.popen()``. - - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - - If *text* is specified, it indicates whether to open the file in binary - mode (the default) or text mode. On some platforms, this makes no - difference. - - :func:`mkstemp` returns a tuple containing an OS-level handle to an open - file (as would be returned by :func:`os.open`) and the absolute pathname - of that file, in that order. - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) - - Creates a temporary directory in the most secure manner possible. There - are no race conditions in the directory's creation. The directory is - readable, writable, and searchable only by the creating user ID. - - The user of :func:`mkdtemp` is responsible for deleting the temporary - directory and its contents when done with it. - - The *prefix*, *suffix*, and *dir* arguments are the same as for - :func:`mkstemp`. - - :func:`mkdtemp` returns the absolute pathname of the new directory. - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - -.. function:: gettempdir() - - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. - - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: - - #. The directory named by the :envvar:`TMPDIR` environment variable. - - #. The directory named by the :envvar:`TEMP` environment variable. - - #. The directory named by the :envvar:`TMP` environment variable. - - #. A platform-specific location: - - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. - - * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and - :file:`/usr/tmp`, in that order. - - #. As a last resort, the current working directory. - - The result of this search is cached, see the description of - :data:`tempdir` below. - -.. function:: gettempdirb() - - Same as :func:`gettempdir` but the return value is in bytes. - - .. versionadded:: 3.5 - -.. function:: gettempprefix() - - Return the filename prefix used to create temporary files. This does not - contain the directory component. - -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommended approach. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to the functions defined in this - module. - - If ``tempdir`` is ``None`` (the default) at any call to any of the above - functions except :func:`gettempprefix` it is initialized following the - algorithm described in :func:`gettempdir`. - -.. _tempfile-examples: - -Examples --------- - -Here are some examples of typical usage of the :mod:`tempfile` module:: - - >>> import tempfile - - # create a temporary file and write some data to it - >>> fp = tempfile.TemporaryFile() - >>> fp.write(b'Hello world!') - # read data from file - >>> fp.seek(0) - >>> fp.read() - b'Hello world!' - # close the file, it will be removed - >>> fp.close() - - # create a temporary file using a context manager - >>> with tempfile.TemporaryFile() as fp: - ... fp.write(b'Hello world!') - ... fp.seek(0) - ... fp.read() - b'Hello world!' - >>> - # file is now closed and removed - - # create a temporary directory using the context manager - >>> with tempfile.TemporaryDirectory() as tmpdirname: - ... print('created temporary directory', tmpdirname) - >>> - # directory and contents have been removed - - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff --git a/third_party/python/Doc/library/termios.rst b/third_party/python/Doc/library/termios.rst deleted file mode 100644 index d75a87c55..000000000 --- a/third_party/python/Doc/library/termios.rst +++ /dev/null @@ -1,105 +0,0 @@ -:mod:`termios` --- POSIX style tty control -========================================== - -.. module:: termios - :platform: Unix - :synopsis: POSIX style tty control. - -.. index:: - pair: POSIX; I/O control - pair: tty; I/O control - --------------- - -This module provides an interface to the POSIX calls for tty I/O control. For a -complete description of these calls, see :manpage:`termios(3)` Unix manual -page. It is only available for those Unix versions that support POSIX -*termios* style tty I/O control configured during installation. - -All functions in this module take a file descriptor *fd* as their first -argument. This can be an integer file descriptor, such as returned by -``sys.stdin.fileno()``, or a :term:`file object`, such as ``sys.stdin`` itself. - -This module also defines all the constants needed to work with the functions -provided here; these have the same name as their counterparts in C. Please -refer to your system documentation for more information on using these terminal -control interfaces. - -The module defines the following functions: - - -.. function:: tcgetattr(fd) - - Return a list containing the tty attributes for file descriptor *fd*, as - follows: ``[iflag, oflag, cflag, lflag, ispeed, ospeed, cc]`` where *cc* is a - list of the tty special characters (each a string of length 1, except the - items with indices :const:`VMIN` and :const:`VTIME`, which are integers when - these fields are defined). The interpretation of the flags and the speeds as - well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. - - -.. function:: tcsetattr(fd, when, attributes) - - Set the tty attributes for file descriptor *fd* from the *attributes*, which is - a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: :const:`TCSANOW` to change - immediately, :const:`TCSADRAIN` to change after transmitting all queued output, - or :const:`TCSAFLUSH` to change after transmitting all queued output and - discarding all queued input. - - -.. function:: tcsendbreak(fd, duration) - - Send a break on file descriptor *fd*. A zero *duration* sends a break for - 0.25--0.5 seconds; a nonzero *duration* has a system dependent meaning. - - -.. function:: tcdrain(fd) - - Wait until all output written to file descriptor *fd* has been transmitted. - - -.. function:: tcflush(fd, queue) - - Discard queued data on file descriptor *fd*. The *queue* selector specifies - which queue: :const:`TCIFLUSH` for the input queue, :const:`TCOFLUSH` for the - output queue, or :const:`TCIOFLUSH` for both queues. - - -.. function:: tcflow(fd, action) - - Suspend or resume input or output on file descriptor *fd*. The *action* - argument can be :const:`TCOOFF` to suspend output, :const:`TCOON` to restart - output, :const:`TCIOFF` to suspend input, or :const:`TCION` to restart input. - - -.. seealso:: - - Module :mod:`tty` - Convenience functions for common terminal control operations. - - -.. _termios-example: - -Example -------- - -Here's a function that prompts for a password with echoing turned off. Note the -technique using a separate :func:`tcgetattr` call and a :keyword:`try` ... -:keyword:`finally` statement to ensure that the old tty attributes are restored -exactly no matter what happens:: - - def getpass(prompt="Password: "): - import termios, sys - fd = sys.stdin.fileno() - old = termios.tcgetattr(fd) - new = termios.tcgetattr(fd) - new[3] = new[3] & ~termios.ECHO # lflags - try: - termios.tcsetattr(fd, termios.TCSADRAIN, new) - passwd = input(prompt) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old) - return passwd - diff --git a/third_party/python/Doc/library/test.rst b/third_party/python/Doc/library/test.rst deleted file mode 100644 index 04d6cd87e..000000000 --- a/third_party/python/Doc/library/test.rst +++ /dev/null @@ -1,686 +0,0 @@ -:mod:`test` --- Regression tests package for Python -=================================================== - -.. module:: test - :synopsis: Regression tests package containing the testing suite for Python. - -.. sectionauthor:: Brett Cannon - -.. note:: - The :mod:`test` package is meant for internal use by Python only. It is - documented for the benefit of the core developers of Python. Any use of - this package outside of Python's standard library is discouraged as code - mentioned here can change or be removed without notice between releases of - Python. - --------------- - -The :mod:`test` package contains all regression tests for Python as well as the -modules :mod:`test.support` and :mod:`test.regrtest`. -:mod:`test.support` is used to enhance your tests while -:mod:`test.regrtest` drives the testing suite. - -Each module in the :mod:`test` package whose name starts with ``test_`` is a -testing suite for a specific module or feature. All new tests should be written -using the :mod:`unittest` or :mod:`doctest` module. Some older tests are -written using a "traditional" testing style that compares output printed to -``sys.stdout``; this style of test is considered deprecated. - - -.. seealso:: - - Module :mod:`unittest` - Writing PyUnit regression tests. - - Module :mod:`doctest` - Tests embedded in documentation strings. - - -.. _writing-tests: - -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- - -It is preferred that tests that use the :mod:`unittest` module follow a few -guidelines. One is to name the test module by starting it with ``test_`` and end -it with the name of the module being tested. The test methods in the test module -should start with ``test_`` and end with a description of what the method is -testing. This is needed so that the methods are recognized by the test driver as -test methods. Also, no documentation string for the method should be included. A -comment (such as ``# Tests function returns only True or False``) should be used -to provide documentation for test methods. This is done because documentation -strings get printed out if they exist and thus what test is being run is not -stated. - -A basic boilerplate is often used:: - - import unittest - from test import support - - class MyTestCase1(unittest.TestCase): - - # Only use setUp() and tearDown() if necessary - - def setUp(self): - ... code to execute in preparation for tests ... - - def tearDown(self): - ... code to execute to clean up after tests ... - - def test_feature_one(self): - # Test feature one. - ... testing code ... - - def test_feature_two(self): - # Test feature two. - ... testing code ... - - ... more test methods ... - - class MyTestCase2(unittest.TestCase): - ... same structure as MyTestCase1 ... - - ... more test classes ... - - if __name__ == '__main__': - unittest.main() - -This code pattern allows the testing suite to be run by :mod:`test.regrtest`, -on its own as a script that supports the :mod:`unittest` CLI, or via the -``python -m unittest`` CLI. - -The goal for regression testing is to try to break code. This leads to a few -guidelines to be followed: - -* The testing suite should exercise all classes, functions, and constants. This - includes not just the external API that is to be presented to the outside - world but also "private" code. - -* Whitebox testing (examining the code being tested when the tests are being - written) is preferred. Blackbox testing (testing only the published user - interface) is not complete enough to make sure all boundary and edge cases - are tested. - -* Make sure all possible values are tested including invalid ones. This makes - sure that not only all valid values are acceptable but also that improper - values are handled correctly. - -* Exhaust as many code paths as possible. Test where branching occurs and thus - tailor input to make sure as many different paths through the code are taken. - -* Add an explicit test for any bugs discovered for the tested code. This will - make sure that the error does not crop up again if the code is changed in the - future. - -* Make sure to clean up after your tests (such as close and remove all temporary - files). - -* If a test is dependent on a specific condition of the operating system then - verify the condition already exists before attempting the test. - -* Import as few modules as possible and do it as soon as possible. This - minimizes external dependencies of tests and also minimizes possible anomalous - behavior from side-effects of importing a module. - -* Try to maximize code reuse. On occasion, tests will vary by something as small - as what type of input is used. Minimize code duplication by subclassing a - basic test class with a class that specifies the input:: - - class TestFuncAcceptsSequencesMixin: - - func = mySuperWhammyFunction - - def test_func(self): - self.func(self.arg) - - class AcceptLists(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = [1, 2, 3] - - class AcceptStrings(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = 'abc' - - class AcceptTuples(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = (1, 2, 3) - - When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above - does not have any data and so can't be run by itself, thus it does not - inherit from :class:`unittest.TestCase`. - - -.. seealso:: - - Test Driven Development - A book by Kent Beck on writing tests before code. - - -.. _regrtest: - -Running tests using the command-line interface ----------------------------------------------- - -The :mod:`test` package can be run as a script to drive Python's regression -test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m -test.regrtest` used in previous Python versions still works. Running the -script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose -name starts with ``test_``, importing them, and executing the function -:func:`test_main` if present or loading the tests via -unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The -names of tests to execute may also be passed to the script. Specifying a single -regression test (:program:`python -m test test_spam`) will minimize output and -only print whether the test passed or failed. - -Running :mod:`test` directly allows what resources are available for -tests to use to be set. You do this by using the ``-u`` command-line -option. Specifying ``all`` as the value for the ``-u`` option enables all -possible resources: :program:`python -m test -uall`. -If all but one resource is desired (a more common case), a -comma-separated list of resources that are not desired may be listed after -``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and -``largefile`` resources. For a list of all resources and more command-line -options, run :program:`python -m test -h`. - -Some other ways to execute the regression tests depend on what platform the -tests are being executed on. On Unix, you can run :program:`make test` at the -top-level directory where Python was built. On Windows, -executing :program:`rt.bat` from your :file:`PCBuild` directory will run all -regression tests. - - -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== - -.. module:: test.support - :synopsis: Support for Python's regression test suite. - - -The :mod:`test.support` module provides support for Python's regression -test suite. - -.. note:: - - :mod:`test.support` is not a public module. It is documented here to help - Python developers write tests. The API of this module is subject to change - without backwards compatibility concerns between releases. - - -This module defines the following exceptions: - -.. exception:: TestFailed - - Exception to be raised when a test fails. This is deprecated in favor of - :mod:`unittest`\ -based tests and :class:`unittest.TestCase`'s assertion - methods. - - -.. exception:: ResourceDenied - - Subclass of :exc:`unittest.SkipTest`. Raised when a resource (such as a - network connection) is not available. Raised by the :func:`requires` - function. - - -The :mod:`test.support` module defines the following constants: - -.. data:: verbose - - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. - - -.. data:: is_jython - - ``True`` if the running interpreter is Jython. - - -.. data:: TESTFN - - Set to a name that is safe to use as the name of a temporary file. Any - temporary file that is created should be closed and unlinked (removed). - - -The :mod:`test.support` module defines the following functions: - -.. function:: forget(module_name) - - Remove the module named *module_name* from ``sys.modules`` and delete any - byte-compiled files of the module. - - -.. function:: is_resource_enabled(resource) - - Return ``True`` if *resource* is enabled and available. The list of - available resources is only set when :mod:`test.regrtest` is executing the - tests. - - -.. function:: requires(resource, msg=None) - - Raise :exc:`ResourceDenied` if *resource* is not available. *msg* is the - argument to :exc:`ResourceDenied` if it is raised. Always returns - ``True`` if called by a function whose ``__name__`` is ``'__main__'``. - Used when tests are executed by :mod:`test.regrtest`. - - -.. function:: findfile(filename, subdir=None) - - Return the path to the file named *filename*. If no match is found - *filename* is returned. This does not equal a failure since it could be the - path to the file. - - Setting *subdir* indicates a relative path to use to find the file - rather than looking directly in the path directories. - - -.. function:: run_unittest(\*classes) - - Execute :class:`unittest.TestCase` subclasses passed to the function. The - function scans the classes for methods starting with the prefix ``test_`` - and executes the tests individually. - - It is also legal to pass strings as parameters; these should be keys in - ``sys.modules``. Each associated module will be scanned by - ``unittest.TestLoader.loadTestsFromModule()``. This is usually seen in the - following :func:`test_main` function:: - - def test_main(): - support.run_unittest(__name__) - - This will run all tests defined in the named module. - - -.. function:: run_doctest(module, verbosity=None) - - Run :func:`doctest.testmod` on the given *module*. Return - ``(failure_count, test_count)``. - - If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity - set to :data:`verbose`. Otherwise, it is run with verbosity set to - ``None``. - -.. function:: check_warnings(\*filters, quiet=True) - - A convenience wrapper for :func:`warnings.catch_warnings()` that makes it - easier to test that a warning was correctly raised. It is approximately - equivalent to calling ``warnings.catch_warnings(record=True)`` with - :meth:`warnings.simplefilter` set to ``always`` and with the option to - automatically validate the results that are recorded. - - ``check_warnings`` accepts 2-tuples of the form ``("message regexp", - WarningCategory)`` as positional arguments. If one or more *filters* are - provided, or if the optional keyword argument *quiet* is ``False``, - it checks to make sure the warnings are as expected: each specified filter - must match at least one of the warnings raised by the enclosed code or the - test fails, and if any warnings are raised that do not match any of the - specified filters the test fails. To disable the first of these checks, - set *quiet* to ``True``. - - If no arguments are specified, it defaults to:: - - check_warnings(("", Warning), quiet=True) - - In this case all warnings are caught and no errors are raised. - - On entry to the context manager, a :class:`WarningRecorder` instance is - returned. The underlying warnings list from - :func:`~warnings.catch_warnings` is available via the recorder object's - :attr:`warnings` attribute. As a convenience, the attributes of the object - representing the most recent warning can also be accessed directly through - the recorder object (see example below). If no warning has been raised, - then any of the attributes that would otherwise be expected on an object - representing a warning will return ``None``. - - The recorder object also has a :meth:`reset` method, which clears the - warnings list. - - The context manager is designed to be used like this:: - - with check_warnings(("assertion is always true", SyntaxWarning), - ("", UserWarning)): - exec('assert(False, "Hey!")') - warnings.warn(UserWarning("Hide me!")) - - In this case if either warning was not raised, or some other warning was - raised, :func:`check_warnings` would raise an error. - - When a test needs to look more deeply into the warnings, rather than - just checking whether or not they occurred, code like this can be used:: - - with check_warnings(quiet=True) as w: - warnings.warn("foo") - assert str(w.args[0]) == "foo" - warnings.warn("bar") - assert str(w.args[0]) == "bar" - assert str(w.warnings[0].args[0]) == "foo" - assert str(w.warnings[1].args[0]) == "bar" - w.reset() - assert len(w.warnings) == 0 - - - Here all warnings will be caught, and the test code tests the captured - warnings directly. - - .. versionchanged:: 3.2 - New optional arguments *filters* and *quiet*. - - -.. function:: captured_stdin() - captured_stdout() - captured_stderr() - - A context managers that temporarily replaces the named stream with - :class:`io.StringIO` object. - - Example use with output streams:: - - with captured_stdout() as stdout, captured_stderr() as stderr: - print("hello") - print("error", file=sys.stderr) - assert stdout.getvalue() == "hello\n" - assert stderr.getvalue() == "error\n" - - Example use with input stream:: - - with captured_stdin() as stdin: - stdin.write('hello\n') - stdin.seek(0) - # call test code that consumes from sys.stdin - captured = input() - self.assertEqual(captured, "hello") - - -.. function:: temp_dir(path=None, quiet=False) - - A context manager that creates a temporary directory at *path* and - yields the directory. - - If *path* is ``None``, the temporary directory is created using - :func:`tempfile.mkdtemp`. If *quiet* is ``False``, the context manager - raises an exception on error. Otherwise, if *path* is specified and - cannot be created, only a warning is issued. - - -.. function:: change_cwd(path, quiet=False) - - A context manager that temporarily changes the current working - directory to *path* and yields the directory. - - If *quiet* is ``False``, the context manager raises an exception - on error. Otherwise, it issues only a warning and keeps the current - working directory the same. - - -.. function:: temp_cwd(name='tempcwd', quiet=False) - - A context manager that temporarily creates a new directory and - changes the current working directory (CWD). - - The context manager creates a temporary directory in the current - directory with name *name* before temporarily changing the current - working directory. If *name* is ``None``, the temporary directory is - created using :func:`tempfile.mkdtemp`. - - If *quiet* is ``False`` and it is not possible to create or change - the CWD, an error is raised. Otherwise, only a warning is raised - and the original CWD is used. - - -.. function:: temp_umask(umask) - - A context manager that temporarily sets the process umask. - - -.. function:: can_symlink() - - Return ``True`` if the OS supports symbolic links, ``False`` - otherwise. - - -.. decorator:: skip_unless_symlink - - A decorator for running tests that require support for symbolic links. - - -.. decorator:: anticipate_failure(condition) - - A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should - have an associated comment identifying the relevant tracker issue. - - -.. decorator:: run_with_locale(catstr, *locales) - - A decorator for running a function in a different locale, correctly - resetting it after it has finished. *catstr* is the locale category as - a string (for example ``"LC_ALL"``). The *locales* passed will be tried - sequentially, and the first valid locale will be used. - - -.. function:: make_bad_fd() - - Create an invalid file descriptor by opening and closing a temporary file, - and returning its descriptor. - - -.. function:: import_module(name, deprecated=False) - - This function imports and returns the named module. Unlike a normal - import, this function raises :exc:`unittest.SkipTest` if the module - cannot be imported. - - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. - - .. versionadded:: 3.1 - - -.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) - - This function imports and returns a fresh copy of the named Python module - by removing the named module from ``sys.modules`` before doing the import. - Note that unlike :func:`reload`, the original module is not affected by - this operation. - - *fresh* is an iterable of additional module names that are also removed - from the ``sys.modules`` cache before doing the import. - - *blocked* is an iterable of module names that are replaced with ``None`` - in the module cache during the import to ensure that attempts to import - them raise :exc:`ImportError`. - - The named module and any modules named in the *fresh* and *blocked* - parameters are saved before starting the import and then reinserted into - ``sys.modules`` when the fresh import is complete. - - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. - - This function will raise :exc:`ImportError` if the named module cannot be - imported. - - Example use:: - - # Get copies of the warnings module for testing without affecting the - # version being used by the rest of the test suite. One copy uses the - # C implementation, the other is forced to use the pure Python fallback - # implementation - py_warnings = import_fresh_module('warnings', blocked=['_warnings']) - c_warnings = import_fresh_module('warnings', fresh=['_warnings']) - - .. versionadded:: 3.1 - - -.. function:: bind_port(sock, host=HOST) - - Bind the socket to a free port and return the port number. Relies on - ephemeral ports in order to ensure we are using an unbound port. This is - important as many tests may be running simultaneously, especially in a - buildbot environment. This method raises an exception if the - ``sock.family`` is :const:`~socket.AF_INET` and ``sock.type`` is - :const:`~socket.SOCK_STREAM`, and the socket has - :const:`~socket.SO_REUSEADDR` or :const:`~socket.SO_REUSEPORT` set on it. - Tests should never set these socket options for TCP/IP sockets. - The only case for setting these options is testing multicasting via - multiple UDP sockets. - - Additionally, if the :const:`~socket.SO_EXCLUSIVEADDRUSE` socket option is - available (i.e. on Windows), it will be set on the socket. This will - prevent anyone else from binding to our host/port for the duration of the - test. - - -.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) - - Returns an unused port that should be suitable for binding. This is - achieved by creating a temporary socket with the same family and type as - the ``sock`` parameter (default is :const:`~socket.AF_INET`, - :const:`~socket.SOCK_STREAM`), - and binding it to the specified host address (defaults to ``0.0.0.0``) - with the port set to 0, eliciting an unused ephemeral port from the OS. - The temporary socket is then closed and deleted, and the ephemeral port is - returned. - - Either this method or :func:`bind_port` should be used for any tests - where a server socket needs to be bound to a particular port for the - duration of the test. - Which one to use depends on whether the calling code is creating a python - socket, or if an unused port needs to be provided in a constructor - or passed to an external program (i.e. the ``-accept`` argument to - openssl's s_server mode). Always prefer :func:`bind_port` over - :func:`find_unused_port` where possible. Using a hard coded port is - discouraged since it can make multiple instances of the test impossible to - run simultaneously, which is a problem for buildbots. - - -.. function:: load_package_tests(pkg_dir, loader, standard_tests, pattern) - - Generic implementation of the :mod:`unittest` ``load_tests`` protocol for - use in test packages. *pkg_dir* is the root directory of the package; - *loader*, *standard_tests*, and *pattern* are the arguments expected by - ``load_tests``. In simple cases, the test package's ``__init__.py`` - can be the following:: - - import os - from test.support import load_package_tests - - def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) - - -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) - - Returns the set of attributes, functions or methods of *ref_api* not - found on *other_api*, except for a defined list of items to be - ignored in this check specified in *ignore*. - - By default this skips private attributes beginning with '_' but - includes all magic methods, i.e. those starting and ending in '__'. - - .. versionadded:: 3.5 - - -.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) - - Assert that the ``__all__`` variable of *module* contains all public names. - - The module's public names (its API) are detected automatically - based on whether they match the public name convention and were defined in - *module*. - - The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in in order to be detected as a public - API. One case for this is when *module* imports part of its public API from - other modules, possibly a C backend (like ``csv`` and its ``_csv``). - - The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` - attribute. If provided, it will be added to the automatically detected ones. - - The *blacklist* argument can be a set of names that must not be treated as part of - the public API even though their names indicate otherwise. - - Example use:: - - import bar - import foo - import unittest - from test import support - - class MiscTestCase(unittest.TestCase): - def test__all__(self): - support.check__all__(self, foo) - - class OtherTestCase(unittest.TestCase): - def test__all__(self): - extra = {'BAR_CONST', 'FOO_CONST'} - blacklist = {'baz'} # Undocumented name. - # bar imports part of its API from _bar. - support.check__all__(self, bar, ('bar', '_bar'), - extra=extra, blacklist=blacklist) - - .. versionadded:: 3.6 - - -The :mod:`test.support` module defines the following classes: - -.. class:: TransientResource(exc, **kwargs) - - Instances are a context manager that raises :exc:`ResourceDenied` if the - specified exception type is raised. Any keyword arguments are treated as - attribute/value pairs to be compared against any exception raised within the - :keyword:`with` statement. Only if all pairs match properly against - attributes on the exception is :exc:`ResourceDenied` raised. - - -.. class:: EnvironmentVarGuard() - - Class used to temporarily set or unset environment variables. Instances can - be used as a context manager and have a complete dictionary interface for - querying/modifying the underlying ``os.environ``. After exit from the - context manager all changes to environment variables done through this - instance will be rolled back. - - .. versionchanged:: 3.1 - Added dictionary interface. - -.. method:: EnvironmentVarGuard.set(envvar, value) - - Temporarily set the environment variable ``envvar`` to the value of - ``value``. - - -.. method:: EnvironmentVarGuard.unset(envvar) - - Temporarily unset the environment variable ``envvar``. - - -.. class:: SuppressCrashReport() - - A context manager used to try to prevent crash dialog popups on tests that - are expected to crash a subprocess. - - On Windows, it disables Windows Error Reporting dialogs using - `SetErrorMode `_. - - On UNIX, :func:`resource.setrlimit` is used to set - :attr:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file - creation. - - On both platforms, the old value is restored by :meth:`__exit__`. - - -.. class:: WarningsRecorder() - - Class used to record warnings for unit tests. See documentation of - :func:`check_warnings` above for more details. - - -.. class:: FakePath(path) - - Simple :term:`path-like object`. It implements the :meth:`__fspath__` - method which just returns the *path* argument. If *path* is an exception, - it will be raised in :meth:`!__fspath__`. diff --git a/third_party/python/Doc/library/text.rst b/third_party/python/Doc/library/text.rst deleted file mode 100644 index 47b678434..000000000 --- a/third_party/python/Doc/library/text.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _stringservices: -.. _textservices: - -************************ -Text Processing Services -************************ - -The modules described in this chapter provide a wide range of string -manipulation operations and other text processing services. - -The :mod:`codecs` module described under :ref:`binaryservices` is also -highly relevant to text processing. In addition, see the documentation for -Python's built-in string type in :ref:`textseq`. - - -.. toctree:: - - string.rst - re.rst - difflib.rst - textwrap.rst - unicodedata.rst - stringprep.rst - readline.rst - rlcompleter.rst - diff --git a/third_party/python/Doc/library/textwrap.rst b/third_party/python/Doc/library/textwrap.rst deleted file mode 100644 index d254466c9..000000000 --- a/third_party/python/Doc/library/textwrap.rst +++ /dev/null @@ -1,296 +0,0 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= - -.. module:: textwrap - :synopsis: Text wrapping and filling - -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - -**Source code:** :source:`Lib/textwrap.py` - --------------- - -The :mod:`textwrap` module provides some convenience functions, -as well as :class:`TextWrapper`, the class that does all the work. -If you're just wrapping or filling one or two text strings, the convenience -functions should be good enough; otherwise, you should use an instance of -:class:`TextWrapper` for efficiency. - -.. function:: wrap(text, width=70, **kwargs) - - Wraps the single paragraph in *text* (a string) so every line is at most - *width* characters long. Returns a list of output lines, without final - newlines. - - Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. *width* defaults to ``70``. - - See the :meth:`TextWrapper.wrap` method for additional details on how - :func:`wrap` behaves. - - -.. function:: fill(text, width=70, **kwargs) - - Wraps the single paragraph in *text*, and returns a single string containing the - wrapped paragraph. :func:`fill` is shorthand for :: - - "\n".join(wrap(text, ...)) - - In particular, :func:`fill` accepts exactly the same keyword arguments as - :func:`wrap`. - - -.. function:: shorten(text, width, **kwargs) - - Collapse and truncate the given *text* to fit in the given *width*. - - First the whitespace in *text* is collapsed (all whitespace is replaced by - single spaces). If the result fits in the *width*, it is returned. - Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`placeholder` fit within :attr:`width`:: - - >>> textwrap.shorten("Hello world!", width=12) - 'Hello world!' - >>> textwrap.shorten("Hello world!", width=11) - 'Hello [...]' - >>> textwrap.shorten("Hello world", width=10, placeholder="...") - 'Hello...' - - Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. Note that the whitespace is - collapsed before the text is passed to the :class:`TextWrapper` :meth:`fill` - function, so changing the value of :attr:`.tabsize`, :attr:`.expand_tabs`, - :attr:`.drop_whitespace`, and :attr:`.replace_whitespace` will have no effect. - - .. versionadded:: 3.4 - - -.. function:: dedent(text) - - Remove any common leading whitespace from every line in *text*. - - This can be used to make triple-quoted strings line up with the left edge of the - display, while still presenting them in the source code in indented form. - - Note that tabs and spaces are both treated as whitespace, but they are not - equal: the lines ``" hello"`` and ``"\thello"`` are considered to have no - common leading whitespace. - - For example:: - - def test(): - # end first line with \ to avoid the empty line! - s = '''\ - hello - world - ''' - print(repr(s)) # prints ' hello\n world\n ' - print(repr(dedent(s))) # prints 'hello\n world\n' - - -.. function:: indent(text, prefix, predicate=None) - - Add *prefix* to the beginning of selected lines in *text*. - - Lines are separated by calling ``text.splitlines(True)``. - - By default, *prefix* is added to all lines that do not consist - solely of whitespace (including any line endings). - - For example:: - - >>> s = 'hello\n\n \nworld' - >>> indent(s, ' ') - ' hello\n\n \n world' - - The optional *predicate* argument can be used to control which lines - are indented. For example, it is easy to add *prefix* to even empty - and whitespace-only lines:: - - >>> print(indent(s, '+ ', lambda line: True)) - + hello - + - + - + world - - .. versionadded:: 3.3 - - -:func:`wrap`, :func:`fill` and :func:`shorten` work by creating a -:class:`TextWrapper` instance and calling a single method on it. That -instance is not reused, so for applications that process many text -strings using :func:`wrap` and/or :func:`fill`, it may be more efficient to -create your own :class:`TextWrapper` object. - -Text is preferably wrapped on whitespaces and right after the hyphens in -hyphenated words; only then will long words be broken if necessary, unless -:attr:`TextWrapper.break_long_words` is set to false. - -.. class:: TextWrapper(**kwargs) - - The :class:`TextWrapper` constructor accepts a number of optional keyword - arguments. Each keyword argument corresponds to an instance attribute, so - for example :: - - wrapper = TextWrapper(initial_indent="* ") - - is the same as :: - - wrapper = TextWrapper() - wrapper.initial_indent = "* " - - You can re-use the same :class:`TextWrapper` object many times, and you can - change any of its options through direct assignment to instance attributes - between uses. - - The :class:`TextWrapper` instance attributes (and keyword arguments to the - constructor) are as follows: - - - .. attribute:: width - - (default: ``70``) The maximum length of wrapped lines. As long as there - are no individual words in the input text longer than :attr:`width`, - :class:`TextWrapper` guarantees that no output line will be longer than - :attr:`width` characters. - - - .. attribute:: expand_tabs - - (default: ``True``) If true, then all tab characters in *text* will be - expanded to spaces using the :meth:`expandtabs` method of *text*. - - - .. attribute:: tabsize - - (default: ``8``) If :attr:`expand_tabs` is true, then all tab characters - in *text* will be expanded to zero or more spaces, depending on the - current column and the given tab size. - - .. versionadded:: 3.3 - - - .. attribute:: replace_whitespace - - (default: ``True``) If true, after tab expansion but before wrapping, - the :meth:`wrap` method will replace each whitespace character - with a single space. The whitespace characters replaced are - as follows: tab, newline, vertical tab, formfeed, and carriage - return (``'\t\n\v\f\r'``). - - .. note:: - - If :attr:`expand_tabs` is false and :attr:`replace_whitespace` is true, - each tab character will be replaced by a single space, which is *not* - the same as tab expansion. - - .. note:: - - If :attr:`replace_whitespace` is false, newlines may appear in the - middle of a line and cause strange output. For this reason, text should - be split into paragraphs (using :meth:`str.splitlines` or similar) - which are wrapped separately. - - - .. attribute:: drop_whitespace - - (default: ``True``) If true, whitespace at the beginning and ending of - every line (after wrapping but before indenting) is dropped. - Whitespace at the beginning of the paragraph, however, is not dropped - if non-whitespace follows it. If whitespace being dropped takes up an - entire line, the whole line is dropped. - - - .. attribute:: initial_indent - - (default: ``''``) String that will be prepended to the first line of - wrapped output. Counts towards the length of the first line. The empty - string is not indented. - - - .. attribute:: subsequent_indent - - (default: ``''``) String that will be prepended to all lines of wrapped - output except the first. Counts towards the length of each line except - the first. - - - .. attribute:: fix_sentence_endings - - (default: ``False``) If true, :class:`TextWrapper` attempts to detect - sentence endings and ensure that sentences are always separated by exactly - two spaces. This is generally desired for text in a monospaced font. - However, the sentence detection algorithm is imperfect: it assumes that a - sentence ending consists of a lowercase letter followed by one of ``'.'``, - ``'!'``, or ``'?'``, possibly followed by one of ``'"'`` or ``"'"``, - followed by a space. One problem with this is algorithm is that it is - unable to detect the difference between "Dr." in :: - - [...] Dr. Frankenstein's monster [...] - - and "Spot." in :: - - [...] See Spot. See Spot run [...] - - :attr:`fix_sentence_endings` is false by default. - - Since the sentence detection algorithm relies on ``string.lowercase`` for - the definition of "lowercase letter," and a convention of using two spaces - after a period to separate sentences on the same line, it is specific to - English-language texts. - - - .. attribute:: break_long_words - - (default: ``True``) If true, then words longer than :attr:`width` will be - broken in order to ensure that no lines are longer than :attr:`width`. If - it is false, long words will not be broken, and some lines may be longer - than :attr:`width`. (Long words will be put on a line by themselves, in - order to minimize the amount by which :attr:`width` is exceeded.) - - - .. attribute:: break_on_hyphens - - (default: ``True``) If true, wrapping will occur preferably on whitespaces - and right after hyphens in compound words, as it is customary in English. - If false, only whitespaces will be considered as potentially good places - for line breaks, but you need to set :attr:`break_long_words` to false if - you want truly insecable words. Default behaviour in previous versions - was to always allow breaking hyphenated words. - - - .. attribute:: max_lines - - (default: ``None``) If not ``None``, then the output will contain at most - *max_lines* lines, with *placeholder* appearing at the end of the output. - - .. versionadded:: 3.4 - - - .. index:: single: ...; placeholder - - .. attribute:: placeholder - - (default: ``' [...]'``) String that will appear at the end of the output - text if it has been truncated. - - .. versionadded:: 3.4 - - - :class:`TextWrapper` also provides some public methods, analogous to the - module-level convenience functions: - - .. method:: wrap(text) - - Wraps the single paragraph in *text* (a string) so every line is at most - :attr:`width` characters long. All wrapping options are taken from - instance attributes of the :class:`TextWrapper` instance. Returns a list - of output lines, without final newlines. If the wrapped output has no - content, the returned list is empty. - - - .. method:: fill(text) - - Wraps the single paragraph in *text*, and returns a single string - containing the wrapped paragraph. diff --git a/third_party/python/Doc/library/threading.rst b/third_party/python/Doc/library/threading.rst deleted file mode 100644 index 063d2c066..000000000 --- a/third_party/python/Doc/library/threading.rst +++ /dev/null @@ -1,997 +0,0 @@ -:mod:`threading` --- Thread-based parallelism -============================================= - -.. module:: threading - :synopsis: Thread-based parallelism. - -**Source code:** :source:`Lib/threading.py` - --------------- - -This module constructs higher-level threading interfaces on top of the lower -level :mod:`_thread` module. See also the :mod:`queue` module. - -The :mod:`dummy_threading` module is provided for situations where -:mod:`threading` cannot be used because :mod:`_thread` is missing. - -.. note:: - - While they are not listed below, the ``camelCase`` names used for some - methods and functions in this module in the Python 2.x series are still - supported by this module. - - -This module defines the following functions: - - -.. function:: active_count() - - Return the number of :class:`Thread` objects currently alive. The returned - count is equal to the length of the list returned by :func:`.enumerate`. - - -.. function:: current_thread() - - Return the current :class:`Thread` object, corresponding to the caller's thread - of control. If the caller's thread of control was not created through the - :mod:`threading` module, a dummy thread object with limited functionality is - returned. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie - to be used e.g. to index a dictionary of thread-specific data. Thread - identifiers may be recycled when a thread exits and another thread is - created. - - .. versionadded:: 3.3 - - -.. function:: enumerate() - - Return a list of all :class:`Thread` objects currently alive. The list - includes daemonic threads, dummy thread objects created by - :func:`current_thread`, and the main thread. It excludes terminated threads - and threads that have not yet been started. - - -.. function:: main_thread() - - Return the main :class:`Thread` object. In normal conditions, the - main thread is the thread from which the Python interpreter was - started. - - .. versionadded:: 3.4 - - -.. function:: settrace(func) - - .. index:: single: trace function - - Set a trace function for all threads started from the :mod:`threading` module. - The *func* will be passed to :func:`sys.settrace` for each thread, before its - :meth:`~Thread.run` method is called. - - -.. function:: setprofile(func) - - .. index:: single: profile function - - Set a profile function for all threads started from the :mod:`threading` module. - The *func* will be passed to :func:`sys.setprofile` for each thread, before its - :meth:`~Thread.run` method is called. - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - Availability: Windows, systems with POSIX threads. - - -This module also defines the following constant: - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of blocking functions - (:meth:`Lock.acquire`, :meth:`RLock.acquire`, :meth:`Condition.wait`, etc.). - Specifying a timeout greater than this value will raise an - :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -This module defines a number of classes, which are detailed in the sections -below. - -The design of this module is loosely based on Java's threading model. However, -where Java makes locks and condition variables basic behavior of every object, -they are separate objects in Python. Python's :class:`Thread` class supports a -subset of the behavior of Java's Thread class; currently, there are no -priorities, no thread groups, and threads cannot be destroyed, stopped, -suspended, resumed, or interrupted. The static methods of Java's Thread class, -when implemented, are mapped to module-level functions. - -All of the methods described below are executed atomically. - - -Thread-Local Data ------------------ - -Thread-local data is data whose values are thread specific. To manage -thread-local data, just create an instance of :class:`local` (or a -subclass) and store attributes on it:: - - mydata = threading.local() - mydata.x = 1 - -The instance's values will be different for separate threads. - - -.. class:: local() - - A class that represents thread-local data. - - For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. - - -.. _thread-objects: - -Thread Objects --------------- - -The :class:`Thread` class represents an activity that is run in a separate -thread of control. There are two ways to specify the activity: by passing a -callable object to the constructor, or by overriding the :meth:`~Thread.run` -method in a subclass. No other methods (except for the constructor) should be -overridden in a subclass. In other words, *only* override the -:meth:`~Thread.__init__` and :meth:`~Thread.run` methods of this class. - -Once a thread object is created, its activity must be started by calling the -thread's :meth:`~Thread.start` method. This invokes the :meth:`~Thread.run` -method in a separate thread of control. - -Once the thread's activity is started, the thread is considered 'alive'. It -stops being alive when its :meth:`~Thread.run` method terminates -- either -normally, or by raising an unhandled exception. The :meth:`~Thread.is_alive` -method tests whether the thread is alive. - -Other threads can call a thread's :meth:`~Thread.join` method. This blocks -the calling thread until the thread whose :meth:`~Thread.join` method is -called is terminated. - -A thread has a name. The name can be passed to the constructor, and read or -changed through the :attr:`~Thread.name` attribute. - -A thread can be flagged as a "daemon thread". The significance of this flag is -that the entire Python program exits when only daemon threads are left. The -initial value is inherited from the creating thread. The flag can be set -through the :attr:`~Thread.daemon` property or the *daemon* constructor -argument. - -.. note:: - Daemon threads are abruptly stopped at shutdown. Their resources (such - as open files, database transactions, etc.) may not be released properly. - If you want your threads to stop gracefully, make them non-daemonic and - use a suitable signalling mechanism such as an :class:`Event`. - -There is a "main thread" object; this corresponds to the initial thread of -control in the Python program. It is not a daemon thread. - -There is the possibility that "dummy thread objects" are created. These are -thread objects corresponding to "alien threads", which are threads of control -started outside the threading module, such as directly from C code. Dummy -thread objects have limited functionality; they are always considered alive and -daemonic, and cannot be :meth:`~Thread.join`\ ed. They are never deleted, -since it is impossible to detect the termination of alien threads. - - -.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={}, *, \ - daemon=None) - - This constructor should always be called with keyword arguments. Arguments - are: - - *group* should be ``None``; reserved for future extension when a - :class:`ThreadGroup` class is implemented. - - *target* is the callable object to be invoked by the :meth:`run` method. - Defaults to ``None``, meaning nothing is called. - - *name* is the thread name. By default, a unique name is constructed of the - form "Thread-*N*" where *N* is a small decimal number. - - *args* is the argument tuple for the target invocation. Defaults to ``()``. - - *kwargs* is a dictionary of keyword arguments for the target invocation. - Defaults to ``{}``. - - If not ``None``, *daemon* explicitly sets whether the thread is daemonic. - If ``None`` (the default), the daemonic property is inherited from the - current thread. - - If the subclass overrides the constructor, it must make sure to invoke the - base class constructor (``Thread.__init__()``) before doing anything else to - the thread. - - .. versionchanged:: 3.3 - Added the *daemon* argument. - - .. method:: start() - - Start the thread's activity. - - It must be called at most once per thread object. It arranges for the - object's :meth:`~Thread.run` method to be invoked in a separate thread - of control. - - This method will raise a :exc:`RuntimeError` if called more than once - on the same thread object. - - .. method:: run() - - Method representing the thread's activity. - - You may override this method in a subclass. The standard :meth:`run` - method invokes the callable object passed to the object's constructor as - the *target* argument, if any, with sequential and keyword arguments taken - from the *args* and *kwargs* arguments, respectively. - - .. method:: join(timeout=None) - - Wait until the thread terminates. This blocks the calling thread until - the thread whose :meth:`~Thread.join` method is called terminates -- either - normally or through an unhandled exception -- or until the optional - timeout occurs. - - When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, - you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to - decide whether a timeout happened -- if the thread is still alive, the - :meth:`~Thread.join` call timed out. - - When the *timeout* argument is not present or ``None``, the operation will - block until the thread terminates. - - A thread can be :meth:`~Thread.join`\ ed many times. - - :meth:`~Thread.join` raises a :exc:`RuntimeError` if an attempt is made - to join the current thread as that would cause a deadlock. It is also - an error to :meth:`~Thread.join` a thread before it has been started - and attempts to do so raise the same exception. - - .. attribute:: name - - A string used for identification purposes only. It has no semantics. - Multiple threads may be given the same name. The initial name is set by - the constructor. - - .. method:: getName() - setName() - - Old getter/setter API for :attr:`~Thread.name`; use it directly as a - property instead. - - .. attribute:: ident - - The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the :func:`get_ident` - function. Thread identifiers may be recycled when a thread exits and - another thread is created. The identifier is available even after the - thread has exited. - - .. method:: is_alive() - - Return whether the thread is alive. - - This method returns ``True`` just before the :meth:`~Thread.run` method - starts until just after the :meth:`~Thread.run` method terminates. The - module function :func:`.enumerate` returns a list of all alive threads. - - .. attribute:: daemon - - A boolean value indicating whether this thread is a daemon thread (True) - or not (False). This must be set before :meth:`~Thread.start` is called, - otherwise :exc:`RuntimeError` is raised. Its initial value is inherited - from the creating thread; the main thread is not a daemon thread and - therefore all threads created in the main thread default to - :attr:`~Thread.daemon` = ``False``. - - The entire Python program exits when no alive non-daemon threads are left. - - .. method:: isDaemon() - setDaemon() - - Old getter/setter API for :attr:`~Thread.daemon`; use it directly as a - property instead. - - -.. impl-detail:: - - In CPython, due to the :term:`Global Interpreter Lock`, only one thread - can execute Python code at once (even though certain performance-oriented - libraries might overcome this limitation). - If you want your application to make better use of the computational - resources of multi-core machines, you are advised to use - :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. - However, threading is still an appropriate model if you want to run - multiple I/O-bound tasks simultaneously. - - -.. _lock-objects: - -Lock Objects ------------- - -A primitive lock is a synchronization primitive that is not owned by a -particular thread when locked. In Python, it is currently the lowest level -synchronization primitive available, implemented directly by the :mod:`_thread` -extension module. - -A primitive lock is in one of two states, "locked" or "unlocked". It is created -in the unlocked state. It has two basic methods, :meth:`~Lock.acquire` and -:meth:`~Lock.release`. When the state is unlocked, :meth:`~Lock.acquire` -changes the state to locked and returns immediately. When the state is locked, -:meth:`~Lock.acquire` blocks until a call to :meth:`~Lock.release` in another -thread changes it to unlocked, then the :meth:`~Lock.acquire` call resets it -to locked and returns. The :meth:`~Lock.release` method should only be -called in the locked state; it changes the state to unlocked and returns -immediately. If an attempt is made to release an unlocked lock, a -:exc:`RuntimeError` will be raised. - -Locks also support the :ref:`context management protocol `. - -When more than one thread is blocked in :meth:`~Lock.acquire` waiting for the -state to turn to unlocked, only one thread proceeds when a :meth:`~Lock.release` -call resets the state to unlocked; which one of the waiting threads proceeds -is not defined, and may vary across implementations. - -All methods are executed atomically. - - -.. class:: Lock() - - The class implementing primitive lock objects. Once a thread has acquired a - lock, subsequent attempts to acquire it block, until it is released; any - thread may release it. - - Note that ``Lock`` is actually a factory function which returns an instance - of the most efficient version of the concrete Lock class that is supported - by the platform. - - - .. method:: acquire(blocking=True, timeout=-1) - - Acquire a lock, blocking or non-blocking. - - When invoked with the *blocking* argument set to ``True`` (the default), - block until the lock is unlocked, then set it to locked and return ``True``. - - When invoked with the *blocking* argument set to ``False``, do not block. - If a call with *blocking* set to ``True`` would block, return ``False`` - immediately; otherwise, set the lock to locked and return ``True``. - - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` - specifies an unbounded wait. It is forbidden to specify a *timeout* - when *blocking* is false. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not (for example if the *timeout* expired). - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquisition can now be interrupted by signals on POSIX if the - underlying threading implementation supports it. - - - .. method:: release() - - Release a lock. This can be called from any thread, not only the thread - which has acquired the lock. - - When the lock is locked, reset it to unlocked, and return. If any other threads - are blocked waiting for the lock to become unlocked, allow exactly one of them - to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - -.. _rlock-objects: - -RLock Objects -------------- - -A reentrant lock is a synchronization primitive that may be acquired multiple -times by the same thread. Internally, it uses the concepts of "owning thread" -and "recursion level" in addition to the locked/unlocked state used by primitive -locks. In the locked state, some thread owns the lock; in the unlocked state, -no thread owns it. - -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. - -Reentrant locks also support the :ref:`context management protocol `. - - -.. class:: RLock() - - This class implements reentrant lock objects. A reentrant lock must be - released by the thread that acquired it. Once a thread has acquired a - reentrant lock, the same thread may acquire it again without blocking; the - thread must release it once for each time it has acquired it. - - Note that ``RLock`` is actually a factory function which returns an instance - of the most efficient version of the concrete RLock class that is supported - by the platform. - - - .. method:: acquire(blocking=True, timeout=-1) - - Acquire a lock, blocking or non-blocking. - - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. - - When invoked with the *blocking* argument set to true, do the same thing as when - called without arguments, and return true. - - When invoked with the *blocking* argument set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, do the - same thing as when called without arguments, and return true. - - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return true if the lock has - been acquired, false if the timeout has elapsed. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the decrement it is - zero, reset the lock to unlocked (not owned by any thread), and if any other - threads are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling thread. - - Only call this method when the calling thread owns the lock. A - :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. - - There is no return value. - - -.. _condition-objects: - -Condition Objects ------------------ - -A condition variable is always associated with some kind of lock; this can be -passed in or one will be created by default. Passing one in is useful when -several condition variables must share the same lock. The lock is part of -the condition object: you don't have to track it separately. - -A condition variable obeys the :ref:`context management protocol `: -using the ``with`` statement acquires the associated lock for the duration of -the enclosed block. The :meth:`~Condition.acquire` and -:meth:`~Condition.release` methods also call the corresponding methods of -the associated lock. - -Other methods must be called with the associated lock held. The -:meth:`~Condition.wait` method releases the lock, and then blocks until -another thread awakens it by calling :meth:`~Condition.notify` or -:meth:`~Condition.notify_all`. Once awakened, :meth:`~Condition.wait` -re-acquires the lock and returns. It is also possible to specify a timeout. - -The :meth:`~Condition.notify` method wakes up one of the threads waiting for -the condition variable, if any are waiting. The :meth:`~Condition.notify_all` -method wakes up all threads waiting for the condition variable. - -Note: the :meth:`~Condition.notify` and :meth:`~Condition.notify_all` methods -don't release the lock; this means that the thread or threads awakened will -not return from their :meth:`~Condition.wait` call immediately, but only when -the thread that called :meth:`~Condition.notify` or :meth:`~Condition.notify_all` -finally relinquishes ownership of the lock. - -The typical programming style using condition variables uses the lock to -synchronize access to some shared state; threads that are interested in a -particular change of state call :meth:`~Condition.wait` repeatedly until they -see the desired state, while threads that modify the state call -:meth:`~Condition.notify` or :meth:`~Condition.notify_all` when they change -the state in such a way that it could possibly be a desired state for one -of the waiters. For example, the following code is a generic -producer-consumer situation with unlimited buffer capacity:: - - # Consume one item - with cv: - while not an_item_is_available(): - cv.wait() - get_an_available_item() - - # Produce one item - with cv: - make_an_item_available() - cv.notify() - -The ``while`` loop checking for the application's condition is necessary -because :meth:`~Condition.wait` can return after an arbitrary long time, -and the condition which prompted the :meth:`~Condition.notify` call may -no longer hold true. This is inherent to multi-threaded programming. The -:meth:`~Condition.wait_for` method can be used to automate the condition -checking, and eases the computation of timeouts:: - - # Consume an item - with cv: - cv.wait_for(an_item_is_available) - get_an_available_item() - -To choose between :meth:`~Condition.notify` and :meth:`~Condition.notify_all`, -consider whether one state change can be interesting for only one or several -waiting threads. E.g. in a typical producer-consumer situation, adding one -item to the buffer only needs to wake up one consumer thread. - - -.. class:: Condition(lock=None) - - This class implements condition variable objects. A condition variable - allows one or more threads to wait until they are notified by another thread. - - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` - or :class:`RLock` object, and it is used as the underlying lock. Otherwise, - a new :class:`RLock` object is created and used as the underlying lock. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: acquire(*args) - - Acquire the underlying lock. This method calls the corresponding method on - the underlying lock; the return value is whatever that method returns. - - .. method:: release() - - Release the underlying lock. This method calls the corresponding method on - the underlying lock; there is no return value. - - .. method:: wait(timeout=None) - - Wait until notified or until a timeout occurs. If the calling thread has - not acquired the lock when this method is called, a :exc:`RuntimeError` is - raised. - - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another thread, or until the optional timeout - occurs. Once awakened or timed out, it re-acquires the lock and returns. - - When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - When the underlying lock is an :class:`RLock`, it is not released using - its :meth:`release` method, since this may not actually unlock the lock - when it was acquired multiple times recursively. Instead, an internal - interface of the :class:`RLock` class is used, which really unlocks it - even when it has been recursively acquired several times. Another internal - interface is then used to restore the recursion level when the lock is - reacquired. - - The return value is ``True`` unless a given *timeout* expired, in which - case it is ``False``. - - .. versionchanged:: 3.2 - Previously, the method always returned ``None``. - - .. method:: wait_for(predicate, timeout=None) - - Wait until a condition evaluates to true. *predicate* should be a - callable which result will be interpreted as a boolean value. - A *timeout* may be provided giving the maximum time to wait. - - This utility method may call :meth:`wait` repeatedly until the predicate - is satisfied, or until a timeout occurs. The return value is - the last return value of the predicate and will evaluate to - ``False`` if the method timed out. - - Ignoring the timeout feature, calling this method is roughly equivalent to - writing:: - - while not predicate(): - cv.wait() - - Therefore, the same rules apply as with :meth:`wait`: The lock must be - held when called and is re-acquired on return. The predicate is evaluated - with the lock held. - - .. versionadded:: 3.2 - - .. method:: notify(n=1) - - By default, wake up one thread waiting on this condition, if any. If the - calling thread has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - This method wakes up at most *n* of the threads waiting for the condition - variable; it is a no-op if no threads are waiting. - - The current implementation wakes up exactly *n* threads, if at least *n* - threads are waiting. However, it's not safe to rely on this behavior. - A future, optimized implementation may occasionally wake up more than - *n* threads. - - Note: an awakened thread does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. - - .. method:: notify_all() - - Wake up all threads waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - -.. _semaphore-objects: - -Semaphore Objects ------------------ - -This is one of the oldest synchronization primitives in the history of computer -science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he -used the names ``P()`` and ``V()`` instead of :meth:`~Semaphore.acquire` and -:meth:`~Semaphore.release`). - -A semaphore manages an internal counter which is decremented by each -:meth:`~Semaphore.acquire` call and incremented by each :meth:`~Semaphore.release` -call. The counter can never go below zero; when :meth:`~Semaphore.acquire` -finds that it is zero, it blocks, waiting until some other thread calls -:meth:`~Semaphore.release`. - -Semaphores also support the :ref:`context management protocol `. - - -.. class:: Semaphore(value=1) - - This class implements semaphore objects. A semaphore manages an atomic - counter representing the number of :meth:`release` calls minus the number of - :meth:`acquire` calls, plus an initial value. The :meth:`acquire` method - blocks if necessary until it can return without making the counter negative. - If not given, *value* defaults to 1. - - The optional argument gives the initial *value* for the internal counter; it - defaults to ``1``. If the *value* given is less than 0, :exc:`ValueError` is - raised. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: acquire(blocking=True, timeout=None) - - Acquire a semaphore. - - When invoked without arguments: - - * If the internal counter is larger than zero on entry, decrement it by - one and return true immediately. - * If the internal counter is zero on entry, block until awoken by a call to - :meth:`~Semaphore.release`. Once awoken (and the counter is greater - than 0), decrement the counter by 1 and return true. Exactly one - thread will be awoken by each call to :meth:`~Semaphore.release`. The - order in which threads are awoken should not be relied on. - - When invoked with *blocking* set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, do - the same thing as when called without arguments, and return true. - - When invoked with a *timeout* other than ``None``, it will block for at - most *timeout* seconds. If acquire does not complete successfully in - that interval, return false. Return true otherwise. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. method:: release() - - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another thread is waiting for it to become larger - than zero again, wake up that thread. - - -.. class:: BoundedSemaphore(value=1) - - Class implementing bounded semaphore objects. A bounded semaphore checks to - make sure its current value doesn't exceed its initial value. If it does, - :exc:`ValueError` is raised. In most situations semaphores are used to guard - resources with limited capacity. If the semaphore is released too many times - it's a sign of a bug. If not given, *value* defaults to 1. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - -.. _semaphore-examples: - -:class:`Semaphore` Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Semaphores are often used to guard resources with limited capacity, for example, -a database server. In any situation where the size of the resource is fixed, -you should use a bounded semaphore. Before spawning any worker threads, your -main thread would initialize the semaphore:: - - maxconnections = 5 - # ... - pool_sema = BoundedSemaphore(value=maxconnections) - -Once spawned, worker threads call the semaphore's acquire and release methods -when they need to connect to the server:: - - with pool_sema: - conn = connectdb() - try: - # ... use connection ... - finally: - conn.close() - -The use of a bounded semaphore reduces the chance that a programming error which -causes the semaphore to be released more than it's acquired will go undetected. - - -.. _event-objects: - -Event Objects -------------- - -This is one of the simplest mechanisms for communication between threads: one -thread signals an event and other threads wait for it. - -An event object manages an internal flag that can be set to true with the -:meth:`~Event.set` method and reset to false with the :meth:`~Event.clear` -method. The :meth:`~Event.wait` method blocks until the flag is true. - - -.. class:: Event() - - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`~Event.set` method and reset to false with the - :meth:`clear` method. The :meth:`wait` method blocks until the flag is true. - The flag is initially false. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: is_set() - - Return true if and only if the internal flag is true. - - .. method:: set() - - Set the internal flag to true. All threads waiting for it to become true - are awakened. Threads that call :meth:`wait` once the flag is true will - not block at all. - - .. method:: clear() - - Reset the internal flag to false. Subsequently, threads calling - :meth:`wait` will block until :meth:`.set` is called to set the internal - flag to true again. - - .. method:: wait(timeout=None) - - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. - - When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns true if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. - - .. versionchanged:: 3.1 - Previously, the method always returned ``None``. - - -.. _timer-objects: - -Timer Objects -------------- - -This class represents an action that should be run only after a certain amount -of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` -and as such also functions as an example of creating custom threads. - -Timers are started, as with threads, by calling their :meth:`~Timer.start` -method. The timer can be stopped (before its action has begun) by calling the -:meth:`~Timer.cancel` method. The interval the timer will wait before -executing its action may not be exactly the same as the interval specified by -the user. - -For example:: - - def hello(): - print("hello, world") - - t = Timer(30.0, hello) - t.start() # after 30 seconds, "hello, world" will be printed - - -.. class:: Timer(interval, function, args=None, kwargs=None) - - Create a timer that will run *function* with arguments *args* and keyword - arguments *kwargs*, after *interval* seconds have passed. - If *args* is ``None`` (the default) then an empty list will be used. - If *kwargs* is ``None`` (the default) then an empty dict will be used. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: cancel() - - Stop the timer, and cancel the execution of the timer's action. This will - only work if the timer is still in its waiting stage. - - -Barrier Objects ---------------- - -.. versionadded:: 3.2 - -This class provides a simple synchronization primitive for use by a fixed number -of threads that need to wait for each other. Each of the threads tries to pass -the barrier by calling the :meth:`~Barrier.wait` method and will block until -all of the threads have made their :meth:`~Barrier.wait` calls. At this point, -the threads are released simultaneously. - -The barrier can be reused any number of times for the same number of threads. - -As an example, here is a simple way to synchronize a client and server thread:: - - b = Barrier(2, timeout=5) - - def server(): - start_server() - b.wait() - while True: - connection = accept_connection() - process_server_connection(connection) - - def client(): - b.wait() - while True: - connection = make_connection() - process_client_connection(connection) - - -.. class:: Barrier(parties, action=None, timeout=None) - - Create a barrier object for *parties* number of threads. An *action*, when - provided, is a callable to be called by one of the threads when they are - released. *timeout* is the default timeout value if none is specified for - the :meth:`wait` method. - - .. method:: wait(timeout=None) - - Pass the barrier. When all the threads party to the barrier have called - this function, they are all released simultaneously. If a *timeout* is - provided, it is used in preference to any that was supplied to the class - constructor. - - The return value is an integer in the range 0 to *parties* -- 1, different - for each thread. This can be used to select a thread to do some special - housekeeping, e.g.:: - - i = barrier.wait() - if i == 0: - # Only one thread needs to print this - print("passed the barrier") - - If an *action* was provided to the constructor, one of the threads will - have called it prior to being released. Should this call raise an error, - the barrier is put into the broken state. - - If the call times out, the barrier is put into the broken state. - - This method may raise a :class:`BrokenBarrierError` exception if the - barrier is broken or reset while a thread is waiting. - - .. method:: reset() - - Return the barrier to the default, empty state. Any threads waiting on it - will receive the :class:`BrokenBarrierError` exception. - - Note that using this function may can require some external - synchronization if there are other threads whose state is unknown. If a - barrier is broken it may be better to just leave it and create a new one. - - .. method:: abort() - - Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. Use - this for example if one of the needs to abort, to avoid deadlocking the - application. - - It may be preferable to simply create the barrier with a sensible - *timeout* value to automatically guard against one of the threads going - awry. - - .. attribute:: parties - - The number of threads required to pass the barrier. - - .. attribute:: n_waiting - - The number of threads currently waiting in the barrier. - - .. attribute:: broken - - A boolean that is ``True`` if the barrier is in the broken state. - - -.. exception:: BrokenBarrierError - - This exception, a subclass of :exc:`RuntimeError`, is raised when the - :class:`Barrier` object is reset or broken. - - -.. _with-locks: - -Using locks, conditions, and semaphores in the :keyword:`with` statement ------------------------------------------------------------------------- - -All of the objects provided by this module that have :meth:`acquire` and -:meth:`release` methods can be used as context managers for a :keyword:`with` -statement. The :meth:`acquire` method will be called when the block is -entered, and :meth:`release` will be called when the block is exited. Hence, -the following snippet:: - - with some_lock: - # do something... - -is equivalent to:: - - some_lock.acquire() - try: - # do something... - finally: - some_lock.release() - -Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, -:class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as -:keyword:`with` statement context managers. diff --git a/third_party/python/Doc/library/time.rst b/third_party/python/Doc/library/time.rst deleted file mode 100644 index 808ae0544..000000000 --- a/third_party/python/Doc/library/time.rst +++ /dev/null @@ -1,761 +0,0 @@ -:mod:`time` --- Time access and conversions -=========================================== - -.. module:: time - :synopsis: Time access and conversions. - --------------- - -This module provides various time-related functions. For related -functionality, see also the :mod:`datetime` and :mod:`calendar` modules. - -Although this module is always available, -not all functions are available on all platforms. Most of the functions -defined in this module call platform C library functions with the same name. It -may sometimes be helpful to consult the platform documentation, because the -semantics of these functions varies among platforms. - -An explanation of some terminology and conventions is in order. - -.. _epoch: - -.. index:: single: epoch - -* The :dfn:`epoch` is the point where the time starts, and is platform - dependent. For Unix, the epoch is January 1, 1970, 00:00:00 (UTC). - To find out what the epoch is on a given platform, look at - ``time.gmtime(0)``. - -.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second - -.. index:: seconds since the epoch - -* The term :dfn:`seconds since the epoch` refers to the total number - of elapsed seconds since the epoch, typically excluding - `leap seconds`_. Leap seconds are excluded from this total on all - POSIX-compliant platforms. - -.. index:: single: Year 2038 - -* The functions in this module may not handle dates and times before the epoch or - far in the future. The cut-off point in the future is determined by the C - library; for 32-bit systems, it is typically in 2038. - -.. index:: - single: Year 2000 - single: Y2K - -.. _time-y2kissues: - -* **Year 2000 (Y2K) issues**: Python depends on the platform's C library, which - generally doesn't have year 2000 issues, since all dates and times are - represented internally as seconds since the epoch. Function :func:`strptime` - can parse 2-digit years when given ``%y`` format code. When 2-digit years are - parsed, they are converted according to the POSIX and ISO C standards: values - 69--99 are mapped to 1969--1999, and values 0--68 are mapped to 2000--2068. - -.. index:: - single: UTC - single: Coordinated Universal Time - single: Greenwich Mean Time - -* UTC is Coordinated Universal Time (formerly known as Greenwich Mean Time, or - GMT). The acronym UTC is not a mistake but a compromise between English and - French. - -.. index:: single: Daylight Saving Time - -* DST is Daylight Saving Time, an adjustment of the timezone by (usually) one - hour during part of the year. DST rules are magic (determined by local law) and - can change from year to year. The C library has a table containing the local - rules (often it is read from a system file for flexibility) and is the only - source of True Wisdom in this respect. - -* The precision of the various real-time functions may be less than suggested by - the units in which their value or argument is expressed. E.g. on most Unix - systems, the clock "ticks" only 50 or 100 times a second. - -* On the other hand, the precision of :func:`.time` and :func:`sleep` is better - than their Unix equivalents: times are expressed as floating point numbers, - :func:`.time` returns the most accurate time available (using Unix - :c:func:`gettimeofday` where available), and :func:`sleep` will accept a time - with a nonzero fraction (Unix :c:func:`select` is used to implement this, where - available). - -* The time value as returned by :func:`gmtime`, :func:`localtime`, and - :func:`strptime`, and accepted by :func:`asctime`, :func:`mktime` and - :func:`strftime`, is a sequence of 9 integers. The return values of - :func:`gmtime`, :func:`localtime`, and :func:`strptime` also offer attribute - names for individual fields. - - See :class:`struct_time` for a description of these objects. - - .. versionchanged:: 3.3 - The :class:`struct_time` type was extended to provide the :attr:`tm_gmtoff` - and :attr:`tm_zone` attributes when platform supports corresponding - ``struct tm`` members. - - .. versionchanged:: 3.6 - The :class:`struct_time` attributes :attr:`tm_gmtoff` and :attr:`tm_zone` - are now available on all platforms. - -* Use the following functions to convert between time representations: - - +-------------------------+-------------------------+-------------------------+ - | From | To | Use | - +=========================+=========================+=========================+ - | seconds since the epoch | :class:`struct_time` in | :func:`gmtime` | - | | UTC | | - +-------------------------+-------------------------+-------------------------+ - | seconds since the epoch | :class:`struct_time` in | :func:`localtime` | - | | local time | | - +-------------------------+-------------------------+-------------------------+ - | :class:`struct_time` in | seconds since the epoch | :func:`calendar.timegm` | - | UTC | | | - +-------------------------+-------------------------+-------------------------+ - | :class:`struct_time` in | seconds since the epoch | :func:`mktime` | - | local time | | | - +-------------------------+-------------------------+-------------------------+ - - -.. _time-functions: - -Functions ---------- - -.. function:: asctime([t]) - - Convert a tuple or :class:`struct_time` representing a time as returned by - :func:`gmtime` or :func:`localtime` to a string of the following - form: ``'Sun Jun 20 23:21:05 1993'``. If *t* is not provided, the current time - as returned by :func:`localtime` is used. Locale information is not used by - :func:`asctime`. - - .. note:: - - Unlike the C function of the same name, :func:`asctime` does not add a - trailing newline. - - -.. function:: clock() - - .. index:: - single: CPU time - single: processor time - single: benchmarking - - On Unix, return the current processor time as a floating point number expressed - in seconds. The precision, and in fact the very definition of the meaning of - "processor time", depends on that of the C function of the same name. - - On Windows, this function returns wall-clock seconds elapsed since the first - call to this function, as a floating point number, based on the Win32 function - :c:func:`QueryPerformanceCounter`. The resolution is typically better than one - microsecond. - - .. deprecated:: 3.3 - The behaviour of this function depends on the platform: use - :func:`perf_counter` or :func:`process_time` instead, depending on your - requirements, to have a well defined behaviour. - - -.. function:: clock_getres(clk_id) - - Return the resolution (precision) of the specified clock *clk_id*. Refer to - :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: clock_gettime(clk_id) - - Return the time of the specified clock *clk_id*. Refer to - :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: clock_settime(clk_id, time) - - Set the time of the specified clock *clk_id*. Currently, - :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: ctime([secs]) - - Convert a time expressed in seconds since the epoch to a string representing - local time. If *secs* is not provided or :const:`None`, the current time as - returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to - ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. - - -.. function:: get_clock_info(name) - - Get information on the specified clock as a namespace object. - Supported clock names and the corresponding functions to read their value - are: - - * ``'clock'``: :func:`time.clock` - * ``'monotonic'``: :func:`time.monotonic` - * ``'perf_counter'``: :func:`time.perf_counter` - * ``'process_time'``: :func:`time.process_time` - * ``'time'``: :func:`time.time` - - The result has the following attributes: - - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise - - *implementation*: The name of the underlying C function used to get - the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - - *monotonic*: ``True`` if the clock cannot go backward, - ``False`` otherwise - - *resolution*: The resolution of the clock in seconds (:class:`float`) - - .. versionadded:: 3.3 - - -.. function:: gmtime([secs]) - - Convert a time expressed in seconds since the epoch to a :class:`struct_time` in - UTC in which the dst flag is always zero. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. Fractions - of a second are ignored. See above for a description of the - :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this - function. - - -.. function:: localtime([secs]) - - Like :func:`gmtime` but converts to local time. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. The dst - flag is set to ``1`` when DST applies to the given time. - - -.. function:: mktime(t) - - This is the inverse function of :func:`localtime`. Its argument is the - :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` - as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. - If the input value cannot be represented as a valid time, either - :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on - whether the invalid value is caught by Python or the underlying C libraries). - The earliest date for which it can generate a time is platform-dependent. - - -.. function:: monotonic() - - Return the value (in fractional seconds) of a monotonic clock, i.e. a clock - that cannot go backwards. The clock is not affected by system clock updates. - The reference point of the returned value is undefined, so that only the - difference between the results of consecutive calls is valid. - - On Windows versions older than Vista, :func:`monotonic` detects - :c:func:`GetTickCount` integer overflow (32 bits, roll-over after 49.7 days). - It increases an internal epoch (reference time) by 2\ :sup:`32` each time - that an overflow is detected. The epoch is stored in the process-local state - and so the value of :func:`monotonic` may be different in two Python - processes running for more than 49 days. On more recent versions of Windows - and on other operating systems, :func:`monotonic` is system-wide. - - .. versionadded:: 3.3 - .. versionchanged:: 3.5 - The function is now always available. - - -.. function:: perf_counter() - - Return the value (in fractional seconds) of a performance counter, i.e. a - clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference - point of the returned value is undefined, so that only the difference between - the results of consecutive calls is valid. - - .. versionadded:: 3.3 - - -.. function:: process_time() - - Return the value (in fractional seconds) of the sum of the system and user - CPU time of the current process. It does not include time elapsed during - sleep. It is process-wide by definition. The reference point of the - returned value is undefined, so that only the difference between the results - of consecutive calls is valid. - - .. versionadded:: 3.3 - -.. function:: sleep(secs) - - Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep - time. The actual suspension time may be less than that requested because any - caught signal will terminate the :func:`sleep` following execution of that - signal's catching routine. Also, the suspension time may be longer than - requested by an arbitrary amount because of the scheduling of other activity - in the system. - - .. versionchanged:: 3.5 - The function now sleeps at least *secs* even if the sleep is interrupted - by a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale). - - -.. index:: - single: % (percent); datetime format - -.. function:: strftime(format[, t]) - - Convert a tuple or :class:`struct_time` representing a time as returned by - :func:`gmtime` or :func:`localtime` to a string as specified by the *format* - argument. If *t* is not provided, the current time as returned by - :func:`localtime` is used. *format* must be a string. :exc:`ValueError` is - raised if any field in *t* is outside of the allowed range. - - 0 is a legal argument for any position in the time tuple; if it is normally - illegal the value is forced to a correct one. - - The following directives can be embedded in the *format* string. They are shown - without the optional field width and precision specification, and are replaced - by the indicated characters in the :func:`strftime` result: - - +-----------+------------------------------------------------+-------+ - | Directive | Meaning | Notes | - +===========+================================================+=======+ - | ``%a`` | Locale's abbreviated weekday name. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%A`` | Locale's full weekday name. | | - +-----------+------------------------------------------------+-------+ - | ``%b`` | Locale's abbreviated month name. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%B`` | Locale's full month name. | | - +-----------+------------------------------------------------+-------+ - | ``%c`` | Locale's appropriate date and time | | - | | representation. | | - +-----------+------------------------------------------------+-------+ - | ``%d`` | Day of the month as a decimal number [01,31]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%H`` | Hour (24-hour clock) as a decimal number | | - | | [00,23]. | | - +-----------+------------------------------------------------+-------+ - | ``%I`` | Hour (12-hour clock) as a decimal number | | - | | [01,12]. | | - +-----------+------------------------------------------------+-------+ - | ``%j`` | Day of the year as a decimal number [001,366]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%m`` | Month as a decimal number [01,12]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%M`` | Minute as a decimal number [00,59]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | - | | day of the week) as a decimal number [00,53]. | | - | | All days in a new year preceding the first | | - | | Sunday are considered to be in week 0. | | - | | | | - | | | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | - | | day of the week) as a decimal number [00,53]. | | - | | All days in a new year preceding the first | | - | | Monday are considered to be in week 0. | | - | | | | - | | | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%x`` | Locale's appropriate date representation. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%X`` | Locale's appropriate time representation. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%y`` | Year without century as a decimal number | | - | | [00,99]. | | - +-----------+------------------------------------------------+-------+ - | ``%Y`` | Year with century as a decimal number. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%z`` | Time zone offset indicating a positive or | | - | | negative time difference from UTC/GMT of the | | - | | form +HHMM or -HHMM, where H represents decimal| | - | | hour digits and M represents decimal minute | | - | | digits [-23:59, +23:59]. | | - +-----------+------------------------------------------------+-------+ - | ``%Z`` | Time zone name (no characters if no time zone | | - | | exists). | | - +-----------+------------------------------------------------+-------+ - | ``%%`` | A literal ``'%'`` character. | | - +-----------+------------------------------------------------+-------+ - - Notes: - - (1) - When used with the :func:`strptime` function, the ``%p`` directive only affects - the output hour field if the ``%I`` directive is used to parse the hour. - - (2) - The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing `leap seconds`_ and value ``61`` is supported - for historical reasons. - - (3) - When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in - calculations when the day of the week and the year are specified. - - Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [#]_ :: - - >>> from time import gmtime, strftime - >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) - 'Thu, 28 Jun 2001 14:17:15 +0000' - - Additional directives may be supported on certain platforms, but only the - ones listed here have a meaning standardized by ANSI C. To see the full set - of format codes supported on your platform, consult the :manpage:`strftime(3)` - documentation. - - On some platforms, an optional field width and precision specification can - immediately follow the initial ``'%'`` of a directive in the following order; - this is also not portable. The field width is normally 2 except for ``%j`` where - it is 3. - - -.. index:: - single: % (percent); datetime format - -.. function:: strptime(string[, format]) - - Parse a string representing a time according to a format. The return value - is a :class:`struct_time` as returned by :func:`gmtime` or - :func:`localtime`. - - The *format* parameter uses the same directives as those used by - :func:`strftime`; it defaults to ``"%a %b %d %H:%M:%S %Y"`` which matches the - formatting returned by :func:`ctime`. If *string* cannot be parsed according - to *format*, or if it has excess data after parsing, :exc:`ValueError` is - raised. The default values used to fill in any missing data when more - accurate values cannot be inferred are ``(1900, 1, 1, 0, 0, 0, 0, 1, -1)``. - Both *string* and *format* must be strings. - - For example: - - >>> import time - >>> time.strptime("30 Nov 00", "%d %b %y") # doctest: +NORMALIZE_WHITESPACE - time.struct_time(tm_year=2000, tm_mon=11, tm_mday=30, tm_hour=0, tm_min=0, - tm_sec=0, tm_wday=3, tm_yday=335, tm_isdst=-1) - - Support for the ``%Z`` directive is based on the values contained in ``tzname`` - and whether ``daylight`` is true. Because of this, it is platform-specific - except for recognizing UTC and GMT which are always known (and are considered to - be non-daylight savings timezones). - - Only the directives specified in the documentation are supported. Because - ``strftime()`` is implemented per platform it can sometimes offer more - directives than those listed. But ``strptime()`` is independent of any platform - and thus does not necessarily support all directives available that are not - documented as supported. - - -.. class:: struct_time - - The type of the time value sequence returned by :func:`gmtime`, - :func:`localtime`, and :func:`strptime`. It is an object with a :term:`named - tuple` interface: values can be accessed by index and by attribute name. The - following values are present: - - +-------+-------------------+---------------------------------+ - | Index | Attribute | Values | - +=======+===================+=================================+ - | 0 | :attr:`tm_year` | (for example, 1993) | - +-------+-------------------+---------------------------------+ - | 1 | :attr:`tm_mon` | range [1, 12] | - +-------+-------------------+---------------------------------+ - | 2 | :attr:`tm_mday` | range [1, 31] | - +-------+-------------------+---------------------------------+ - | 3 | :attr:`tm_hour` | range [0, 23] | - +-------+-------------------+---------------------------------+ - | 4 | :attr:`tm_min` | range [0, 59] | - +-------+-------------------+---------------------------------+ - | 5 | :attr:`tm_sec` | range [0, 61]; see **(2)** in | - | | | :func:`strftime` description | - +-------+-------------------+---------------------------------+ - | 6 | :attr:`tm_wday` | range [0, 6], Monday is 0 | - +-------+-------------------+---------------------------------+ - | 7 | :attr:`tm_yday` | range [1, 366] | - +-------+-------------------+---------------------------------+ - | 8 | :attr:`tm_isdst` | 0, 1 or -1; see below | - +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_zone` | abbreviation of timezone name | - +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_gmtoff` | offset east of UTC in seconds | - +-------+-------------------+---------------------------------+ - - Note that unlike the C structure, the month value is a range of [1, 12], not - [0, 11]. - - In calls to :func:`mktime`, :attr:`tm_isdst` may be set to 1 when daylight - savings time is in effect, and 0 when it is not. A value of -1 indicates that - this is not known, and will usually result in the correct state being filled in. - - When a tuple with an incorrect length is passed to a function expecting a - :class:`struct_time`, or having elements of the wrong type, a - :exc:`TypeError` is raised. - -.. function:: time() - - Return the time in seconds since the epoch_ as a floating point - number. The specific date of the epoch and the handling of - `leap seconds`_ is platform dependent. - On Windows and most Unix systems, the epoch is January 1, 1970, - 00:00:00 (UTC) and leap seconds are not counted towards the time - in seconds since the epoch. This is commonly referred to as - `Unix time `_. - To find out what the epoch is on a given platform, look at - ``gmtime(0)``. - - Note that even though the time is always returned as a floating point - number, not all systems provide time with a better precision than 1 second. - While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back - between the two calls. - - The number returned by :func:`.time` may be converted into a more common - time format (i.e. year, month, day, hour, etc...) in UTC by passing it to - :func:`gmtime` function or in local time by passing it to the - :func:`localtime` function. In both cases a - :class:`struct_time` object is returned, from which the components - of the calendar date may be accessed as attributes. - - -.. function:: tzset() - - Resets the time conversion rules used by the library routines. The environment - variable :envvar:`TZ` specifies how this is done. - - Availability: Unix. - - .. note:: - - Although in many cases, changing the :envvar:`TZ` environment variable may - affect the output of functions like :func:`localtime` without calling - :func:`tzset`, this behavior should not be relied on. - - The :envvar:`TZ` environment variable should contain no whitespace. - - The standard format of the :envvar:`TZ` environment variable is (whitespace - added for clarity):: - - std offset [dst [offset [,start[/time], end[/time]]]] - - Where the components are: - - ``std`` and ``dst`` - Three or more alphanumerics giving the timezone abbreviations. These will be - propagated into time.tzname - - ``offset`` - The offset has the form: ``± hh[:mm[:ss]]``. This indicates the value - added the local time to arrive at UTC. If preceded by a '-', the timezone - is east of the Prime Meridian; otherwise, it is west. If no offset follows - dst, summer time is assumed to be one hour ahead of standard time. - - ``start[/time], end[/time]`` - Indicates when to change to and back from DST. The format of the - start and end dates are one of the following: - - :samp:`J{n}` - The Julian day *n* (1 <= *n* <= 365). Leap days are not counted, so in - all years February 28 is day 59 and March 1 is day 60. - - :samp:`{n}` - The zero-based Julian day (0 <= *n* <= 365). Leap days are counted, and - it is possible to refer to February 29. - - :samp:`M{m}.{n}.{d}` - The *d*'th day (0 <= *d* <= 6) of week *n* of month *m* of the year (1 - <= *n* <= 5, 1 <= *m* <= 12, where week 5 means "the last *d* day in - month *m*" which may occur in either the fourth or the fifth - week). Week 1 is the first week in which the *d*'th day occurs. Day - zero is a Sunday. - - ``time`` has the same format as ``offset`` except that no leading sign - ('-' or '+') is allowed. The default, if time is not given, is 02:00:00. - - :: - - >>> os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0' - >>> time.tzset() - >>> time.strftime('%X %x %Z') - '02:07:36 05/08/03 EDT' - >>> os.environ['TZ'] = 'AEST-10AEDT-11,M10.5.0,M3.5.0' - >>> time.tzset() - >>> time.strftime('%X %x %Z') - '16:08:12 05/08/03 AEST' - - On many Unix systems (including \*BSD, Linux, Solaris, and Darwin), it is more - convenient to use the system's zoneinfo (:manpage:`tzfile(5)`) database to - specify the timezone rules. To do this, set the :envvar:`TZ` environment - variable to the path of the required timezone datafile, relative to the root of - the systems 'zoneinfo' timezone database, usually located at - :file:`/usr/share/zoneinfo`. For example, ``'US/Eastern'``, - ``'Australia/Melbourne'``, ``'Egypt'`` or ``'Europe/Amsterdam'``. :: - - >>> os.environ['TZ'] = 'US/Eastern' - >>> time.tzset() - >>> time.tzname - ('EST', 'EDT') - >>> os.environ['TZ'] = 'Egypt' - >>> time.tzset() - >>> time.tzname - ('EET', 'EEST') - - -.. _time-clock-id-constants: - -Clock ID Constants ------------------- - -These constants are used as parameters for :func:`clock_getres` and -:func:`clock_gettime`. - -.. data:: CLOCK_HIGHRES - - The Solaris OS has a ``CLOCK_HIGHRES`` timer that attempts to use an optimal - hardware source, and may give close to nanosecond resolution. - ``CLOCK_HIGHRES`` is the nonadjustable, high-resolution clock. - - Availability: Solaris. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC - - Clock that cannot be set and represents monotonic time since some unspecified - starting point. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC_RAW - - Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw - hardware-based time that is not subject to NTP adjustments. - - Availability: Linux 2.6.28 or later. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_PROCESS_CPUTIME_ID - - High-resolution per-process timer from the CPU. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_THREAD_CPUTIME_ID - - Thread-specific CPU-time clock. - - Availability: Unix. - - .. versionadded:: 3.3 - - -The following constant is the only parameter that can be sent to -:func:`clock_settime`. - -.. data:: CLOCK_REALTIME - - System-wide real-time clock. Setting this clock requires appropriate - privileges. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. _time-timezone-constants: - -Timezone Constants -------------------- - -.. data:: altzone - - The offset of the local DST timezone, in seconds west of UTC, if one is defined. - This is negative if the local DST timezone is east of UTC (as in Western Europe, - including the UK). Only use this if ``daylight`` is nonzero. See note below. - -.. data:: daylight - - Nonzero if a DST timezone is defined. See note below. - -.. data:: timezone - - The offset of the local (non-DST) timezone, in seconds west of UTC (negative in - most of Western Europe, positive in the US, zero in the UK). See note below. - -.. data:: tzname - - A tuple of two strings: the first is the name of the local non-DST timezone, the - second is the name of the local DST timezone. If no DST timezone is defined, - the second string should not be used. See note below. - -.. note:: - - For the above Timezone constants (:data:`altzone`, :data:`daylight`, :data:`timezone`, - and :data:`tzname`), the value is determined by the timezone rules in effect - at module load time or the last time :func:`tzset` is called and may be incorrect - for times in the past. It is recommended to use the :attr:`tm_gmtoff` and - :attr:`tm_zone` results from :func:`localtime` to obtain timezone information. - - -.. seealso:: - - Module :mod:`datetime` - More object-oriented interface to dates and times. - - Module :mod:`locale` - Internationalization services. The locale setting affects the interpretation - of many format specifiers in :func:`strftime` and :func:`strptime`. - - Module :mod:`calendar` - General calendar-related functions. :func:`~calendar.timegm` is the - inverse of :func:`gmtime` from this module. - -.. rubric:: Footnotes - -.. [#] The use of ``%Z`` is now deprecated, but the ``%z`` escape that expands to the - preferred hour/minute offset is not supported by all ANSI C libraries. Also, a - strict reading of the original 1982 :rfc:`822` standard calls for a two-digit - year (%y rather than %Y), but practice moved to 4-digit years long before the - year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. - diff --git a/third_party/python/Doc/library/timeit.rst b/third_party/python/Doc/library/timeit.rst deleted file mode 100644 index c5efdff50..000000000 --- a/third_party/python/Doc/library/timeit.rst +++ /dev/null @@ -1,370 +0,0 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== - -.. module:: timeit - :synopsis: Measure the execution time of small code snippets. - -**Source code:** :source:`Lib/timeit.py` - -.. index:: - single: Benchmarking - single: Performance - --------------- - -This module provides a simple way to time small bits of Python code. It has both -a :ref:`timeit-command-line-interface` as well as a :ref:`callable ` -one. It avoids a number of common traps for measuring execution times. -See also Tim Peters' introduction to the "Algorithms" chapter in the *Python -Cookbook*, published by O'Reilly. - - -Basic Examples --------------- - -The following example shows how the :ref:`timeit-command-line-interface` -can be used to compare three different expressions: - -.. code-block:: shell-session - - $ python3 -m timeit '"-".join(str(n) for n in range(100))' - 10000 loops, best of 3: 30.2 usec per loop - $ python3 -m timeit '"-".join([str(n) for n in range(100)])' - 10000 loops, best of 3: 27.5 usec per loop - $ python3 -m timeit '"-".join(map(str, range(100)))' - 10000 loops, best of 3: 23.2 usec per loop - -This can be achieved from the :ref:`python-interface` with:: - - >>> import timeit - >>> timeit.timeit('"-".join(str(n) for n in range(100))', number=10000) - 0.3018611848820001 - >>> timeit.timeit('"-".join([str(n) for n in range(100)])', number=10000) - 0.2727368790656328 - >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) - 0.23702679807320237 - - -Note however that :mod:`timeit` will automatically determine the number of -repetitions only when the command-line interface is used. In the -:ref:`timeit-examples` section you can find more advanced examples. - - -.. _python-interface: - -Python Interface ----------------- - -The module defines three convenience functions and a public class: - - -.. function:: timeit(stmt='pass', setup='pass', timer=, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.timeit` method with *number* executions. - The optional *globals* argument specifies a namespace in which to execute the - code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - -.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=3, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.repeat` method with the given *repeat* - count and *number* executions. The optional *globals* argument specifies a - namespace in which to execute the code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - -.. function:: default_timer() - - The default timer, which is always :func:`time.perf_counter`. - - .. versionchanged:: 3.3 - :func:`time.perf_counter` is now the default timer. - - -.. class:: Timer(stmt='pass', setup='pass', timer=, globals=None) - - Class for timing execution speed of small code snippets. - - The constructor takes a statement to be timed, an additional statement used - for setup, and a timer function. Both statements default to ``'pass'``; - the timer function is platform-dependent (see the module doc string). - *stmt* and *setup* may also contain multiple statements separated by ``;`` - or newlines, as long as they don't contain multi-line string literals. The - statement will by default be executed within timeit's namespace; this behavior - can be controlled by passing a namespace to *globals*. - - To measure the execution time of the first statement, use the :meth:`.timeit` - method. The :meth:`.repeat` and :meth:`.autorange` methods are convenience - methods to call :meth:`.timeit` multiple times. - - The execution time of *setup* is excluded from the overall timed execution run. - - The *stmt* and *setup* parameters can also take objects that are callable - without arguments. This will embed calls to them in a timer function that - will then be executed by :meth:`.timeit`. Note that the timing overhead is a - little larger in this case because of the extra function calls. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - .. method:: Timer.timeit(number=1000000) - - Time *number* executions of the main statement. This executes the setup - statement once, and then returns the time it takes to execute the main - statement a number of times, measured in seconds as a float. - The argument is the number of times through the loop, defaulting to one - million. The main statement, the setup statement and the timer function - to be used are passed to the constructor. - - .. note:: - - By default, :meth:`.timeit` temporarily turns off :term:`garbage - collection` during the timing. The advantage of this approach is that - it makes independent timings more comparable. This disadvantage is - that GC may be an important component of the performance of the - function being measured. If so, GC can be re-enabled as the first - statement in the *setup* string. For example:: - - timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - - - .. method:: Timer.autorange(callback=None) - - Automatically determine how many times to call :meth:`.timeit`. - - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with *number* set to successive powers of ten (10, - 100, 1000, ...) up to a maximum of one billion, until the time taken - is at least 0.2 second, or the maximum is reached. - - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. - - .. versionadded:: 3.6 - - - .. method:: Timer.repeat(repeat=3, number=1000000) - - Call :meth:`.timeit` a few times. - - This is a convenience function that calls the :meth:`.timeit` repeatedly, - returning a list of results. The first argument specifies how many times - to call :meth:`.timeit`. The second argument specifies the *number* - argument for :meth:`.timeit`. - - .. note:: - - It's tempting to calculate mean and standard deviation from the result - vector and report these. However, this is not very useful. - In a typical case, the lowest value gives a lower bound for how fast - your machine can run the given code snippet; higher values in the - result vector are typically not caused by variability in Python's - speed, but by other processes interfering with your timing accuracy. - So the :func:`min` of the result is probably the only number you - should be interested in. After that, you should look at the entire - vector and apply common sense rather than statistics. - - - .. method:: Timer.print_exc(file=None) - - Helper to print a traceback from the timed code. - - Typical use:: - - t = Timer(...) # outside the try/except - try: - t.timeit(...) # or t.repeat(...) - except Exception: - t.print_exc() - - The advantage over the standard traceback is that source lines in the - compiled template will be displayed. The optional *file* argument directs - where the traceback is sent; it defaults to :data:`sys.stderr`. - - -.. _timeit-command-line-interface: - -Command-Line Interface ----------------------- - -When called as a program from the command line, the following form is used:: - - python -m timeit [-n N] [-r N] [-u U] [-s S] [-t] [-c] [-h] [statement ...] - -Where the following options are understood: - -.. program:: timeit - -.. cmdoption:: -n N, --number=N - - how many times to execute 'statement' - -.. cmdoption:: -r N, --repeat=N - - how many times to repeat the timer (default 3) - -.. cmdoption:: -s S, --setup=S - - statement to be executed once initially (default ``pass``) - -.. cmdoption:: -p, --process - - measure process time, not wallclock time, using :func:`time.process_time` - instead of :func:`time.perf_counter`, which is the default - - .. versionadded:: 3.3 - -.. cmdoption:: -t, --time - - use :func:`time.time` (deprecated) - -.. cmdoption:: -u, --unit=U - - specify a time unit for timer output; can select usec, msec, or sec - - .. versionadded:: 3.5 - -.. cmdoption:: -c, --clock - - use :func:`time.clock` (deprecated) - -.. cmdoption:: -v, --verbose - - print raw timing results; repeat for more digits precision - -.. cmdoption:: -h, --help - - print a short usage message and exit - -A multi-line statement may be given by specifying each line as a separate -statement argument; indented lines are possible by enclosing an argument in -quotes and using leading spaces. Multiple :option:`-s` options are treated -similarly. - -If :option:`-n` is not given, a suitable number of loops is calculated by trying -successive powers of 10 until the total time is at least 0.2 seconds. - -:func:`default_timer` measurements can be affected by other programs running on -the same machine, so the best thing to do when accurate timing is necessary is -to repeat the timing a few times and use the best time. The :option:`-r` -option is good for this; the default of 3 repetitions is probably enough in -most cases. You can use :func:`time.process_time` to measure CPU time. - -.. note:: - - There is a certain baseline overhead associated with executing a pass statement. - The code here doesn't try to hide it, but you should be aware of it. The - baseline overhead can be measured by invoking the program without arguments, - and it might differ between Python versions. - - -.. _timeit-examples: - -Examples --------- - -It is possible to provide a setup statement that is executed only once at the beginning: - -.. code-block:: shell-session - - $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' - 10000000 loops, best of 3: 0.0877 usec per loop - $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' - 1000000 loops, best of 3: 0.342 usec per loop - -:: - - >>> import timeit - >>> timeit.timeit('char in text', setup='text = "sample string"; char = "g"') - 0.41440500499993504 - >>> timeit.timeit('text.find(char)', setup='text = "sample string"; char = "g"') - 1.7246671520006203 - -The same can be done using the :class:`Timer` class and its methods:: - - >>> import timeit - >>> t = timeit.Timer('char in text', setup='text = "sample string"; char = "g"') - >>> t.timeit() - 0.3955516149999312 - >>> t.repeat() - [0.40193588800002544, 0.3960157959998014, 0.39594301399984033] - - -The following examples show how to time expressions that contain multiple lines. -Here we compare the cost of using :func:`hasattr` vs. :keyword:`try`/:keyword:`except` -to test for missing and present object attributes: - -.. code-block:: shell-session - - $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' - 100000 loops, best of 3: 15.7 usec per loop - $ python -m timeit 'if hasattr(str, "__bool__"): pass' - 100000 loops, best of 3: 4.26 usec per loop - - $ python -m timeit 'try:' ' int.__bool__' 'except AttributeError:' ' pass' - 1000000 loops, best of 3: 1.43 usec per loop - $ python -m timeit 'if hasattr(int, "__bool__"): pass' - 100000 loops, best of 3: 2.23 usec per loop - -:: - - >>> import timeit - >>> # attribute is missing - >>> s = """\ - ... try: - ... str.__bool__ - ... except AttributeError: - ... pass - ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.9138244460009446 - >>> s = "if hasattr(str, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.5829014980008651 - >>> - >>> # attribute is present - >>> s = """\ - ... try: - ... int.__bool__ - ... except AttributeError: - ... pass - ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.04215312199994514 - >>> s = "if hasattr(int, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.08588060699912603 - - -To give the :mod:`timeit` module access to functions you define, you can pass a -*setup* parameter which contains an import statement:: - - def test(): - """Stupid test function""" - L = [i for i in range(100)] - - if __name__ == '__main__': - import timeit - print(timeit.timeit("test()", setup="from __main__ import test")) - -Another option is to pass :func:`globals` to the *globals* parameter, which will cause the code -to be executed within your current global namespace. This can be more convenient -than individually specifying imports:: - - def f(x): - return x**2 - def g(x): - return x**4 - def h(x): - return x**8 - - import timeit - print(timeit.timeit('[func(42) for func in (f,g,h)]', globals=globals())) diff --git a/third_party/python/Doc/library/tk.rst b/third_party/python/Doc/library/tk.rst deleted file mode 100644 index 95cd1c771..000000000 --- a/third_party/python/Doc/library/tk.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. _tkinter: - -********************************* -Graphical User Interfaces with Tk -********************************* - -.. index:: - single: GUI - single: Graphical User Interface - single: Tkinter - single: Tk - -Tk/Tcl has long been an integral part of Python. It provides a robust and -platform independent windowing toolkit, that is available to Python programmers -using the :mod:`tkinter` package, and its extension, the :mod:`tkinter.tix` and -the :mod:`tkinter.ttk` modules. - -The :mod:`tkinter` package is a thin object-oriented layer on top of Tcl/Tk. To -use :mod:`tkinter`, you don't need to write Tcl code, but you will need to -consult the Tk documentation, and occasionally the Tcl documentation. -:mod:`tkinter` is a set of wrappers that implement the Tk widgets as Python -classes. In addition, the internal module :mod:`_tkinter` provides a threadsafe -mechanism which allows Python and Tcl to interact. - -:mod:`tkinter`'s chief virtues are that it is fast, and that it usually comes -bundled with Python. Although its standard documentation is weak, good -material is available, which includes: references, tutorials, a book and -others. :mod:`tkinter` is also famous for having an outdated look and feel, -which has been vastly improved in Tk 8.5. Nevertheless, there are many other -GUI libraries that you could be interested in. For more information about -alternatives, see the :ref:`other-gui-packages` section. - -.. toctree:: - - tkinter.rst - tkinter.ttk.rst - tkinter.tix.rst - tkinter.scrolledtext.rst - idle.rst - othergui.rst - -.. Other sections I have in mind are - Tkinter internals - Freezing Tkinter applications - - diff --git a/third_party/python/Doc/library/tkinter.rst b/third_party/python/Doc/library/tkinter.rst deleted file mode 100644 index 2515cf4f8..000000000 --- a/third_party/python/Doc/library/tkinter.rst +++ /dev/null @@ -1,863 +0,0 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= - -.. module:: tkinter - :synopsis: Interface to Tcl/Tk for graphical user interfaces - -.. moduleauthor:: Guido van Rossum - -**Source code:** :source:`Lib/tkinter/__init__.py` - --------------- - -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix -platforms, as well as on Windows systems. (Tk itself is not part of Python; it -is maintained at ActiveState.) - -Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is -properly installed on your system, and also showing what version of Tcl/Tk is -installed, so you can read the Tcl/Tk documentation specific to that version. - -.. seealso:: - - Tkinter documentation: - - `Python Tkinter Resources `_ - The Python Tkinter Topic Guide provides a great deal of information on using Tk - from Python and links to other sources of information on Tk. - - `TKDocs `_ - Extensive tutorial plus friendlier widget pages for some of the widgets. - - `Tkinter 8.5 reference: a GUI for Python `_ - On-line reference material. - - `Tkinter docs from effbot `_ - Online reference for tkinter supported by effbot.org. - - `Programming Python `_ - Book by Mark Lutz, has excellent coverage of Tkinter. - - `Modern Tkinter for Busy Python Developers `_ - Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. - - `Python and Tkinter Programming `_ - Book by John Grayson (ISBN 1-884777-81-3). - - Tcl/Tk documentation: - - `Tk commands `_ - Most commands are available as :mod:`tkinter` or :mod:`tkinter.ttk` classes. - Change '8.6' to match the version of your Tcl/Tk installation. - - `Tcl/Tk recent man pages `_ - Recent Tcl/Tk manuals on www.tcl.tk. - - `ActiveState Tcl Home Page `_ - The Tk/Tcl development is largely taking place at ActiveState. - - `Tcl and the Tk Toolkit `_ - Book by John Ousterhout, the inventor of Tcl. - - `Practical Programming in Tcl and Tk `_ - Brent Welch's encyclopedic book. - - -Tkinter Modules ---------------- - -Most of the time, :mod:`tkinter` is all you really need, but a number of -additional modules are available as well. The Tk interface is located in a -binary module named :mod:`_tkinter`. This module contains the low-level -interface to Tk, and should never be used directly by application programmers. -It is usually a shared library (or DLL), but might in some cases be statically -linked with the Python interpreter. - -In addition to the Tk interface module, :mod:`tkinter` includes a number of -Python modules, :mod:`tkinter.constants` being one of the most important. -Importing :mod:`tkinter` will automatically import :mod:`tkinter.constants`, -so, usually, to use Tkinter all you need is a simple import statement:: - - import tkinter - -Or, more often:: - - from tkinter import * - - -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=1) - - The :class:`Tk` class is instantiated without arguments. This creates a toplevel - widget of Tk which usually is the main window of an application. Each instance - has its own associated Tcl interpreter. - - .. FIXME: The following keyword arguments are currently recognized: - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=0) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - -Other modules that provide Tk support include: - -:mod:`tkinter.scrolledtext` - Text widget with a vertical scroll bar built in. - -:mod:`tkinter.colorchooser` - Dialog to let the user choose a color. - -:mod:`tkinter.commondialog` - Base class for the dialogs defined in the other modules listed here. - -:mod:`tkinter.filedialog` - Common dialogs to allow the user to specify a file to open or save. - -:mod:`tkinter.font` - Utilities to help work with fonts. - -:mod:`tkinter.messagebox` - Access to standard Tk dialog boxes. - -:mod:`tkinter.simpledialog` - Basic dialogs and convenience functions. - -:mod:`tkinter.dnd` - Drag-and-drop support for :mod:`tkinter`. This is experimental and should - become deprecated when it is replaced with the Tk DND. - -:mod:`turtle` - Turtle graphics in a Tk window. - - -Tkinter Life Preserver ----------------------- - -.. sectionauthor:: Matt Conway - - -This section is not designed to be an exhaustive tutorial on either Tk or -Tkinter. Rather, it is intended as a stop gap, providing some introductory -orientation on the system. - -Credits: - -* Tk was written by John Ousterhout while at Berkeley. - -* Tkinter was written by Steen Lumholt and Guido van Rossum. - -* This Life Preserver was written by Matt Conway at the University of Virginia. - -* The HTML rendering, and some liberal editing, was produced from a FrameMaker - version by Ken Manheimer. - -* Fredrik Lundh elaborated and revised the class interface descriptions, to get - them current with Tk 4.2. - -* Mike Clarkson converted the documentation to LaTeX, and compiled the User - Interface chapter of the reference manual. - - -How To Use This Section -^^^^^^^^^^^^^^^^^^^^^^^ - -This section is designed in two parts: the first half (roughly) covers -background material, while the second half can be taken to the keyboard as a -handy reference. - -When trying to answer questions of the form "how do I do blah", it is often best -to find out how to do "blah" in straight Tk, and then convert this back into the -corresponding :mod:`tkinter` call. Python programmers can often guess at the -correct Python command by looking at the Tk documentation. This means that in -order to use Tkinter, you will have to know a little bit about Tk. This document -can't fulfill that role, so the best we can do is point you to the best -documentation that exists. Here are some hints: - -* The authors strongly suggest getting a copy of the Tk man pages. - Specifically, the man pages in the ``manN`` directory are most useful. - The ``man3`` man pages describe the C interface to the Tk library and thus - are not especially helpful for script writers. - -* Addison-Wesley publishes a book called Tcl and the Tk Toolkit by John - Ousterhout (ISBN 0-201-63337-X) which is a good introduction to Tcl and Tk for - the novice. The book is not exhaustive, and for many details it defers to the - man pages. - -* :file:`tkinter/__init__.py` is a last resort for most, but can be a good - place to go when nothing else makes sense. - - -A Simple Hello World Program -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - import tkinter as tk - - class Application(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.master = master - self.pack() - self.create_widgets() - - def create_widgets(self): - self.hi_there = tk.Button(self) - self.hi_there["text"] = "Hello World\n(click me)" - self.hi_there["command"] = self.say_hi - self.hi_there.pack(side="top") - - self.quit = tk.Button(self, text="QUIT", fg="red", - command=self.master.destroy) - self.quit.pack(side="bottom") - - def say_hi(self): - print("hi there, everyone!") - - root = tk.Tk() - app = Application(master=root) - app.mainloop() - - -A (Very) Quick Look at Tcl/Tk ------------------------------ - -The class hierarchy looks complicated, but in actual practice, application -programmers almost always refer to the classes at the very bottom of the -hierarchy. - -Notes: - -* These classes are provided for the purposes of organizing certain functions - under one namespace. They aren't meant to be instantiated independently. - -* The :class:`Tk` class is meant to be instantiated only once in an application. - Application programmers need not instantiate one explicitly, the system creates - one whenever any of the other classes are instantiated. - -* The :class:`Widget` class is not meant to be instantiated, it is meant only - for subclassing to make "real" widgets (in C++, this is called an 'abstract - class'). - -To make use of this reference material, there will be times when you will need -to know how to read short passages of Tk and how to identify the various parts -of a Tk command. (See section :ref:`tkinter-basic-mapping` for the -:mod:`tkinter` equivalents of what's below.) - -Tk scripts are Tcl programs. Like all Tcl programs, Tk scripts are just lists -of tokens separated by spaces. A Tk widget is just its *class*, the *options* -that help configure it, and the *actions* that make it do useful things. - -To make a widget in Tk, the command is always of the form:: - - classCommand newPathname options - -*classCommand* - denotes which kind of widget to make (a button, a label, a menu...) - -.. index:: single: . (dot); in Tkinter - -*newPathname* - is the new name for this widget. All names in Tk must be unique. To help - enforce this, widgets in Tk are named with *pathnames*, just like files in a - file system. The top level widget, the *root*, is called ``.`` (period) and - children are delimited by more periods. For example, - ``.myApp.controlPanel.okButton`` might be the name of a widget. - -*options* - configure the widget's appearance and in some cases, its behavior. The options - come in the form of a list of flags and values. Flags are preceded by a '-', - like Unix shell command flags, and values are put in quotes if they are more - than one word. - -For example:: - - button .fred -fg red -text "hi there" - ^ ^ \______________________/ - | | | - class new options - command widget (-opt val -opt val ...) - -Once created, the pathname to the widget becomes a new command. This new -*widget command* is the programmer's handle for getting the new widget to -perform some *action*. In C, you'd express this as someAction(fred, -someOptions), in C++, you would express this as fred.someAction(someOptions), -and in Tk, you say:: - - .fred someAction someOptions - -Note that the object name, ``.fred``, starts with a dot. - -As you'd expect, the legal values for *someAction* will depend on the widget's -class: ``.fred disable`` works if fred is a button (fred gets greyed out), but -does not work if fred is a label (disabling of labels is not supported in Tk). - -The legal values of *someOptions* is action dependent. Some actions, like -``disable``, require no arguments, others, like a text-entry box's ``delete`` -command, would need arguments to specify what range of text to delete. - - -.. _tkinter-basic-mapping: - -Mapping Basic Tk into Tkinter ------------------------------ - -Class commands in Tk correspond to class constructors in Tkinter. :: - - button .fred =====> fred = Button() - -The master of an object is implicit in the new name given to it at creation -time. In Tkinter, masters are specified explicitly. :: - - button .panel.fred =====> fred = Button(panel) - -The configuration options in Tk are given in lists of hyphened tags followed by -values. In Tkinter, options are specified as keyword-arguments in the instance -constructor, and keyword-args for configure calls or as instance indices, in -dictionary style, for established instances. See section -:ref:`tkinter-setting-options` on setting options. :: - - button .fred -fg red =====> fred = Button(panel, fg="red") - .fred configure -fg red =====> fred["fg"] = red - OR ==> fred.config(fg="red") - -In Tk, to perform an action on a widget, use the widget name as a command, and -follow it with an action name, possibly with arguments (options). In Tkinter, -you call methods on the class instance to invoke actions on the widget. The -actions (methods) that a given widget can perform are listed in -:file:`tkinter/__init__.py`. :: - - .fred invoke =====> fred.invoke() - -To give a widget to the packer (geometry manager), you call pack with optional -arguments. In Tkinter, the Pack class holds all this functionality, and the -various forms of the pack command are implemented as methods. All widgets in -:mod:`tkinter` are subclassed from the Packer, and so inherit all the packing -methods. See the :mod:`tkinter.tix` module documentation for additional -information on the Form geometry manager. :: - - pack .fred -side left =====> fred.pack(side="left") - - -How Tk and Tkinter are Related ------------------------------- - -From the top down: - -Your App Here (Python) - A Python application makes a :mod:`tkinter` call. - -tkinter (Python Package) - This call (say, for example, creating a button widget), is implemented in - the :mod:`tkinter` package, which is written in Python. This Python - function will parse the commands and the arguments and convert them into a - form that makes them look as if they had come from a Tk script instead of - a Python script. - -_tkinter (C) - These commands and their arguments will be passed to a C function in the - :mod:`_tkinter` - note the underscore - extension module. - -Tk Widgets (C and Tcl) - This C function is able to make calls into other C modules, including the C - functions that make up the Tk library. Tk is implemented in C and some Tcl. - The Tcl part of the Tk widgets is used to bind certain default behaviors to - widgets, and is executed once at the point where the Python :mod:`tkinter` - package is imported. (The user never sees this stage). - -Tk (C) - The Tk part of the Tk Widgets implement the final mapping to ... - -Xlib (C) - the Xlib library to draw graphics on the screen. - - -Handy Reference ---------------- - - -.. _tkinter-setting-options: - -Setting Options -^^^^^^^^^^^^^^^ - -Options control things like the color and border width of a widget. Options can -be set in three ways: - -At object creation time, using keyword arguments - :: - - fred = Button(self, fg="red", bg="blue") - -After object creation, treating the option name like a dictionary index - :: - - fred["fg"] = "red" - fred["bg"] = "blue" - -Use the config() method to update multiple attrs subsequent to object creation - :: - - fred.config(fg="red", bg="blue") - -For a complete explanation of a given option and its behavior, see the Tk man -pages for the widget in question. - -Note that the man pages list "STANDARD OPTIONS" and "WIDGET SPECIFIC OPTIONS" -for each widget. The former is a list of options that are common to many -widgets, the latter are the options that are idiosyncratic to that particular -widget. The Standard Options are documented on the :manpage:`options(3)` man -page. - -No distinction between standard and widget-specific options is made in this -document. Some options don't apply to some kinds of widgets. Whether a given -widget responds to a particular option depends on the class of the widget; -buttons have a ``command`` option, labels do not. - -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. - -Some options, like ``bg`` are synonyms for common options with long names -(``bg`` is shorthand for "background"). Passing the ``config()`` method the name -of a shorthand option will return a 2-tuple, not 5-tuple. The 2-tuple passed -back will contain the name of the synonym and the "real" option (such as -``('bg', 'background')``). - -+-------+---------------------------------+--------------+ -| Index | Meaning | Example | -+=======+=================================+==============+ -| 0 | option name | ``'relief'`` | -+-------+---------------------------------+--------------+ -| 1 | option name for database lookup | ``'relief'`` | -+-------+---------------------------------+--------------+ -| 2 | option class for database | ``'Relief'`` | -| | lookup | | -+-------+---------------------------------+--------------+ -| 3 | default value | ``'raised'`` | -+-------+---------------------------------+--------------+ -| 4 | current value | ``'groove'`` | -+-------+---------------------------------+--------------+ - -Example:: - - >>> print(fred.config()) - {'relief': ('relief', 'relief', 'Relief', 'raised', 'groove')} - -Of course, the dictionary printed will include all the options available and -their values. This is meant only as an example. - - -The Packer -^^^^^^^^^^ - -.. index:: single: packing (widgets) - -The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of the positioning of widgets -within their container - their mutual *master*. In contrast to the more -cumbersome *placer* (which is used less commonly, and we do not cover here), the -packer takes qualitative relationship specification - *above*, *to the left of*, -*filling*, etc - and works everything out to determine the exact placement -coordinates for you. - -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames -into other frames, in order to achieve the kind of layout you desire. -Additionally, the arrangement is dynamically adjusted to accommodate incremental -changes to the configuration, once it is packed. - -Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. - -The pack() method can be called with keyword-option/value pairs that control -where the widget is to appear within its container, and how it is to behave when -the main application window is resized. Here are some examples:: - - fred.pack() # defaults to side = "top" - fred.pack(side="left") - fred.pack(expand=1) - - -Packer Options -^^^^^^^^^^^^^^ - -For more extensive information on the packer and the options that it can take, -see the man pages and page 183 of John Ousterhout's book. - -anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. - -expand - Boolean, ``0`` or ``1``. - -fill - Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. - -ipadx and ipady - A distance - designating internal padding on each side of the slave widget. - -padx and pady - A distance - designating external padding on each side of the slave widget. - -side - Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. - - -Coupling Widget Variables -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The current-value setting of some widgets (like text entry widgets) can be -connected directly to application variables by using special options. These -options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and -``value``. This connection works both ways: if the variable changes for any -reason, the widget it's connected to will be updated to reflect the new value. - -Unfortunately, in the current implementation of :mod:`tkinter` it is not -possible to hand over an arbitrary Python variable to a widget through a -``variable`` or ``textvariable`` option. The only kinds of variables for which -this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. - -There are many useful subclasses of Variable already defined: -:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. - -For example:: - - class App(Frame): - def __init__(self, master=None): - super().__init__(master) - self.pack() - - self.entrythingy = Entry() - self.entrythingy.pack() - - # here is the application variable - self.contents = StringVar() - # set it to some value - self.contents.set("this is a variable") - # tell the entry widget to watch this variable - self.entrythingy["textvariable"] = self.contents - - # and here we get a callback when the user hits return. - # we will have the program print out the value of the - # application variable when the user hits return - self.entrythingy.bind('', - self.print_contents) - - def print_contents(self, event): - print("hi. contents of entry is now ---->", - self.contents.get()) - - -The Window Manager -^^^^^^^^^^^^^^^^^^ - -.. index:: single: window manager (widgets) - -In Tk, there is a utility command, ``wm``, for interacting with the window -manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have -been implemented as methods on the :class:`Wm` class. Toplevel widgets are -subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods -directly. - -To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. - -Here are some examples of typical usage:: - - import tkinter as tk - - class App(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.pack() - - # create the application - myapp = App() - - # - # here are method calls to the window manager class - # - myapp.master.title("My Do-Nothing Application") - myapp.master.maxsize(1000, 400) - - # start the program - myapp.mainloop() - - -Tk Option Data Types -^^^^^^^^^^^^^^^^^^^^ - -.. index:: single: Tk Option Data Types - -anchor - Legal values are points of the compass: ``"n"``, ``"ne"``, ``"e"``, ``"se"``, - ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. - -bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. - -boolean - You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. - -callback - This is any Python function that takes no arguments. For example:: - - def print_it(): - print("hi there") - fred["command"] = print_it - -color - Colors can be given as the names of X colors in the rgb.txt file, or as strings - representing RGB values in 4 bit: ``"#RGB"``, 8 bit: ``"#RRGGBB"``, 12 bit" - ``"#RRRGGGBBB"``, or 16 bit ``"#RRRRGGGGBBBB"`` ranges, where R,G,B here - represent any legal hex digit. See page 160 of Ousterhout's book for details. - -cursor - The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the - string ``"hand2"``. You can also specify a bitmap and mask file of your own. - See page 179 of Ousterhout's book. - -distance - Screen distances can be specified in either pixels or absolute distances. - Pixels are given as numbers and absolute distances as strings, with the trailing - character denoting units: ``c`` for centimetres, ``i`` for inches, ``m`` for - millimetres, ``p`` for printer's points. For example, 3.5 inches is expressed - as ``"3.5i"``. - -font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. - -geometry - This is a string of the form ``widthxheight``, where width and height are - measured in pixels for most widgets (in characters for widgets displaying text). - For example: ``fred["geometry"] = "200x100"``. - -justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. - -region - This is a string with four space-delimited elements, each of which is a legal - distance (see above). For example: ``"2 3 4 5"`` and ``"3i 2i 4.5i 2i"`` and - ``"3c 2c 4c 10.43c"`` are all legal regions. - -relief - Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. - -scrollcommand - This is almost always the :meth:`!set` method of some scrollbar widget, but can - be any widget method that takes a single argument. - -wrap: - Must be one of: ``"none"``, ``"char"``, or ``"word"``. - - -Bindings and Events -^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: bind (widgets) - single: events (widgets) - -The bind method from the widget command allows you to watch for certain events -and to have a callback function trigger when that event type occurs. The form -of the bind method is:: - - def bind(self, sequence, func, add=''): - -where: - -sequence - is a string that denotes the target kind of event. (See the bind man page and - page 201 of John Ousterhout's book for details). - -func - is a Python function, taking one argument, to be invoked when the event occurs. - An Event instance will be passed as the argument. (Functions deployed this way - are commonly known as *callbacks*.) - -add - is optional, either ``''`` or ``'+'``. Passing an empty string denotes that - this binding is to replace any other bindings that this event is associated - with. Passing a ``'+'`` means that this function is to be added to the list - of functions bound to this event type. - -For example:: - - def turn_red(self, event): - event.widget["activeforeground"] = "red" - - self.button.bind("", self.turn_red) - -Notice how the widget field of the event is being accessed in the -``turn_red()`` callback. This field contains the widget that caught the X -event. The following table lists the other event fields you can access, and how -they are denoted in Tk, which can be useful when referring to the Tk man pages. - -+----+---------------------+----+---------------------+ -| Tk | Tkinter Event Field | Tk | Tkinter Event Field | -+====+=====================+====+=====================+ -| %f | focus | %A | char | -+----+---------------------+----+---------------------+ -| %h | height | %E | send_event | -+----+---------------------+----+---------------------+ -| %k | keycode | %K | keysym | -+----+---------------------+----+---------------------+ -| %s | state | %N | keysym_num | -+----+---------------------+----+---------------------+ -| %t | time | %T | type | -+----+---------------------+----+---------------------+ -| %w | width | %W | widget | -+----+---------------------+----+---------------------+ -| %x | x | %X | x_root | -+----+---------------------+----+---------------------+ -| %y | y | %Y | y_root | -+----+---------------------+----+---------------------+ - - -The index Parameter -^^^^^^^^^^^^^^^^^^^ - -A number of widgets require "index" parameters to be passed. These are used to -point at a specific place in a Text widget, or to particular characters in an -Entry widget, or to particular menu items in a Menu widget. - -Entry widget indexes (index, view index, etc.) - Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special - points in text widgets: - -Text widget indexes - The index notation for Text widgets is very rich and is best described in the Tk - man pages. - -Menu indexes (menu.invoke(), menu.entryconfig(), etc.) - Some options and methods for menus manipulate specific menu entries. Anytime a - menu index is needed for an option or a parameter, you may pass in: - - * an integer which refers to the numeric position of the entry in the widget, - counted from the top, starting with 0; - - * the string ``"active"``, which refers to the menu position that is currently - under the cursor; - - * the string ``"last"`` which refers to the last menu item; - - * An integer preceded by ``@``, as in ``@6``, where the integer is interpreted - as a y pixel coordinate in the menu's coordinate system; - - * the string ``"none"``, which indicates no menu entry at all, most often used - with menu.activate() to deactivate all entries, and finally, - - * a text string that is pattern matched against the label of the menu entry, as - scanned from the top of the menu to the bottom. Note that this index type is - considered after all the others, which means that matches for menu items - labelled ``last``, ``active``, or ``none`` may be interpreted as the above - literals, instead. - - -Images -^^^^^^ - -Images of different formats can be created through the corresponding subclass -of :class:`tkinter.Image`: - -* :class:`BitmapImage` for images in XBM format. - -* :class:`PhotoImage` for images in PGM, PPM, GIF and PNG formats. The latter - is supported starting with Tk 8.6. - -Either type of image is created through either the ``file`` or the ``data`` -option (other options are available as well). - -The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a -reference to the image. When the last Python reference to the image object is -deleted, the image data is deleted as well, and Tk will display an empty box -wherever the image was used. - -.. seealso:: - - The `Pillow `_ package adds support for - formats such as BMP, JPEG, TIFF, and WebP, among others. - -.. _tkinter-file-handlers: - -File Handlers -------------- - -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: - - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) - -This feature is not available on Windows. - -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. - - -.. method:: Widget.tk.createfilehandler(file, mask, func) - - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: - - callback(file, mask) - - -.. method:: Widget.tk.deletefilehandler(file) - - Unregisters a file handler. - - -.. data:: READABLE - WRITABLE - EXCEPTION - - Constants used in the *mask* arguments. diff --git a/third_party/python/Doc/library/tkinter.scrolledtext.rst b/third_party/python/Doc/library/tkinter.scrolledtext.rst deleted file mode 100644 index 138720e47..000000000 --- a/third_party/python/Doc/library/tkinter.scrolledtext.rst +++ /dev/null @@ -1,36 +0,0 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== - -.. module:: tkinter.scrolledtext - :platform: Tk - :synopsis: Text widget with a vertical scroll bar. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/tkinter/scrolledtext.py` - --------------- - -The :mod:`tkinter.scrolledtext` module provides a class of the same name which -implements a basic text widget which has a vertical scroll bar configured to do -the "right thing." Using the :class:`ScrolledText` class is a lot easier than -setting up a text widget and scroll bar directly. The constructor is the same -as that of the :class:`tkinter.Text` class. - -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. - -Should more specific control be necessary, the following attributes are -available: - - -.. attribute:: ScrolledText.frame - - The frame which surrounds the text and scroll bar widgets. - - -.. attribute:: ScrolledText.vbar - - The scroll bar widget. diff --git a/third_party/python/Doc/library/tkinter.tix.rst b/third_party/python/Doc/library/tkinter.tix.rst deleted file mode 100644 index 11ed75513..000000000 --- a/third_party/python/Doc/library/tkinter.tix.rst +++ /dev/null @@ -1,594 +0,0 @@ -:mod:`tkinter.tix` --- Extension widgets for Tk -=============================================== - -.. module:: tkinter.tix - :synopsis: Tk Extension Widgets for Tkinter - -.. sectionauthor:: Mike Clarkson - -**Source code:** :source:`Lib/tkinter/tix.py` - -.. index:: single: Tix - -.. deprecated:: 3.6 - This Tk extension is unmaintained and should not be used in new code. Use - :mod:`tkinter.ttk` instead. - --------------- - -The :mod:`tkinter.tix` (Tk Interface Extension) module provides an additional -rich set of widgets. Although the standard Tk library has many useful widgets, -they are far from complete. The :mod:`tkinter.tix` library provides most of the -commonly needed widgets that are missing from standard Tk: :class:`HList`, -:class:`ComboBox`, :class:`Control` (a.k.a. SpinBox) and an assortment of -scrollable widgets. -:mod:`tkinter.tix` also includes many more widgets that are generally useful in -a wide range of applications: :class:`NoteBook`, :class:`FileEntry`, -:class:`PanedWindow`, etc; there are more than 40 of them. - -With all these new widgets, you can introduce new interaction techniques into -applications, creating more useful and more intuitive user interfaces. You can -design your application by choosing the most appropriate widgets to match the -special needs of your application and users. - -.. seealso:: - - `Tix Homepage `_ - The home page for :mod:`Tix`. This includes links to additional documentation - and downloads. - - `Tix Man Pages `_ - On-line version of the man pages and reference material. - - `Tix Programming Guide `_ - On-line version of the programmer's reference material. - - `Tix Development Applications `_ - Tix applications for development of Tix and Tkinter programs. Tide applications - work under Tk or Tkinter, and include :program:`TixInspect`, an inspector to - remotely modify and debug Tix/Tk/Tkinter applications. - - -Using Tix ---------- - - -.. class:: Tk(screenName=None, baseName=None, className='Tix') - - Toplevel widget of Tix which represents mostly the main window of an - application. It has an associated Tcl interpreter. - - Classes in the :mod:`tkinter.tix` module subclasses the classes in the - :mod:`tkinter`. The former imports the latter, so to use :mod:`tkinter.tix` - with Tkinter, all you need to do is to import one module. In general, you - can just import :mod:`tkinter.tix`, and replace the toplevel call to - :class:`tkinter.Tk` with :class:`tix.Tk`:: - - from tkinter import tix - from tkinter.constants import * - root = tix.Tk() - -To use :mod:`tkinter.tix`, you must have the Tix widgets installed, usually -alongside your installation of the Tk widgets. To test your installation, try -the following:: - - from tkinter import tix - root = tix.Tk() - root.tk.eval('package require Tix') - -If this fails, you have a Tk installation problem which must be resolved before -proceeding. Use the environment variable :envvar:`TIX_LIBRARY` to point to the -installed Tix library directory, and make sure you have the dynamic -object library (:file:`tix8183.dll` or :file:`libtix8183.so`) in the same -directory that contains your Tk dynamic object library (:file:`tk8183.dll` or -:file:`libtk8183.so`). The directory with the dynamic object library should also -have a file called :file:`pkgIndex.tcl` (case sensitive), which contains the -line:: - - package ifneeded Tix 8.1 [list load "[file join $dir tix8183.dll]" Tix] - - -Tix Widgets ------------ - -`Tix `_ -introduces over 40 widget classes to the :mod:`tkinter` repertoire. - - -Basic Widgets -^^^^^^^^^^^^^ - - -.. class:: Balloon() - - A `Balloon - `_ that - pops up over a widget to provide help. When the user moves the cursor inside a - widget to which a Balloon widget has been bound, a small pop-up window with a - descriptive message will be shown on the screen. - -.. Python Demo of: -.. \ulink{Balloon}{http://tix.sourceforge.net/dist/current/demos/samples/Balloon.tcl} - - -.. class:: ButtonBox() - - The `ButtonBox - `_ - widget creates a box of buttons, such as is commonly used for ``Ok Cancel``. - -.. Python Demo of: -.. \ulink{ButtonBox}{http://tix.sourceforge.net/dist/current/demos/samples/BtnBox.tcl} - - -.. class:: ComboBox() - - The `ComboBox - `_ - widget is similar to the combo box control in MS Windows. The user can select a - choice by either typing in the entry subwidget or selecting from the listbox - subwidget. - -.. Python Demo of: -.. \ulink{ComboBox}{http://tix.sourceforge.net/dist/current/demos/samples/ComboBox.tcl} - - -.. class:: Control() - - The `Control - `_ - widget is also known as the :class:`SpinBox` widget. The user can adjust the - value by pressing the two arrow buttons or by entering the value directly into - the entry. The new value will be checked against the user-defined upper and - lower limits. - -.. Python Demo of: -.. \ulink{Control}{http://tix.sourceforge.net/dist/current/demos/samples/Control.tcl} - - -.. class:: LabelEntry() - - The `LabelEntry - `_ - widget packages an entry widget and a label into one mega widget. It can - be used to simplify the creation of "entry-form" type of interface. - -.. Python Demo of: -.. \ulink{LabelEntry}{http://tix.sourceforge.net/dist/current/demos/samples/LabEntry.tcl} - - -.. class:: LabelFrame() - - The `LabelFrame - `_ - widget packages a frame widget and a label into one mega widget. To create - widgets inside a LabelFrame widget, one creates the new widgets relative to the - :attr:`frame` subwidget and manage them inside the :attr:`frame` subwidget. - -.. Python Demo of: -.. \ulink{LabelFrame}{http://tix.sourceforge.net/dist/current/demos/samples/LabFrame.tcl} - - -.. class:: Meter() - - The `Meter - `_ widget - can be used to show the progress of a background job which may take a long time - to execute. - -.. Python Demo of: -.. \ulink{Meter}{http://tix.sourceforge.net/dist/current/demos/samples/Meter.tcl} - - -.. class:: OptionMenu() - - The `OptionMenu - `_ - creates a menu button of options. - -.. Python Demo of: -.. \ulink{OptionMenu}{http://tix.sourceforge.net/dist/current/demos/samples/OptMenu.tcl} - - -.. class:: PopupMenu() - - The `PopupMenu - `_ - widget can be used as a replacement of the ``tk_popup`` command. The advantage - of the :mod:`Tix` :class:`PopupMenu` widget is it requires less application code - to manipulate. - -.. Python Demo of: -.. \ulink{PopupMenu}{http://tix.sourceforge.net/dist/current/demos/samples/PopMenu.tcl} - - -.. class:: Select() - - The `Select - `_ widget - is a container of button subwidgets. It can be used to provide radio-box or - check-box style of selection options for the user. - -.. Python Demo of: -.. \ulink{Select}{http://tix.sourceforge.net/dist/current/demos/samples/Select.tcl} - - -.. class:: StdButtonBox() - - The `StdButtonBox - `_ - widget is a group of standard buttons for Motif-like dialog boxes. - -.. Python Demo of: -.. \ulink{StdButtonBox}{http://tix.sourceforge.net/dist/current/demos/samples/StdBBox.tcl} - - -File Selectors -^^^^^^^^^^^^^^ - - -.. class:: DirList() - - The `DirList - `_ - widget displays a list view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirList}{http://tix.sourceforge.net/dist/current/demos/samples/DirList.tcl} - - -.. class:: DirTree() - - The `DirTree - `_ - widget displays a tree view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirTree}{http://tix.sourceforge.net/dist/current/demos/samples/DirTree.tcl} - - -.. class:: DirSelectDialog() - - The `DirSelectDialog - `_ - widget presents the directories in the file system in a dialog window. The user - can use this dialog window to navigate through the file system to select the - desired directory. - -.. Python Demo of: -.. \ulink{DirSelectDialog}{http://tix.sourceforge.net/dist/current/demos/samples/DirDlg.tcl} - - -.. class:: DirSelectBox() - - The :class:`DirSelectBox` is similar to the standard Motif(TM) - directory-selection box. It is generally used for the user to choose a - directory. DirSelectBox stores the directories mostly recently selected into - a ComboBox widget so that they can be quickly selected again. - - -.. class:: ExFileSelectBox() - - The `ExFileSelectBox - `_ - widget is usually embedded in a tixExFileSelectDialog widget. It provides a - convenient method for the user to select files. The style of the - :class:`ExFileSelectBox` widget is very similar to the standard file dialog on - MS Windows 3.1. - -.. Python Demo of: -.. \ulink{ExFileSelectDialog}{http://tix.sourceforge.net/dist/current/demos/samples/EFileDlg.tcl} - - -.. class:: FileSelectBox() - - The `FileSelectBox - `_ - is similar to the standard Motif(TM) file-selection box. It is generally used - for the user to choose a file. FileSelectBox stores the files mostly recently - selected into a :class:`ComboBox` widget so that they can be quickly selected - again. - -.. Python Demo of: -.. \ulink{FileSelectDialog}{http://tix.sourceforge.net/dist/current/demos/samples/FileDlg.tcl} - - -.. class:: FileEntry() - - The `FileEntry - `_ - widget can be used to input a filename. The user can type in the filename - manually. Alternatively, the user can press the button widget that sits next to - the entry, which will bring up a file selection dialog. - -.. Python Demo of: -.. \ulink{FileEntry}{http://tix.sourceforge.net/dist/current/demos/samples/FileEnt.tcl} - - -Hierarchical ListBox -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: HList() - - The `HList - `_ widget - can be used to display any data that have a hierarchical structure, for example, - file system directory trees. The list entries are indented and connected by - branch lines according to their places in the hierarchy. - -.. Python Demo of: -.. \ulink{HList}{http://tix.sourceforge.net/dist/current/demos/samples/HList1.tcl} - - -.. class:: CheckList() - - The `CheckList - `_ - widget displays a list of items to be selected by the user. CheckList acts - similarly to the Tk checkbutton or radiobutton widgets, except it is capable of - handling many more items than checkbuttons or radiobuttons. - -.. Python Demo of: -.. \ulink{ CheckList}{http://tix.sourceforge.net/dist/current/demos/samples/ChkList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (1)}{http://tix.sourceforge.net/dist/current/demos/samples/SHList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (2)}{http://tix.sourceforge.net/dist/current/demos/samples/SHList2.tcl} - - -.. class:: Tree() - - The `Tree - `_ widget - can be used to display hierarchical data in a tree form. The user can adjust the - view of the tree by opening or closing parts of the tree. - -.. Python Demo of: -.. \ulink{Tree}{http://tix.sourceforge.net/dist/current/demos/samples/Tree.tcl} -.. Python Demo of: -.. \ulink{Tree (Dynamic)}{http://tix.sourceforge.net/dist/current/demos/samples/DynTree.tcl} - - -Tabular ListBox -^^^^^^^^^^^^^^^ - - -.. class:: TList() - - The `TList - `_ widget - can be used to display data in a tabular format. The list entries of a - :class:`TList` widget are similar to the entries in the Tk listbox widget. The - main differences are (1) the :class:`TList` widget can display the list entries - in a two dimensional format and (2) you can use graphical images as well as - multiple colors and fonts for the list entries. - -.. Python Demo of: -.. \ulink{ScrolledTList (1)}{http://tix.sourceforge.net/dist/current/demos/samples/STList1.tcl} -.. Python Demo of: -.. \ulink{ScrolledTList (2)}{http://tix.sourceforge.net/dist/current/demos/samples/STList2.tcl} -.. Grid has yet to be added to Python -.. \subsubsection{Grid Widget} -.. Python Demo of: -.. \ulink{Simple Grid}{http://tix.sourceforge.net/dist/current/demos/samples/SGrid0.tcl} -.. Python Demo of: -.. \ulink{ScrolledGrid}{http://tix.sourceforge.net/dist/current/demos/samples/SGrid1.tcl} -.. Python Demo of: -.. \ulink{Editable Grid}{http://tix.sourceforge.net/dist/current/demos/samples/EditGrid.tcl} - - -Manager Widgets -^^^^^^^^^^^^^^^ - - -.. class:: PanedWindow() - - The `PanedWindow - `_ - widget allows the user to interactively manipulate the sizes of several panes. - The panes can be arranged either vertically or horizontally. The user changes - the sizes of the panes by dragging the resize handle between two panes. - -.. Python Demo of: -.. \ulink{PanedWindow}{http://tix.sourceforge.net/dist/current/demos/samples/PanedWin.tcl} - - -.. class:: ListNoteBook() - - The `ListNoteBook - `_ - widget is very similar to the :class:`TixNoteBook` widget: it can be used to - display many windows in a limited space using a notebook metaphor. The notebook - is divided into a stack of pages (windows). At one time only one of these pages - can be shown. The user can navigate through these pages by choosing the name of - the desired page in the :attr:`hlist` subwidget. - -.. Python Demo of: -.. \ulink{ListNoteBook}{http://tix.sourceforge.net/dist/current/demos/samples/ListNBK.tcl} - - -.. class:: NoteBook() - - The `NoteBook - `_ - widget can be used to display many windows in a limited space using a notebook - metaphor. The notebook is divided into a stack of pages. At one time only one of - these pages can be shown. The user can navigate through these pages by choosing - the visual "tabs" at the top of the NoteBook widget. - -.. Python Demo of: -.. \ulink{NoteBook}{http://tix.sourceforge.net/dist/current/demos/samples/NoteBook.tcl} - -.. \subsubsection{Scrolled Widgets} -.. Python Demo of: -.. \ulink{ScrolledListBox}{http://tix.sourceforge.net/dist/current/demos/samples/SListBox.tcl} -.. Python Demo of: -.. \ulink{ScrolledText}{http://tix.sourceforge.net/dist/current/demos/samples/SText.tcl} -.. Python Demo of: -.. \ulink{ScrolledWindow}{http://tix.sourceforge.net/dist/current/demos/samples/SWindow.tcl} -.. Python Demo of: -.. \ulink{Canvas Object View}{http://tix.sourceforge.net/dist/current/demos/samples/CObjView.tcl} - - -Image Types -^^^^^^^^^^^ - -The :mod:`tkinter.tix` module adds: - -* `pixmap `_ - capabilities to all :mod:`tkinter.tix` and :mod:`tkinter` widgets to create - color images from XPM files. - - .. Python Demo of: - .. \ulink{XPM Image In Button}{http://tix.sourceforge.net/dist/current/demos/samples/Xpm.tcl} - .. Python Demo of: - .. \ulink{XPM Image In Menu}{http://tix.sourceforge.net/dist/current/demos/samples/Xpm1.tcl} - -* `Compound - `_ image - types can be used to create images that consists of multiple horizontal lines; - each line is composed of a series of items (texts, bitmaps, images or spaces) - arranged from left to right. For example, a compound image can be used to - display a bitmap and a text string simultaneously in a Tk :class:`Button` - widget. - - .. Python Demo of: - .. \ulink{Compound Image In Buttons}{http://tix.sourceforge.net/dist/current/demos/samples/CmpImg.tcl} - .. Python Demo of: - .. \ulink{Compound Image In NoteBook}{http://tix.sourceforge.net/dist/current/demos/samples/CmpImg2.tcl} - .. Python Demo of: - .. \ulink{Compound Image Notebook Color Tabs}{http://tix.sourceforge.net/dist/current/demos/samples/CmpImg4.tcl} - .. Python Demo of: - .. \ulink{Compound Image Icons}{http://tix.sourceforge.net/dist/current/demos/samples/CmpImg3.tcl} - - -Miscellaneous Widgets -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: InputOnly() - - The `InputOnly - `_ - widgets are to accept inputs from the user, which can be done with the ``bind`` - command (Unix only). - - -Form Geometry Manager -^^^^^^^^^^^^^^^^^^^^^ - -In addition, :mod:`tkinter.tix` augments :mod:`tkinter` by providing: - - -.. class:: Form() - - The `Form - `_ geometry - manager based on attachment rules for all Tk widgets. - - -Tix Commands ------------- - - -.. class:: tixCommand() - - The `tix commands - `_ provide - access to miscellaneous elements of :mod:`Tix`'s internal state and the - :mod:`Tix` application context. Most of the information manipulated by these - methods pertains to the application as a whole, or to a screen or display, - rather than to a particular window. - - To view the current settings, the common usage is:: - - from tkinter import tix - root = tix.Tk() - print(root.tix_configure()) - - -.. method:: tixCommand.tix_configure(cnf=None, **kw) - - Query or modify the configuration options of the Tix application context. If no - option is specified, returns a dictionary all of the available options. If - option is specified with no value, then the method returns a list describing the - one named option (this list will be identical to the corresponding sublist of - the value returned if no option is specified). If one or more option-value - pairs are specified, then the method modifies the given option(s) to have the - given value(s); in this case the method returns an empty string. Option may be - any of the configuration options. - - -.. method:: tixCommand.tix_cget(option) - - Returns the current value of the configuration option given by *option*. Option - may be any of the configuration options. - - -.. method:: tixCommand.tix_getbitmap(name) - - Locates a bitmap file of the name ``name.xpm`` or ``name`` in one of the bitmap - directories (see the :meth:`tix_addbitmapdir` method). By using - :meth:`tix_getbitmap`, you can avoid hard coding the pathnames of the bitmap - files in your application. When successful, it returns the complete pathname of - the bitmap file, prefixed with the character ``@``. The returned value can be - used to configure the ``bitmap`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_addbitmapdir(directory) - - Tix maintains a list of directories under which the :meth:`tix_getimage` and - :meth:`tix_getbitmap` methods will search for image files. The standard bitmap - directory is :file:`$TIX_LIBRARY/bitmaps`. The :meth:`tix_addbitmapdir` method - adds *directory* into this list. By using this method, the image files of an - applications can also be located using the :meth:`tix_getimage` or - :meth:`tix_getbitmap` method. - - -.. method:: tixCommand.tix_filedialog([dlgclass]) - - Returns the file selection dialog that may be shared among different calls from - this application. This method will create a file selection dialog widget when - it is called the first time. This dialog will be returned by all subsequent - calls to :meth:`tix_filedialog`. An optional dlgclass parameter can be passed - as a string to specified what type of file selection dialog widget is desired. - Possible options are ``tix``, ``FileSelectDialog`` or ``tixExFileSelectDialog``. - - -.. method:: tixCommand.tix_getimage(self, name) - - Locates an image file of the name :file:`name.xpm`, :file:`name.xbm` or - :file:`name.ppm` in one of the bitmap directories (see the - :meth:`tix_addbitmapdir` method above). If more than one file with the same name - (but different extensions) exist, then the image type is chosen according to the - depth of the X display: xbm images are chosen on monochrome displays and color - images are chosen on color displays. By using :meth:`tix_getimage`, you can - avoid hard coding the pathnames of the image files in your application. When - successful, this method returns the name of the newly created image, which can - be used to configure the ``image`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_option_get(name) - - Gets the options maintained by the Tix scheme mechanism. - - -.. method:: tixCommand.tix_resetoptions(newScheme, newFontSet[, newScmPrio]) - - Resets the scheme and fontset of the Tix application to *newScheme* and - *newFontSet*, respectively. This affects only those widgets created after this - call. Therefore, it is best to call the resetoptions method before the creation - of any widgets in a Tix application. - - The optional parameter *newScmPrio* can be given to reset the priority level of - the Tk options set by the Tix schemes. - - Because of the way Tk handles the X option database, after Tix has been has - imported and inited, it is not possible to reset the color schemes and font sets - using the :meth:`tix_config` method. Instead, the :meth:`tix_resetoptions` - method must be used. diff --git a/third_party/python/Doc/library/tkinter.ttk.rst b/third_party/python/Doc/library/tkinter.ttk.rst deleted file mode 100644 index 927f7f3c6..000000000 --- a/third_party/python/Doc/library/tkinter.ttk.rst +++ /dev/null @@ -1,1447 +0,0 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== - -.. module:: tkinter.ttk - :synopsis: Tk themed widget set - -.. sectionauthor:: Guilherme Polo - -**Source code:** :source:`Lib/tkinter/ttk.py` - -.. index:: single: ttk - --------------- - -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, -introduced in Tk 8.5. If Python has not been compiled against Tk 8.5, this -module can still be accessed if *Tile* has been installed. The former -method using Tk 8.5 provides additional benefits including anti-aliased font -rendering under X11 and window transparency (requiring a composition -window manager on X11). - -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, -the code implementing a widget's behavior from the code implementing its -appearance. - - -.. seealso:: - - `Tk Widget Styling Support `_ - A document introducing theming support for Tk - - -Using Ttk ---------- - -To start using Ttk, import its module:: - - from tkinter import ttk - -To override the basic Tk widgets, the import should follow the Tk import:: - - from tkinter import * - from tkinter.ttk import * - -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, -:class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, -:class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to -automatically replace the Tk widgets. - -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. - - -.. seealso:: - - `Converting existing applications to use Tile widgets `_ - A monograph (using Tcl terminology) about differences typically - encountered when moving applications to use the new widgets. - - -Ttk Widgets ------------ - -Ttk comes with 17 widgets, eleven of which already existed in tkinter: -:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, -:class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`. The other six are -new: :class:`Combobox`, :class:`Notebook`, :class:`Progressbar`, -:class:`Separator`, :class:`Sizegrip` and :class:`Treeview`. And all them are -subclasses of :class:`Widget`. - -Using the Ttk widgets gives the application an improved look and feel. -As discussed above, there are differences in how the styling is coded. - -Tk code:: - - l1 = tkinter.Label(text="Test", fg="black", bg="white") - l2 = tkinter.Label(text="Test", fg="black", bg="white") - - -Ttk code:: - - style = ttk.Style() - style.configure("BW.TLabel", foreground="black", background="white") - - l1 = ttk.Label(text="Test", style="BW.TLabel") - l2 = ttk.Label(text="Test", style="BW.TLabel") - -For more information about TtkStyling_, see the :class:`Style` class -documentation. - -Widget ------- - -:class:`ttk.Widget` defines standard options and methods supported by Tk -themed widgets and is not supposed to be directly instantiated. - - -Standard Options -^^^^^^^^^^^^^^^^ - -All the :mod:`ttk` Widgets accepts the following options: - - .. tabularcolumns:: |l|L| - - +-----------+--------------------------------------------------------------+ - | Option | Description | - +===========+==============================================================+ - | class | Specifies the window class. The class is used when querying | - | | the option database for the window's other options, to | - | | determine the default bindtags for the window, and to select | - | | the widget's default layout and style. This option is | - | | read-only, and may only be specified when the window is | - | | created. | - +-----------+--------------------------------------------------------------+ - | cursor | Specifies the mouse cursor to be used for the widget. If set | - | | to the empty string (the default), the cursor is inherited | - | | for the parent widget. | - +-----------+--------------------------------------------------------------+ - | takefocus | Determines whether the window accepts the focus during | - | | keyboard traversal. 0, 1 or an empty string is returned. | - | | If 0 is returned, it means that the window should be skipped | - | | entirely during keyboard traversal. If 1, it means that the | - | | window should receive the input focus as long as it is | - | | viewable. And an empty string means that the traversal | - | | scripts make the decision about whether or not to focus | - | | on the window. | - +-----------+--------------------------------------------------------------+ - | style | May be used to specify a custom widget style. | - +-----------+--------------------------------------------------------------+ - - -Scrollable Widget Options -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following options are supported by widgets that are controlled by a -scrollbar. - - .. tabularcolumns:: |l|L| - - +----------------+---------------------------------------------------------+ - | Option | Description | - +================+=========================================================+ - | xscrollcommand | Used to communicate with horizontal scrollbars. | - | | | - | | When the view in the widget's window change, the widget | - | | will generate a Tcl command based on the scrollcommand. | - | | | - | | Usually this option consists of the method | - | | :meth:`Scrollbar.set` of some scrollbar. This will cause| - | | the scrollbar to be updated whenever the view in the | - | | window changes. | - +----------------+---------------------------------------------------------+ - | yscrollcommand | Used to communicate with vertical scrollbars. | - | | For some more information, see above. | - +----------------+---------------------------------------------------------+ - - -Label Options -^^^^^^^^^^^^^ - -The following options are supported by labels, buttons and other button-like -widgets. - - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +--------------+-----------------------------------------------------------+ - | Option | Description | - +==============+===========================================================+ - | text | Specifies a text string to be displayed inside the widget.| - +--------------+-----------------------------------------------------------+ - | textvariable | Specifies a name whose value will be used in place of the | - | | text option resource. | - +--------------+-----------------------------------------------------------+ - | underline | If set, specifies the index (0-based) of a character to | - | | underline in the text string. The underline character is | - | | used for mnemonic activation. | - +--------------+-----------------------------------------------------------+ - | image | Specifies an image to display. This is a list of 1 or more| - | | elements. The first element is the default image name. The| - | | rest of the list if a sequence of statespec/value pairs as| - | | defined by :meth:`Style.map`, specifying different images | - | | to use when the widget is in a particular state or a | - | | combination of states. All images in the list should have | - | | the same size. | - +--------------+-----------------------------------------------------------+ - | compound | Specifies how to display the image relative to the text, | - | | in the case both text and images options are present. | - | | Valid values are: | - | | | - | | * text: display text only | - | | * image: display image only | - | | * top, bottom, left, right: display image above, below, | - | | left of, or right of the text, respectively. | - | | * none: the default. display the image if present, | - | | otherwise the text. | - +--------------+-----------------------------------------------------------+ - | width | If greater than zero, specifies how much space, in | - | | character widths, to allocate for the text label, if less | - | | than zero, specifies a minimum width. If zero or | - | | unspecified, the natural width of the text label is used. | - +--------------+-----------------------------------------------------------+ - - -Compatibility Options -^^^^^^^^^^^^^^^^^^^^^ - - .. tabularcolumns:: |l|L| - - +--------+----------------------------------------------------------------+ - | Option | Description | - +========+================================================================+ - | state | May be set to "normal" or "disabled" to control the "disabled" | - | | state bit. This is a write-only option: setting it changes the | - | | widget state, but the :meth:`Widget.state` method does not | - | | affect this option. | - +--------+----------------------------------------------------------------+ - -Widget States -^^^^^^^^^^^^^ - -The widget state is a bitmap of independent state flags. - - .. tabularcolumns:: |l|L| - - +------------+-------------------------------------------------------------+ - | Flag | Description | - +============+=============================================================+ - | active | The mouse cursor is over the widget and pressing a mouse | - | | button will cause some action to occur | - +------------+-------------------------------------------------------------+ - | disabled | Widget is disabled under program control | - +------------+-------------------------------------------------------------+ - | focus | Widget has keyboard focus | - +------------+-------------------------------------------------------------+ - | pressed | Widget is being pressed | - +------------+-------------------------------------------------------------+ - | selected | "On", "true", or "current" for things like Checkbuttons and | - | | radiobuttons | - +------------+-------------------------------------------------------------+ - | background | Windows and Mac have a notion of an "active" or foreground | - | | window. The *background* state is set for widgets in a | - | | background window, and cleared for those in the foreground | - | | window | - +------------+-------------------------------------------------------------+ - | readonly | Widget should not allow user modification | - +------------+-------------------------------------------------------------+ - | alternate | A widget-specific alternate display format | - +------------+-------------------------------------------------------------+ - | invalid | The widget's value is invalid | - +------------+-------------------------------------------------------------+ - -A state specification is a sequence of state names, optionally prefixed with -an exclamation point indicating that the bit is off. - - -ttk.Widget -^^^^^^^^^^ - -Besides the methods described below, the :class:`ttk.Widget` supports the -methods :meth:`tkinter.Widget.cget` and :meth:`tkinter.Widget.configure`. - -.. class:: Widget - - .. method:: identify(x, y) - - Returns the name of the element at position *x* *y*, or the empty string - if the point does not lie within any element. - - *x* and *y* are pixel coordinates relative to the widget. - - - .. method:: instate(statespec, callback=None, *args, **kw) - - Test the widget's state. If a callback is not specified, returns ``True`` - if the widget state matches *statespec* and ``False`` otherwise. If callback - is specified then it is called with args if widget state matches - *statespec*. - - - .. method:: state(statespec=None) - - Modify or inquire widget state. If *statespec* is specified, sets the - widget state according to it and return a new *statespec* indicating - which flags were changed. If *statespec* is not specified, returns - the currently-enabled state flags. - - *statespec* will usually be a list or a tuple. - - -Combobox --------- - -The :class:`ttk.Combobox` widget combines a text field with a pop-down list of -values. This widget is a subclass of :class:`Entry`. - -Besides the methods inherited from :class:`Widget`: :meth:`Widget.cget`, -:meth:`Widget.configure`, :meth:`Widget.identify`, :meth:`Widget.instate` -and :meth:`Widget.state`, and the following inherited from :class:`Entry`: -:meth:`Entry.bbox`, :meth:`Entry.delete`, :meth:`Entry.icursor`, -:meth:`Entry.index`, :meth:`Entry.insert`, :meth:`Entry.selection`, -:meth:`Entry.xview`, it has some other methods, described at -:class:`ttk.Combobox`. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - - .. tabularcolumns:: |l|L| - - +-----------------+--------------------------------------------------------+ - | Option | Description | - +=================+========================================================+ - | exportselection | Boolean value. If set, the widget selection is linked | - | | to the Window Manager selection (which can be returned | - | | by invoking Misc.selection_get, for example). | - +-----------------+--------------------------------------------------------+ - | justify | Specifies how the text is aligned within the widget. | - | | One of "left", "center", or "right". | - +-----------------+--------------------------------------------------------+ - | height | Specifies the height of the pop-down listbox, in rows. | - +-----------------+--------------------------------------------------------+ - | postcommand | A script (possibly registered with Misc.register) that | - | | is called immediately before displaying the values. It | - | | may specify which values to display. | - +-----------------+--------------------------------------------------------+ - | state | One of "normal", "readonly", or "disabled". In the | - | | "readonly" state, the value may not be edited directly,| - | | and the user can only selection of the values from the | - | | dropdown list. In the "normal" state, the text field is| - | | directly editable. In the "disabled" state, no | - | | interaction is possible. | - +-----------------+--------------------------------------------------------+ - | textvariable | Specifies a name whose value is linked to the widget | - | | value. Whenever the value associated with that name | - | | changes, the widget value is updated, and vice versa. | - | | See :class:`tkinter.StringVar`. | - +-----------------+--------------------------------------------------------+ - | values | Specifies the list of values to display in the | - | | drop-down listbox. | - +-----------------+--------------------------------------------------------+ - | width | Specifies an integer value indicating the desired width| - | | of the entry window, in average-size characters of the | - | | widget's font. | - +-----------------+--------------------------------------------------------+ - - -Virtual events -^^^^^^^^^^^^^^ - -The combobox widgets generates a **<>** virtual event -when the user selects an element from the list of values. - - -ttk.Combobox -^^^^^^^^^^^^ - -.. class:: Combobox - - .. method:: current(newindex=None) - - If *newindex* is specified, sets the combobox value to the element - position *newindex*. Otherwise, returns the index of the current value or - -1 if the current value is not in the values list. - - - .. method:: get() - - Returns the current value of the combobox. - - - .. method:: set(value) - - Sets the value of the combobox to *value*. - - -Notebook --------- - -Ttk Notebook widget manages a collection of windows and displays a single -one at a time. Each child window is associated with a tab, which the user -may select to change the currently-displayed window. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - - .. tabularcolumns:: |l|L| - - +---------+----------------------------------------------------------------+ - | Option | Description | - +=========+================================================================+ - | height | If present and greater than zero, specifies the desired height | - | | of the pane area (not including internal padding or tabs). | - | | Otherwise, the maximum height of all panes is used. | - +---------+----------------------------------------------------------------+ - | padding | Specifies the amount of extra space to add around the outside | - | | of the notebook. The padding is a list up to four length | - | | specifications left top right bottom. If fewer than four | - | | elements are specified, bottom defaults to top, right defaults | - | | to left, and top defaults to left. | - +---------+----------------------------------------------------------------+ - | width | If present and greater than zero, specified the desired width | - | | of the pane area (not including internal padding). Otherwise, | - | | the maximum width of all panes is used. | - +---------+----------------------------------------------------------------+ - - -Tab Options -^^^^^^^^^^^ - -There are also specific options for tabs: - - .. tabularcolumns:: |l|L| - - +-----------+--------------------------------------------------------------+ - | Option | Description | - +===========+==============================================================+ - | state | Either "normal", "disabled" or "hidden". If "disabled", then | - | | the tab is not selectable. If "hidden", then the tab is not | - | | shown. | - +-----------+--------------------------------------------------------------+ - | sticky | Specifies how the child window is positioned within the pane | - | | area. Value is a string containing zero or more of the | - | | characters "n", "s", "e" or "w". Each letter refers to a | - | | side (north, south, east or west) that the child window will | - | | stick to, as per the :meth:`grid` geometry manager. | - +-----------+--------------------------------------------------------------+ - | padding | Specifies the amount of extra space to add between the | - | | notebook and this pane. Syntax is the same as for the option | - | | padding used by this widget. | - +-----------+--------------------------------------------------------------+ - | text | Specifies a text to be displayed in the tab. | - +-----------+--------------------------------------------------------------+ - | image | Specifies an image to display in the tab. See the option | - | | image described in :class:`Widget`. | - +-----------+--------------------------------------------------------------+ - | compound | Specifies how to display the image relative to the text, in | - | | the case both options text and image are present. See | - | | `Label Options`_ for legal values. | - +-----------+--------------------------------------------------------------+ - | underline | Specifies the index (0-based) of a character to underline in | - | | the text string. The underlined character is used for | - | | mnemonic activation if :meth:`Notebook.enable_traversal` is | - | | called. | - +-----------+--------------------------------------------------------------+ - - -Tab Identifiers -^^^^^^^^^^^^^^^ - -The tab_id present in several methods of :class:`ttk.Notebook` may take any -of the following forms: - -* An integer between zero and the number of tabs -* The name of a child window -* A positional specification of the form "@x,y", which identifies the tab -* The literal string "current", which identifies the currently-selected tab -* The literal string "end", which returns the number of tabs (only valid for - :meth:`Notebook.index`) - - -Virtual Events -^^^^^^^^^^^^^^ - -This widget generates a **<>** virtual event after a new -tab is selected. - - -ttk.Notebook -^^^^^^^^^^^^ - -.. class:: Notebook - - .. method:: add(child, **kw) - - Adds a new tab to the notebook. - - If window is currently managed by the notebook but hidden, it is - restored to its previous position. - - See `Tab Options`_ for the list of available options. - - - .. method:: forget(tab_id) - - Removes the tab specified by *tab_id*, unmaps and unmanages the - associated window. - - - .. method:: hide(tab_id) - - Hides the tab specified by *tab_id*. - - The tab will not be displayed, but the associated window remains - managed by the notebook and its configuration remembered. Hidden tabs - may be restored with the :meth:`add` command. - - - .. method:: identify(x, y) - - Returns the name of the tab element at position *x*, *y*, or the empty - string if none. - - - .. method:: index(tab_id) - - Returns the numeric index of the tab specified by *tab_id*, or the total - number of tabs if *tab_id* is the string "end". - - - .. method:: insert(pos, child, **kw) - - Inserts a pane at the specified position. - - *pos* is either the string "end", an integer index, or the name of a - managed child. If *child* is already managed by the notebook, moves it to - the specified position. - - See `Tab Options`_ for the list of available options. - - - .. method:: select(tab_id=None) - - Selects the specified *tab_id*. - - The associated child window will be displayed, and the - previously-selected window (if different) is unmapped. If *tab_id* is - omitted, returns the widget name of the currently selected pane. - - - .. method:: tab(tab_id, option=None, **kw) - - Query or modify the options of the specific *tab_id*. - - If *kw* is not given, returns a dictionary of the tab option values. If - *option* is specified, returns the value of that *option*. Otherwise, - sets the options to the corresponding values. - - - .. method:: tabs() - - Returns a list of windows managed by the notebook. - - - .. method:: enable_traversal() - - Enable keyboard traversal for a toplevel window containing this notebook. - - This will extend the bindings for the toplevel window containing the - notebook as follows: - - * :kbd:`Control-Tab`: selects the tab following the currently selected one. - * :kbd:`Shift-Control-Tab`: selects the tab preceding the currently selected one. - * :kbd:`Alt-K`: where *K* is the mnemonic (underlined) character of any tab, will - select that tab. - - Multiple notebooks in a single toplevel may be enabled for traversal, - including nested notebooks. However, notebook traversal only works - properly if all panes have the notebook they are in as master. - - -Progressbar ------------ - -The :class:`ttk.Progressbar` widget shows the status of a long-running -operation. It can operate in two modes: 1) the determinate mode which shows the -amount completed relative to the total amount of work to be done and 2) the -indeterminate mode which provides an animated display to let the user know that -work is progressing. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - - .. tabularcolumns:: |l|L| - - +----------+---------------------------------------------------------------+ - | Option | Description | - +==========+===============================================================+ - | orient | One of "horizontal" or "vertical". Specifies the orientation | - | | of the progress bar. | - +----------+---------------------------------------------------------------+ - | length | Specifies the length of the long axis of the progress bar | - | | (width if horizontal, height if vertical). | - +----------+---------------------------------------------------------------+ - | mode | One of "determinate" or "indeterminate". | - +----------+---------------------------------------------------------------+ - | maximum | A number specifying the maximum value. Defaults to 100. | - +----------+---------------------------------------------------------------+ - | value | The current value of the progress bar. In "determinate" mode, | - | | this represents the amount of work completed. In | - | | "indeterminate" mode, it is interpreted as modulo *maximum*; | - | | that is, the progress bar completes one "cycle" when its value| - | | increases by *maximum*. | - +----------+---------------------------------------------------------------+ - | variable | A name which is linked to the option value. If specified, the | - | | value of the progress bar is automatically set to the value of| - | | this name whenever the latter is modified. | - +----------+---------------------------------------------------------------+ - | phase | Read-only option. The widget periodically increments the value| - | | of this option whenever its value is greater than 0 and, in | - | | determinate mode, less than maximum. This option may be used | - | | by the current theme to provide additional animation effects. | - +----------+---------------------------------------------------------------+ - - -ttk.Progressbar -^^^^^^^^^^^^^^^ - -.. class:: Progressbar - - .. method:: start(interval=None) - - Begin autoincrement mode: schedules a recurring timer event that calls - :meth:`Progressbar.step` every *interval* milliseconds. If omitted, - *interval* defaults to 50 milliseconds. - - - .. method:: step(amount=None) - - Increments the progress bar's value by *amount*. - - *amount* defaults to 1.0 if omitted. - - - .. method:: stop() - - Stop autoincrement mode: cancels any recurring timer event initiated by - :meth:`Progressbar.start` for this progress bar. - - -Separator ---------- - -The :class:`ttk.Separator` widget displays a horizontal or vertical separator -bar. - -It has no other methods besides the ones inherited from :class:`ttk.Widget`. - - -Options -^^^^^^^ - -This widget accepts the following specific option: - - .. tabularcolumns:: |l|L| - - +--------+----------------------------------------------------------------+ - | Option | Description | - +========+================================================================+ - | orient | One of "horizontal" or "vertical". Specifies the orientation of| - | | the separator. | - +--------+----------------------------------------------------------------+ - - -Sizegrip --------- - -The :class:`ttk.Sizegrip` widget (also known as a grow box) allows the user to -resize the containing toplevel window by pressing and dragging the grip. - -This widget has neither specific options nor specific methods, besides the -ones inherited from :class:`ttk.Widget`. - - -Platform-specific notes -^^^^^^^^^^^^^^^^^^^^^^^ - -* On MacOS X, toplevel windows automatically include a built-in size grip - by default. Adding a :class:`Sizegrip` is harmless, since the built-in - grip will just mask the widget. - - -Bugs -^^^^ - -* If the containing toplevel's position was specified relative to the right - or bottom of the screen (e.g. ....), the :class:`Sizegrip` widget will - not resize the window. -* This widget supports only "southeast" resizing. - - -Treeview --------- - -The :class:`ttk.Treeview` widget displays a hierarchical collection of items. -Each item has a textual label, an optional image, and an optional list of data -values. The data values are displayed in successive columns after the tree -label. - -The order in which data values are displayed may be controlled by setting -the widget option ``displaycolumns``. The tree widget can also display column -headings. Columns may be accessed by number or symbolic names listed in the -widget option columns. See `Column Identifiers`_. - -Each item is identified by a unique name. The widget will generate item IDs -if they are not supplied by the caller. There is a distinguished root item, -named ``{}``. The root item itself is not displayed; its children appear at the -top level of the hierarchy. - -Each item also has a list of tags, which can be used to associate event bindings -with individual items and control the appearance of the item. - -The Treeview widget supports horizontal and vertical scrolling, according to -the options described in `Scrollable Widget Options`_ and the methods -:meth:`Treeview.xview` and :meth:`Treeview.yview`. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +----------------+--------------------------------------------------------+ - | Option | Description | - +================+========================================================+ - | columns | A list of column identifiers, specifying the number of | - | | columns and their names. | - +----------------+--------------------------------------------------------+ - | displaycolumns | A list of column identifiers (either symbolic or | - | | integer indices) specifying which data columns are | - | | displayed and the order in which they appear, or the | - | | string "#all". | - +----------------+--------------------------------------------------------+ - | height | Specifies the number of rows which should be visible. | - | | Note: the requested width is determined from the sum | - | | of the column widths. | - +----------------+--------------------------------------------------------+ - | padding | Specifies the internal padding for the widget. The | - | | padding is a list of up to four length specifications. | - +----------------+--------------------------------------------------------+ - | selectmode | Controls how the built-in class bindings manage the | - | | selection. One of "extended", "browse" or "none". | - | | If set to "extended" (the default), multiple items may | - | | be selected. If "browse", only a single item will be | - | | selected at a time. If "none", the selection will not | - | | be changed. | - | | | - | | Note that the application code and tag bindings can set| - | | the selection however they wish, regardless of the | - | | value of this option. | - +----------------+--------------------------------------------------------+ - | show | A list containing zero or more of the following values,| - | | specifying which elements of the tree to display. | - | | | - | | * tree: display tree labels in column #0. | - | | * headings: display the heading row. | - | | | - | | The default is "tree headings", i.e., show all | - | | elements. | - | | | - | | **Note**: Column #0 always refers to the tree column, | - | | even if show="tree" is not specified. | - +----------------+--------------------------------------------------------+ - - -Item Options -^^^^^^^^^^^^ - -The following item options may be specified for items in the insert and item -widget commands. - - .. tabularcolumns:: |l|L| - - +--------+---------------------------------------------------------------+ - | Option | Description | - +========+===============================================================+ - | text | The textual label to display for the item. | - +--------+---------------------------------------------------------------+ - | image | A Tk Image, displayed to the left of the label. | - +--------+---------------------------------------------------------------+ - | values | The list of values associated with the item. | - | | | - | | Each item should have the same number of values as the widget | - | | option columns. If there are fewer values than columns, the | - | | remaining values are assumed empty. If there are more values | - | | than columns, the extra values are ignored. | - +--------+---------------------------------------------------------------+ - | open | True/False value indicating whether the item's children should| - | | be displayed or hidden. | - +--------+---------------------------------------------------------------+ - | tags | A list of tags associated with this item. | - +--------+---------------------------------------------------------------+ - - -Tag Options -^^^^^^^^^^^ - -The following options may be specified on tags: - - .. tabularcolumns:: |l|L| - - +------------+-----------------------------------------------------------+ - | Option | Description | - +============+===========================================================+ - | foreground | Specifies the text foreground color. | - +------------+-----------------------------------------------------------+ - | background | Specifies the cell or item background color. | - +------------+-----------------------------------------------------------+ - | font | Specifies the font to use when drawing text. | - +------------+-----------------------------------------------------------+ - | image | Specifies the item image, in case the item's image option | - | | is empty. | - +------------+-----------------------------------------------------------+ - - -Column Identifiers -^^^^^^^^^^^^^^^^^^ - -Column identifiers take any of the following forms: - -* A symbolic name from the list of columns option. -* An integer n, specifying the nth data column. -* A string of the form #n, where n is an integer, specifying the nth display - column. - -Notes: - -* Item's option values may be displayed in a different order than the order - in which they are stored. -* Column #0 always refers to the tree column, even if show="tree" is not - specified. - -A data column number is an index into an item's option values list; a display -column number is the column number in the tree where the values are displayed. -Tree labels are displayed in column #0. If option displaycolumns is not set, -then data column n is displayed in column #n+1. Again, **column #0 always -refers to the tree column**. - - -Virtual Events -^^^^^^^^^^^^^^ - -The Treeview widget generates the following virtual events. - - .. tabularcolumns:: |l|L| - - +--------------------+--------------------------------------------------+ - | Event | Description | - +====================+==================================================+ - | <> | Generated whenever the selection changes. | - +--------------------+--------------------------------------------------+ - | <> | Generated just before settings the focus item to | - | | open=True. | - +--------------------+--------------------------------------------------+ - | <> | Generated just after setting the focus item to | - | | open=False. | - +--------------------+--------------------------------------------------+ - -The :meth:`Treeview.focus` and :meth:`Treeview.selection` methods can be used -to determine the affected item or items. - - -ttk.Treeview -^^^^^^^^^^^^ - -.. class:: Treeview - - .. method:: bbox(item, column=None) - - Returns the bounding box (relative to the treeview widget's window) of - the specified *item* in the form (x, y, width, height). - - If *column* is specified, returns the bounding box of that cell. If the - *item* is not visible (i.e., if it is a descendant of a closed item or is - scrolled offscreen), returns an empty string. - - - .. method:: get_children(item=None) - - Returns the list of children belonging to *item*. - - If *item* is not specified, returns root children. - - - .. method:: set_children(item, *newchildren) - - Replaces *item*'s child with *newchildren*. - - Children present in *item* that are not present in *newchildren* are - detached from the tree. No items in *newchildren* may be an ancestor of - *item*. Note that not specifying *newchildren* results in detaching - *item*'s children. - - - .. method:: column(column, option=None, **kw) - - Query or modify the options for the specified *column*. - - If *kw* is not given, returns a dict of the column option values. If - *option* is specified then the value for that *option* is returned. - Otherwise, sets the options to the corresponding values. - - The valid options/values are: - - * id - Returns the column name. This is a read-only option. - * anchor: One of the standard Tk anchor values. - Specifies how the text in this column should be aligned with respect - to the cell. - * minwidth: width - The minimum width of the column in pixels. The treeview widget will - not make the column any smaller than specified by this option when - the widget is resized or the user drags a column. - * stretch: True/False - Specifies whether the column's width should be adjusted when - the widget is resized. - * width: width - The width of the column in pixels. - - To configure the tree column, call this with column = "#0" - - .. method:: delete(*items) - - Delete all specified *items* and all their descendants. - - The root item may not be deleted. - - - .. method:: detach(*items) - - Unlinks all of the specified *items* from the tree. - - The items and all of their descendants are still present, and may be - reinserted at another point in the tree, but will not be displayed. - - The root item may not be detached. - - - .. method:: exists(item) - - Returns ``True`` if the specified *item* is present in the tree. - - - .. method:: focus(item=None) - - If *item* is specified, sets the focus item to *item*. Otherwise, returns - the current focus item, or '' if there is none. - - - .. method:: heading(column, option=None, **kw) - - Query or modify the heading options for the specified *column*. - - If *kw* is not given, returns a dict of the heading option values. If - *option* is specified then the value for that *option* is returned. - Otherwise, sets the options to the corresponding values. - - The valid options/values are: - - * text: text - The text to display in the column heading. - * image: imageName - Specifies an image to display to the right of the column heading. - * anchor: anchor - Specifies how the heading text should be aligned. One of the standard - Tk anchor values. - * command: callback - A callback to be invoked when the heading label is pressed. - - To configure the tree column heading, call this with column = "#0". - - - .. method:: identify(component, x, y) - - Returns a description of the specified *component* under the point given - by *x* and *y*, or the empty string if no such *component* is present at - that position. - - - .. method:: identify_row(y) - - Returns the item ID of the item at position *y*. - - - .. method:: identify_column(x) - - Returns the data column identifier of the cell at position *x*. - - The tree column has ID #0. - - - .. method:: identify_region(x, y) - - Returns one of: - - +-----------+--------------------------------------+ - | region | meaning | - +===========+======================================+ - | heading | Tree heading area. | - +-----------+--------------------------------------+ - | separator | Space between two columns headings. | - +-----------+--------------------------------------+ - | tree | The tree area. | - +-----------+--------------------------------------+ - | cell | A data cell. | - +-----------+--------------------------------------+ - - Availability: Tk 8.6. - - - .. method:: identify_element(x, y) - - Returns the element at position *x*, *y*. - - Availability: Tk 8.6. - - - .. method:: index(item) - - Returns the integer index of *item* within its parent's list of children. - - - .. method:: insert(parent, index, iid=None, **kw) - - Creates a new item and returns the item identifier of the newly created - item. - - *parent* is the item ID of the parent item, or the empty string to create - a new top-level item. *index* is an integer, or the value "end", - specifying where in the list of parent's children to insert the new item. - If *index* is less than or equal to zero, the new node is inserted at - the beginning; if *index* is greater than or equal to the current number - of children, it is inserted at the end. If *iid* is specified, it is used - as the item identifier; *iid* must not already exist in the tree. - Otherwise, a new unique identifier is generated. - - See `Item Options`_ for the list of available points. - - - .. method:: item(item, option=None, **kw) - - Query or modify the options for the specified *item*. - - If no options are given, a dict with options/values for the item is - returned. - If *option* is specified then the value for that option is returned. - Otherwise, sets the options to the corresponding values as given by *kw*. - - - .. method:: move(item, parent, index) - - Moves *item* to position *index* in *parent*'s list of children. - - It is illegal to move an item under one of its descendants. If *index* is - less than or equal to zero, *item* is moved to the beginning; if greater - than or equal to the number of children, it is moved to the end. If *item* - was detached it is reattached. - - - .. method:: next(item) - - Returns the identifier of *item*'s next sibling, or '' if *item* is the - last child of its parent. - - - .. method:: parent(item) - - Returns the ID of the parent of *item*, or '' if *item* is at the top - level of the hierarchy. - - - .. method:: prev(item) - - Returns the identifier of *item*'s previous sibling, or '' if *item* is - the first child of its parent. - - - .. method:: reattach(item, parent, index) - - An alias for :meth:`Treeview.move`. - - - .. method:: see(item) - - Ensure that *item* is visible. - - Sets all of *item*'s ancestors open option to ``True``, and scrolls the - widget if necessary so that *item* is within the visible portion of - the tree. - - - .. method:: selection(selop=None, items=None) - - If *selop* is not specified, returns selected items. Otherwise, it will - act according to the following selection methods. - - .. deprecated-removed:: 3.6 3.8 - Using ``selection()`` for changing the selection state is deprecated. - Use the following selection methods instead. - - - .. method:: selection_set(*items) - - *items* becomes the new selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_add(*items) - - Add *items* to the selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_remove(*items) - - Remove *items* from the selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_toggle(*items) - - Toggle the selection state of each item in *items*. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: set(item, column=None, value=None) - - With one argument, returns a dictionary of column/value pairs for the - specified *item*. With two arguments, returns the current value of the - specified *column*. With three arguments, sets the value of given - *column* in given *item* to the specified *value*. - - - .. method:: tag_bind(tagname, sequence=None, callback=None) - - Bind a callback for the given event *sequence* to the tag *tagname*. - When an event is delivered to an item, the callbacks for each of the - item's tags option are called. - - - .. method:: tag_configure(tagname, option=None, **kw) - - Query or modify the options for the specified *tagname*. - - If *kw* is not given, returns a dict of the option settings for - *tagname*. If *option* is specified, returns the value for that *option* - for the specified *tagname*. Otherwise, sets the options to the - corresponding values for the given *tagname*. - - - .. method:: tag_has(tagname, item=None) - - If *item* is specified, returns 1 or 0 depending on whether the specified - *item* has the given *tagname*. Otherwise, returns a list of all items - that have the specified tag. - - Availability: Tk 8.6 - - - .. method:: xview(*args) - - Query or modify horizontal position of the treeview. - - - .. method:: yview(*args) - - Query or modify vertical position of the treeview. - - -.. _TtkStyling: - -Ttk Styling ------------ - -Each widget in :mod:`ttk` is assigned a style, which specifies the set of -elements making up the widget and how they are arranged, along with dynamic -and default settings for element options. By default the style name is the -same as the widget's class name, but it may be overridden by the widget's style -option. If you don't know the class name of a widget, use the method -:meth:`Misc.winfo_class` (somewidget.winfo_class()). - -.. seealso:: - - `Tcl'2004 conference presentation `_ - This document explains how the theme engine works - - -.. class:: Style - - This class is used to manipulate the style database. - - - .. method:: configure(style, query_opt=None, **kw) - - Query or set the default value of the specified option(s) in *style*. - - Each key in *kw* is an option and each value is a string identifying - the value for that option. - - For example, to change every default button to be a flat button with - some padding and a different background color:: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - ttk.Style().configure("TButton", padding=6, relief="flat", - background="#ccc") - - btn = ttk.Button(text="Sample") - btn.pack() - - root.mainloop() - - - .. method:: map(style, query_opt=None, **kw) - - Query or sets dynamic values of the specified option(s) in *style*. - - Each key in *kw* is an option and each value should be a list or a - tuple (usually) containing statespecs grouped in tuples, lists, or - some other preference. A statespec is a compound of one - or more states and then a value. - - An example may make it more understandable:: - - import tkinter - from tkinter import ttk - - root = tkinter.Tk() - - style = ttk.Style() - style.map("C.TButton", - foreground=[('pressed', 'red'), ('active', 'blue')], - background=[('pressed', '!disabled', 'black'), ('active', 'white')] - ) - - colored_btn = ttk.Button(text="Test", style="C.TButton").pack() - - root.mainloop() - - - Note that the order of the (states, value) sequences for an option does - matter, if the order is changed to ``[('active', 'blue'), ('pressed', - 'red')]`` in the foreground option, for example, the result would be a - blue foreground when the widget were in active or pressed states. - - - .. method:: lookup(style, option, state=None, default=None) - - Returns the value specified for *option* in *style*. - - If *state* is specified, it is expected to be a sequence of one or more - states. If the *default* argument is set, it is used as a fallback value - in case no specification for option is found. - - To check what font a Button uses by default:: - - from tkinter import ttk - - print(ttk.Style().lookup("TButton", "font")) - - - .. method:: layout(style, layoutspec=None) - - Define the widget layout for given *style*. If *layoutspec* is omitted, - return the layout specification for given style. - - *layoutspec*, if specified, is expected to be a list or some other - sequence type (excluding strings), where each item should be a tuple and - the first item is the layout name and the second item should have the - format described in `Layouts`_. - - To understand the format, see the following example (it is not - intended to do anything useful):: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - style = ttk.Style() - style.layout("TMenubutton", [ - ("Menubutton.background", None), - ("Menubutton.button", {"children": - [("Menubutton.focus", {"children": - [("Menubutton.padding", {"children": - [("Menubutton.label", {"side": "left", "expand": 1})] - })] - })] - }), - ]) - - mbtn = ttk.Menubutton(text='Text') - mbtn.pack() - root.mainloop() - - - .. method:: element_create(elementname, etype, *args, **kw) - - Create a new element in the current theme, of the given *etype* which is - expected to be either "image", "from" or "vsapi". The latter is only - available in Tk 8.6a for Windows XP and Vista and is not described here. - - If "image" is used, *args* should contain the default image name followed - by statespec/value pairs (this is the imagespec), and *kw* may have the - following options: - - * border=padding - padding is a list of up to four integers, specifying the left, top, - right, and bottom borders, respectively. - - * height=height - Specifies a minimum height for the element. If less than zero, the - base image's height is used as a default. - - * padding=padding - Specifies the element's interior padding. Defaults to border's value - if not specified. - - * sticky=spec - Specifies how the image is placed within the final parcel. spec - contains zero or more characters "n", "s", "w", or "e". - - * width=width - Specifies a minimum width for the element. If less than zero, the - base image's width is used as a default. - - If "from" is used as the value of *etype*, - :meth:`element_create` will clone an existing - element. *args* is expected to contain a themename, from which - the element will be cloned, and optionally an element to clone from. - If this element to clone from is not specified, an empty element will - be used. *kw* is discarded. - - - .. method:: element_names() - - Returns the list of elements defined in the current theme. - - - .. method:: element_options(elementname) - - Returns the list of *elementname*'s options. - - - .. method:: theme_create(themename, parent=None, settings=None) - - Create a new theme. - - It is an error if *themename* already exists. If *parent* is specified, - the new theme will inherit styles, elements and layouts from the parent - theme. If *settings* are present they are expected to have the same - syntax used for :meth:`theme_settings`. - - - .. method:: theme_settings(themename, settings) - - Temporarily sets the current theme to *themename*, apply specified - *settings* and then restore the previous theme. - - Each key in *settings* is a style and each value may contain the keys - 'configure', 'map', 'layout' and 'element create' and they are expected - to have the same format as specified by the methods - :meth:`Style.configure`, :meth:`Style.map`, :meth:`Style.layout` and - :meth:`Style.element_create` respectively. - - As an example, let's change the Combobox for the default theme a bit:: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - style = ttk.Style() - style.theme_settings("default", { - "TCombobox": { - "configure": {"padding": 5}, - "map": { - "background": [("active", "green2"), - ("!disabled", "green4")], - "fieldbackground": [("!disabled", "green3")], - "foreground": [("focus", "OliveDrab1"), - ("!disabled", "OliveDrab2")] - } - } - }) - - combo = ttk.Combobox().pack() - - root.mainloop() - - - .. method:: theme_names() - - Returns a list of all known themes. - - - .. method:: theme_use(themename=None) - - If *themename* is not given, returns the theme in use. Otherwise, sets - the current theme to *themename*, refreshes all widgets and emits a - <> event. - - -Layouts -^^^^^^^ - -A layout can be just ``None``, if it takes no options, or a dict of -options specifying how to arrange the element. The layout mechanism -uses a simplified version of the pack geometry manager: given an -initial cavity, each element is allocated a parcel. Valid -options/values are: - - * side: whichside - Specifies which side of the cavity to place the element; one of - top, right, bottom or left. If omitted, the element occupies the - entire cavity. - - * sticky: nswe - Specifies where the element is placed inside its allocated parcel. - - * unit: 0 or 1 - If set to 1, causes the element and all of its descendants to be treated as - a single element for the purposes of :meth:`Widget.identify` et al. It's - used for things like scrollbar thumbs with grips. - - * children: [sublayout... ] - Specifies a list of elements to place inside the element. Each - element is a tuple (or other sequence type) where the first item is - the layout name, and the other is a `Layout`_. - -.. _Layout: `Layouts`_ diff --git a/third_party/python/Doc/library/token.rst b/third_party/python/Doc/library/token.rst deleted file mode 100644 index effb71132..000000000 --- a/third_party/python/Doc/library/token.rst +++ /dev/null @@ -1,110 +0,0 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= - -.. module:: token - :synopsis: Constants representing terminal nodes of the parse tree. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/token.py` - --------------- - -This module provides constants which represent the numeric values of leaf nodes -of the parse tree (terminal tokens). Refer to the file :file:`Grammar/Grammar` -in the Python distribution for the definitions of the names in the context of -the language grammar. The specific numeric values which the names map to may -change between Python versions. - -The module also provides a mapping from numeric codes to names and some -functions. The functions mirror definitions in the Python C header files. - - -.. data:: tok_name - - Dictionary mapping the numeric values of the constants defined in this module - back to name strings, allowing more human-readable representation of parse trees - to be generated. - - -.. function:: ISTERMINAL(x) - - Return true for terminal token values. - - -.. function:: ISNONTERMINAL(x) - - Return true for non-terminal token values. - - -.. function:: ISEOF(x) - - Return true if *x* is the marker indicating the end of input. - - -The token constants are: - -.. data:: ENDMARKER - NAME - NUMBER - STRING - NEWLINE - INDENT - DEDENT - LPAR - RPAR - LSQB - RSQB - COLON - COMMA - SEMI - PLUS - MINUS - STAR - SLASH - VBAR - AMPER - LESS - GREATER - EQUAL - DOT - PERCENT - LBRACE - RBRACE - EQEQUAL - NOTEQUAL - LESSEQUAL - GREATEREQUAL - TILDE - CIRCUMFLEX - LEFTSHIFT - RIGHTSHIFT - DOUBLESTAR - PLUSEQUAL - MINEQUAL - STAREQUAL - SLASHEQUAL - PERCENTEQUAL - AMPEREQUAL - VBAREQUAL - CIRCUMFLEXEQUAL - LEFTSHIFTEQUAL - RIGHTSHIFTEQUAL - DOUBLESTAREQUAL - DOUBLESLASH - DOUBLESLASHEQUAL - AT - ATEQUAL - RARROW - ELLIPSIS - OP - AWAIT - ASYNC - ERRORTOKEN - N_TOKENS - NT_OFFSET - - .. versionchanged:: 3.5 - Added :data:`AWAIT` and :data:`ASYNC` tokens. Starting with - Python 3.7, "async" and "await" will be tokenized as :data:`NAME` - tokens, and :data:`AWAIT` and :data:`ASYNC` will be removed. diff --git a/third_party/python/Doc/library/tokenize.rst b/third_party/python/Doc/library/tokenize.rst deleted file mode 100644 index ba5b83c61..000000000 --- a/third_party/python/Doc/library/tokenize.rst +++ /dev/null @@ -1,288 +0,0 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== - -.. module:: tokenize - :synopsis: Lexical scanner for Python source code. - -.. moduleauthor:: Ka Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/tokenize.py` - --------------- - -The :mod:`tokenize` module provides a lexical scanner for Python source code, -implemented in Python. The scanner in this module returns comments as tokens -as well, making it useful for implementing "pretty-printers," including -colorizers for on-screen displays. - -To simplify token stream handling, all :ref:`operator ` and -:ref:`delimiter ` tokens and :data:`Ellipsis` are returned using -the generic :data:`~token.OP` token type. The exact -type can be determined by checking the ``exact_type`` property on the -:term:`named tuple` returned from :func:`tokenize.tokenize`. - -Tokenizing Input ----------------- - -The primary entry point is a :term:`generator`: - -.. function:: tokenize(readline) - - The :func:`.tokenize` generator requires one argument, *readline*, which - must be a callable object which provides the same interface as the - :meth:`io.IOBase.readline` method of file objects. Each call to the - function should return one line of input as bytes. - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple ``(srow, scol)`` of ints specifying the row and - column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of - ints specifying the row and column where the token ends in the source; and - the line on which the token was found. The line passed (the last tuple item) - is the *logical* line; continuation lines are included. The 5 tuple is - returned as a :term:`named tuple` with the field names: - ``type string start end line``. - - The returned :term:`named tuple` has an additional property named - ``exact_type`` that contains the exact operator type for - :data:`token.OP` tokens. For all other token types ``exact_type`` - equals the named tuple ``type`` field. - - .. versionchanged:: 3.1 - Added support for named tuples. - - .. versionchanged:: 3.3 - Added support for ``exact_type``. - - :func:`.tokenize` determines the source encoding of the file by looking for a - UTF-8 BOM or encoding cookie, according to :pep:`263`. - - -All constants from the :mod:`token` module are also exported from -:mod:`tokenize`, as are three additional token type values: - -.. data:: COMMENT - - Token value used to indicate a comment. - - -.. data:: NL - - Token value used to indicate a non-terminating newline. The NEWLINE token - indicates the end of a logical line of Python code; NL tokens are generated - when a logical line of code is continued over multiple physical lines. - - -.. data:: ENCODING - - Token value that indicates the encoding used to decode the source bytes - into text. The first token returned by :func:`.tokenize` will always be an - ENCODING token. - - -Another function is provided to reverse the tokenization process. This is -useful for creating tools that tokenize a script, modify the token stream, and -write back the modified script. - - -.. function:: untokenize(iterable) - - Converts tokens back into Python source code. The *iterable* must return - sequences with at least two elements, the token type and the token string. - Any additional sequence elements are ignored. - - The reconstructed script is returned as a single string. The result is - guaranteed to tokenize back to match the input so that the conversion is - lossless and round-trips are assured. The guarantee applies only to the - token type and token string as the spacing between tokens (column - positions) may change. - - It returns bytes, encoded using the ENCODING token, which is the first - token sequence output by :func:`.tokenize`. - - -:func:`.tokenize` needs to detect the encoding of source files it tokenizes. The -function it uses to do this is available: - -.. function:: detect_encoding(readline) - - The :func:`detect_encoding` function is used to detect the encoding that - should be used to decode a Python source file. It requires one argument, - readline, in the same way as the :func:`.tokenize` generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (not decoded from bytes) it has read - in. - - It detects the encoding from the presence of a UTF-8 BOM or an encoding - cookie as specified in :pep:`263`. If both a BOM and a cookie are present, - but disagree, a SyntaxError will be raised. Note that if the BOM is found, - ``'utf-8-sig'`` will be returned as an encoding. - - If no encoding is specified, then the default of ``'utf-8'`` will be - returned. - - Use :func:`.open` to open Python source files: it uses - :func:`detect_encoding` to detect the file encoding. - - -.. function:: open(filename) - - Open a file in read only mode using the encoding detected by - :func:`detect_encoding`. - - .. versionadded:: 3.2 - -.. exception:: TokenError - - Raised when either a docstring or expression that may be split over several - lines is not completed anywhere in the file, for example:: - - """Beginning of - docstring - - or:: - - [1, - 2, - 3 - -Note that unclosed single-quoted strings do not cause an error to be -raised. They are tokenized as ``ERRORTOKEN``, followed by the tokenization of -their contents. - - -.. _tokenize-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.3 - -The :mod:`tokenize` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m tokenize [-e] [filename.py] - -The following options are accepted: - -.. program:: tokenize - -.. cmdoption:: -h, --help - - show this help message and exit - -.. cmdoption:: -e, --exact - - display token names using the exact type - -If :file:`filename.py` is specified its contents are tokenized to stdout. -Otherwise, tokenization is performed on stdin. - -Examples ------------------- - -Example of a script rewriter that transforms float literals into Decimal -objects:: - - from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP - from io import BytesIO - - def decistmt(s): - """Substitute Decimals for floats in a string of statements. - - >>> from decimal import Decimal - >>> s = 'print(+21.3e-5*-.1234/81.7)' - >>> decistmt(s) - "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" - - The format of the exponent is inherited from the platform C library. - Known cases are "e-007" (Windows) and "e-07" (not Windows). Since - we're only showing 12 digits, and the 13th isn't close to 5, the - rest of the output should be platform-independent. - - >>> exec(s) #doctest: +ELLIPSIS - -3.21716034272e-0...7 - - Output from calculations with Decimal should be identical across all - platforms. - - >>> exec(decistmt(s)) - -3.217160342717258261933904529E-7 - """ - result = [] - g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string - for toknum, tokval, _, _, _ in g: - if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens - result.extend([ - (NAME, 'Decimal'), - (OP, '('), - (STRING, repr(tokval)), - (OP, ')') - ]) - else: - result.append((toknum, tokval)) - return untokenize(result).decode('utf-8') - -Example of tokenizing from the command line. The script:: - - def say_hello(): - print("Hello, World!") - - say_hello() - -will be tokenized to the following output where the first column is the range -of the line/column coordinates where the token is found, the second column is -the name of the token, and the final column is the value of the token (if any) - -.. code-block:: shell-session - - $ python -m tokenize hello.py - 0,0-0,0: ENCODING 'utf-8' - 1,0-1,3: NAME 'def' - 1,4-1,13: NAME 'say_hello' - 1,13-1,14: OP '(' - 1,14-1,15: OP ')' - 1,15-1,16: OP ':' - 1,16-1,17: NEWLINE '\n' - 2,0-2,4: INDENT ' ' - 2,4-2,9: NAME 'print' - 2,9-2,10: OP '(' - 2,10-2,25: STRING '"Hello, World!"' - 2,25-2,26: OP ')' - 2,26-2,27: NEWLINE '\n' - 3,0-3,1: NL '\n' - 4,0-4,0: DEDENT '' - 4,0-4,9: NAME 'say_hello' - 4,9-4,10: OP '(' - 4,10-4,11: OP ')' - 4,11-4,12: NEWLINE '\n' - 5,0-5,0: ENDMARKER '' - -The exact token type names can be displayed using the ``-e`` option: - -.. code-block:: shell-session - - $ python -m tokenize -e hello.py - 0,0-0,0: ENCODING 'utf-8' - 1,0-1,3: NAME 'def' - 1,4-1,13: NAME 'say_hello' - 1,13-1,14: LPAR '(' - 1,14-1,15: RPAR ')' - 1,15-1,16: COLON ':' - 1,16-1,17: NEWLINE '\n' - 2,0-2,4: INDENT ' ' - 2,4-2,9: NAME 'print' - 2,9-2,10: LPAR '(' - 2,10-2,25: STRING '"Hello, World!"' - 2,25-2,26: RPAR ')' - 2,26-2,27: NEWLINE '\n' - 3,0-3,1: NL '\n' - 4,0-4,0: DEDENT '' - 4,0-4,9: NAME 'say_hello' - 4,9-4,10: LPAR '(' - 4,10-4,11: RPAR ')' - 4,11-4,12: NEWLINE '\n' - 5,0-5,0: ENDMARKER '' diff --git a/third_party/python/Doc/library/trace.rst b/third_party/python/Doc/library/trace.rst deleted file mode 100644 index 5cb7029ad..000000000 --- a/third_party/python/Doc/library/trace.rst +++ /dev/null @@ -1,213 +0,0 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== - -.. module:: trace - :synopsis: Trace or track Python statement execution. - -**Source code:** :source:`Lib/trace.py` - --------------- - -The :mod:`trace` module allows you to trace program execution, generate -annotated statement coverage listings, print caller/callee relationships and -list functions executed during a program run. It can be used in another program -or from the command line. - -.. seealso:: - - `Coverage.py `_ - A popular third-party coverage tool that provides HTML - output along with advanced features such as branch coverage. - -.. _trace-cli: - -Command-Line Usage ------------------- - -The :mod:`trace` module can be invoked from the command line. It can be as -simple as :: - - python -m trace --count -C . somefile.py ... - -The above will execute :file:`somefile.py` and generate annotated listings of -all Python modules imported during the execution into the current directory. - -.. program:: trace - -.. cmdoption:: --help - - Display usage and exit. - -.. cmdoption:: --version - - Display the version of the module and exit. - -Main options -^^^^^^^^^^^^ - -At least one of the following options must be specified when invoking -:mod:`trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with -the :option:`--trace <-t>` and :option:`--count <-c>` options. When -:option:`--listfuncs <-l>` is provided, neither :option:`--count <-c>` nor -:option:`--trace <-t>` are accepted, and vice versa. - -.. program:: trace - -.. cmdoption:: -c, --count - - Produce a set of annotated listing files upon program completion that shows - how many times each statement was executed. See also - :option:`--coverdir <-C>`, :option:`--file <-f>` and - :option:`--no-report <-R>` below. - -.. cmdoption:: -t, --trace - - Display lines as they are executed. - -.. cmdoption:: -l, --listfuncs - - Display the functions executed by running the program. - -.. cmdoption:: -r, --report - - Produce an annotated list from an earlier program run that used the - :option:`--count <-c>` and :option:`--file <-f>` option. This does not - execute any code. - -.. cmdoption:: -T, --trackcalls - - Display the calling relationships exposed by running the program. - -Modifiers -^^^^^^^^^ - -.. program:: trace - -.. cmdoption:: -f, --file= - - Name of a file to accumulate counts over several tracing runs. Should be - used with the :option:`--count <-c>` option. - -.. cmdoption:: -C, --coverdir= - - Directory where the report files go. The coverage report for - ``package.module`` is written to file :file:`{dir}/{package}/{module}.cover`. - -.. cmdoption:: -m, --missing - - When generating annotated listings, mark lines which were not executed with - ``>>>>>>``. - -.. cmdoption:: -s, --summary - - When using :option:`--count <-c>` or :option:`--report <-r>`, write a brief - summary to stdout for each file processed. - -.. cmdoption:: -R, --no-report - - Do not generate annotated listings. This is useful if you intend to make - several runs with :option:`--count <-c>`, and then produce a single set of - annotated listings at the end. - -.. cmdoption:: -g, --timing - - Prefix each line with the time since the program started. Only used while - tracing. - -Filters -^^^^^^^ - -These options may be repeated multiple times. - -.. program:: trace - -.. cmdoption:: --ignore-module= - - Ignore each of the given module names and its submodules (if it is a - package). The argument can be a list of names separated by a comma. - -.. cmdoption:: --ignore-dir= - - Ignore all modules and packages in the named directory and subdirectories. - The argument can be a list of directories separated by :data:`os.pathsep`. - -.. _trace-api: - -Programmatic Interface ----------------------- - -.. class:: Trace(count=1, trace=1, countfuncs=0, countcallers=0, ignoremods=(),\ - ignoredirs=(), infile=None, outfile=None, timing=False) - - Create an object to trace execution of a single statement or expression. All - parameters are optional. *count* enables counting of line numbers. *trace* - enables line execution tracing. *countfuncs* enables listing of the - functions called during the run. *countcallers* enables call relationship - tracking. *ignoremods* is a list of modules or packages to ignore. - *ignoredirs* is a list of directories whose modules or packages should be - ignored. *infile* is the name of the file from which to read stored count - information. *outfile* is the name of the file in which to write updated - count information. *timing* enables a timestamp relative to when tracing was - started to be displayed. - - .. method:: run(cmd) - - Execute the command and gather statistics from the execution with - the current tracing parameters. *cmd* must be a string or code object, - suitable for passing into :func:`exec`. - - .. method:: runctx(cmd, globals=None, locals=None) - - Execute the command and gather statistics from the execution with the - current tracing parameters, in the defined global and local - environments. If not defined, *globals* and *locals* default to empty - dictionaries. - - .. method:: runfunc(func, *args, **kwds) - - Call *func* with the given arguments under control of the :class:`Trace` - object with the current tracing parameters. - - .. method:: results() - - Return a :class:`CoverageResults` object that contains the cumulative - results of all previous calls to ``run``, ``runctx`` and ``runfunc`` - for the given :class:`Trace` instance. Does not reset the accumulated - trace results. - -.. class:: CoverageResults - - A container for coverage results, created by :meth:`Trace.results`. Should - not be created directly by the user. - - .. method:: update(other) - - Merge in data from another :class:`CoverageResults` object. - - .. method:: write_results(show_missing=True, summary=False, coverdir=None) - - Write coverage results. Set *show_missing* to show lines that had no - hits. Set *summary* to include in the output the coverage summary per - module. *coverdir* specifies the directory into which the coverage - result files will be output. If ``None``, the results for each source - file are placed in its directory. - -A simple example demonstrating the use of the programmatic interface:: - - import sys - import trace - - # create a Trace object, telling it what to ignore, and whether to - # do tracing or line-counting or both. - tracer = trace.Trace( - ignoredirs=[sys.prefix, sys.exec_prefix], - trace=0, - count=1) - - # run the new command using the given tracer - tracer.run('main()') - - # make a report, placing output in the current directory - r = tracer.results() - r.write_results(show_missing=True, coverdir=".") - diff --git a/third_party/python/Doc/library/traceback.rst b/third_party/python/Doc/library/traceback.rst deleted file mode 100644 index 462a6a556..000000000 --- a/third_party/python/Doc/library/traceback.rst +++ /dev/null @@ -1,486 +0,0 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== - -.. module:: traceback - :synopsis: Print or retrieve a stack traceback. - -**Source code:** :source:`Lib/traceback.py` - --------------- - -This module provides a standard interface to extract, format and print stack -traces of Python programs. It exactly mimics the behavior of the Python -interpreter when it prints a stack trace. This is useful when you want to print -stack traces under program control, such as in a "wrapper" around the -interpreter. - -.. index:: object: traceback - -The module uses traceback objects --- this is the object type that is stored in -the :data:`sys.last_traceback` variable and returned as the third item from -:func:`sys.exc_info`. - -The module defines the following functions: - - -.. function:: print_tb(tb, limit=None, file=None) - - Print up to *limit* stack trace entries from traceback object *tb* (starting - from the caller's frame) if *limit* is positive. Otherwise, print the last - ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are - printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to - receive the output. - - .. versionchanged:: 3.5 - Added negative *limit* support. - - -.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True) - - Print exception information and stack trace entries from traceback object - *tb* to *file*. This differs from :func:`print_tb` in the following - ways: - - * if *tb* is not ``None``, it prints a header ``Traceback (most recent - call last):`` - - * it prints the exception *etype* and *value* after the stack trace - - .. index:: single: ^ (caret); marker - - * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate - format, it prints the line where the syntax error occurred with a caret - indicating the approximate position of the error. - - The optional *limit* argument has the same meaning as for :func:`print_tb`. - If *chain* is true (the default), then chained exceptions (the - :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be - printed as well, like the interpreter itself does when printing an unhandled - exception. - - .. versionchanged:: 3.5 - The *etype* argument is ignored and inferred from the type of *value*. - - -.. function:: print_exc(limit=None, file=None, chain=True) - - This is a shorthand for ``print_exception(*sys.exc_info(), limit, file, - chain)``. - - -.. function:: print_last(limit=None, file=None, chain=True) - - This is a shorthand for ``print_exception(sys.last_type, sys.last_value, - sys.last_traceback, limit, file, chain)``. In general it will work only - after an exception has reached an interactive prompt (see - :data:`sys.last_type`). - - -.. function:: print_stack(f=None, limit=None, file=None) - - Print up to *limit* stack trace entries (starting from the invocation - point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` - entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame - to start. The optional *file* argument has the same meaning as for - :func:`print_tb`. - - .. versionchanged:: 3.5 - Added negative *limit* support. - - -.. function:: extract_tb(tb, limit=None) - - Return a :class:`StackSummary` object representing a list of "pre-processed" - stack trace entries extracted from the traceback object *tb*. It is useful - for alternate formatting of stack traces. The optional *limit* argument has - the same meaning as for :func:`print_tb`. A "pre-processed" stack trace - entry is a :class:`FrameSummary` object containing attributes - :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, - :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the - information that is usually printed for a stack trace. The - :attr:`~FrameSummary.line` is a string with leading and trailing - whitespace stripped; if the source is not available it is ``None``. - - -.. function:: extract_stack(f=None, limit=None) - - Extract the raw traceback from the current stack frame. The return value has - the same format as for :func:`extract_tb`. The optional *f* and *limit* - arguments have the same meaning as for :func:`print_stack`. - - -.. function:: format_list(extracted_list) - - Given a list of tuples or :class:`FrameSummary` objects as returned by - :func:`extract_tb` or :func:`extract_stack`, return a list of strings ready - for printing. Each string in the resulting list corresponds to the item with - the same index in the argument list. Each string ends in a newline; the - strings may contain internal newlines as well, for those items whose source - text line is not ``None``. - - -.. function:: format_exception_only(etype, value) - - Format the exception part of a traceback. The arguments are the exception - type and value such as given by ``sys.last_type`` and ``sys.last_value``. - The return value is a list of strings, each ending in a newline. Normally, - the list contains a single string; however, for :exc:`SyntaxError` - exceptions, it contains several lines that (when printed) display detailed - information about where the syntax error occurred. The message indicating - which exception occurred is the always last string in the list. - - -.. function:: format_exception(etype, value, tb, limit=None, chain=True) - - Format a stack trace and the exception information. The arguments have the - same meaning as the corresponding arguments to :func:`print_exception`. The - return value is a list of strings, each ending in a newline and some - containing internal newlines. When these lines are concatenated and printed, - exactly the same text is printed as does :func:`print_exception`. - - .. versionchanged:: 3.5 - The *etype* argument is ignored and inferred from the type of *value*. - - -.. function:: format_exc(limit=None, chain=True) - - This is like ``print_exc(limit)`` but returns a string instead of printing to - a file. - - -.. function:: format_tb(tb, limit=None) - - A shorthand for ``format_list(extract_tb(tb, limit))``. - - -.. function:: format_stack(f=None, limit=None) - - A shorthand for ``format_list(extract_stack(f, limit))``. - -.. function:: clear_frames(tb) - - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. - - .. versionadded:: 3.4 - -.. function:: walk_stack(f) - - Walk a stack following ``f.f_back`` from the given frame, yielding the frame - and line number for each frame. If *f* is ``None``, the current stack is - used. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -.. function:: walk_tb(tb) - - Walk a traceback following ``tb_next`` yielding the frame and line number - for each frame. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -The module also defines the following classes: - -:class:`TracebackException` Objects ------------------------------------ - -.. versionadded:: 3.5 - -:class:`TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. - -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - Note that when locals are captured, they are also shown in the traceback. - - .. attribute:: __cause__ - - A :class:`TracebackException` of the original ``__cause__``. - - .. attribute:: __context__ - - A :class:`TracebackException` of the original ``__context__``. - - .. attribute:: __suppress_context__ - - The ``__suppress_context__`` value from the original exception. - - .. attribute:: stack - - A :class:`StackSummary` representing the traceback. - - .. attribute:: exc_type - - The class of the original traceback. - - .. attribute:: filename - - For syntax errors - the file name where the error occurred. - - .. attribute:: lineno - - For syntax errors - the line number where the error occurred. - - .. attribute:: text - - For syntax errors - the text where the error occurred. - - .. attribute:: offset - - For syntax errors - the offset into the text where the error occurred. - - .. attribute:: msg - - For syntax errors - the compiler error message. - - .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - Note that when locals are captured, they are also shown in the traceback. - - .. method:: format(*, chain=True) - - Format the exception. - - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. - - The return value is a generator of strings, each ending in a newline and - some containing internal newlines. :func:`~traceback.print_exception` - is a wrapper around this method which just prints the lines to a file. - - The message indicating which exception occurred is always the last - string in the output. - - .. method:: format_exception_only() - - Format the exception part of the traceback. - - The return value is a generator of strings, each ending in a newline. - - Normally, the generator emits a single string; however, for - :exc:`SyntaxError` exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. - - -:class:`StackSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -:class:`StackSummary` objects represent a call stack ready for formatting. - -.. class:: StackSummary - - .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - - Construct a :class:`StackSummary` object from a frame generator (such as - is returned by :func:`~traceback.walk_stack` or - :func:`~traceback.walk_tb`). - - If *limit* is supplied, only this many frames are taken from *frame_gen*. - If *lookup_lines* is ``False``, the returned :class:`FrameSummary` - objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it - may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object - representations. - - .. classmethod:: from_list(a_list) - - Construct a :class:`StackSummary` object from a supplied list of - :class:`FrameSummary` objects or old-style list of tuples. Each tuple - should be a 4-tuple with filename, lineno, name, line as the elements. - - .. method:: format() - - Returns a list of strings ready for printing. Each string in the - resulting list corresponds to a single frame from the stack. - Each string ends in a newline; the strings may contain internal - newlines as well, for those items with source text lines. - - For long sequences of the same frame and line, the first few - repetitions are shown, followed by a summary line stating the exact - number of further repetitions. - - .. versionchanged:: 3.6 - Long sequences of repeated frames are now abbreviated. - - -:class:`FrameSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -:class:`FrameSummary` objects represent a single frame in a traceback. - -.. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames - locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). - :attr:`~FrameSummary.line` may be directly provided, and will prevent line - lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the - summary for later display. - -.. _traceback-example: - -Traceback Examples ------------------- - -This simple example implements a basic read-eval-print loop, similar to (but -less useful than) the standard Python interactive interpreter loop. For a more -complete implementation of the interpreter loop, refer to the :mod:`code` -module. :: - - import sys, traceback - - def run_user_code(envdir): - source = input(">>> ") - try: - exec(source, envdir) - except Exception: - print("Exception in user code:") - print("-"*60) - traceback.print_exc(file=sys.stdout) - print("-"*60) - - envdir = {} - while True: - run_user_code(envdir) - - -The following example demonstrates the different ways to print and format the -exception and traceback: - -.. testcode:: - - import sys, traceback - - def lumberjack(): - bright_side_of_death() - - def bright_side_of_death(): - return tuple()[0] - - try: - lumberjack() - except IndexError: - exc_type, exc_value, exc_traceback = sys.exc_info() - print("*** print_tb:") - traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) - print("*** print_exception:") - # exc_type below is ignored on 3.5 and later - traceback.print_exception(exc_type, exc_value, exc_traceback, - limit=2, file=sys.stdout) - print("*** print_exc:") - traceback.print_exc(limit=2, file=sys.stdout) - print("*** format_exc, first and last line:") - formatted_lines = traceback.format_exc().splitlines() - print(formatted_lines[0]) - print(formatted_lines[-1]) - print("*** format_exception:") - # exc_type below is ignored on 3.5 and later - print(repr(traceback.format_exception(exc_type, exc_value, - exc_traceback))) - print("*** extract_tb:") - print(repr(traceback.extract_tb(exc_traceback))) - print("*** format_tb:") - print(repr(traceback.format_tb(exc_traceback))) - print("*** tb_lineno:", exc_traceback.tb_lineno) - -The output for the example would look similar to this: - -.. testoutput:: - :options: +NORMALIZE_WHITESPACE - - *** print_tb: - File "", line 10, in - lumberjack() - *** print_exception: - Traceback (most recent call last): - File "", line 10, in - lumberjack() - File "", line 4, in lumberjack - bright_side_of_death() - IndexError: tuple index out of range - *** print_exc: - Traceback (most recent call last): - File "", line 10, in - lumberjack() - File "", line 4, in lumberjack - bright_side_of_death() - IndexError: tuple index out of range - *** format_exc, first and last line: - Traceback (most recent call last): - IndexError: tuple index out of range - *** format_exception: - ['Traceback (most recent call last):\n', - ' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_death()\n', - ' File "", line 7, in bright_side_of_death\n return tuple()[0]\n', - 'IndexError: tuple index out of range\n'] - *** extract_tb: - [, line 10 in >, - , line 4 in lumberjack>, - , line 7 in bright_side_of_death>] - *** format_tb: - [' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_death()\n', - ' File "", line 7, in bright_side_of_death\n return tuple()[0]\n'] - *** tb_lineno: 10 - - -The following example shows the different ways to print and format the stack:: - - >>> import traceback - >>> def another_function(): - ... lumberstack() - ... - >>> def lumberstack(): - ... traceback.print_stack() - ... print(repr(traceback.extract_stack())) - ... print(repr(traceback.format_stack())) - ... - >>> another_function() - File "", line 10, in - another_function() - File "", line 3, in another_function - lumberstack() - File "", line 6, in lumberstack - traceback.print_stack() - [('', 10, '', 'another_function()'), - ('', 3, 'another_function', 'lumberstack()'), - ('', 7, 'lumberstack', 'print(repr(traceback.extract_stack()))')] - [' File "", line 10, in \n another_function()\n', - ' File "", line 3, in another_function\n lumberstack()\n', - ' File "", line 8, in lumberstack\n print(repr(traceback.format_stack()))\n'] - - -This last example demonstrates the final few formatting functions: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> import traceback - >>> traceback.format_list([('spam.py', 3, '', 'spam.eggs()'), - ... ('eggs.py', 42, 'eggs', 'return "bacon"')]) - [' File "spam.py", line 3, in \n spam.eggs()\n', - ' File "eggs.py", line 42, in eggs\n return "bacon"\n'] - >>> an_error = IndexError('tuple index out of range') - >>> traceback.format_exception_only(type(an_error), an_error) - ['IndexError: tuple index out of range\n'] diff --git a/third_party/python/Doc/library/tracemalloc.rst b/third_party/python/Doc/library/tracemalloc.rst deleted file mode 100644 index e16566963..000000000 --- a/third_party/python/Doc/library/tracemalloc.rst +++ /dev/null @@ -1,674 +0,0 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== - -.. module:: tracemalloc - :synopsis: Trace memory allocations. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/tracemalloc.py` - --------------- - -The tracemalloc module is a debug tool to trace memory blocks allocated by -Python. It provides the following information: - -* Traceback where an object was allocated -* Statistics on allocated memory blocks per filename and per line number: - total size, number and average size of allocated memory blocks -* Compute the differences between two snapshots to detect memory leaks - -To trace most memory blocks allocated by Python, the module should be started -as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment -variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line -option. The :func:`tracemalloc.start` function can be called at runtime to -start tracing Python memory allocations. - -By default, a trace of an allocated memory block only stores the most recent -frame (1 frame). To store 25 frames at startup: set the -:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the -:option:`-X` ``tracemalloc=25`` command line option. - - -Examples --------- - -Display the top 10 -^^^^^^^^^^^^^^^^^^ - -Display the 10 files allocating the most memory:: - - import tracemalloc - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('lineno') - - print("[ Top 10 ]") - for stat in top_stats[:10]: - print(stat) - - -Example of output of the Python test suite:: - - [ Top 10 ] - :716: size=4855 KiB, count=39328, average=126 B - :284: size=521 KiB, count=3199, average=167 B - /usr/lib/python3.4/collections/__init__.py:368: size=244 KiB, count=2315, average=108 B - /usr/lib/python3.4/unittest/case.py:381: size=185 KiB, count=779, average=243 B - /usr/lib/python3.4/unittest/case.py:402: size=154 KiB, count=378, average=416 B - /usr/lib/python3.4/abc.py:133: size=88.7 KiB, count=347, average=262 B - :1446: size=70.4 KiB, count=911, average=79 B - :1454: size=52.0 KiB, count=25, average=2131 B - :5: size=49.7 KiB, count=148, average=344 B - /usr/lib/python3.4/sysconfig.py:411: size=48.0 KiB, count=1, average=48.0 KiB - -We can see that Python loaded ``4855 KiB`` data (bytecode and constants) from -modules and that the :mod:`collections` module allocated ``244 KiB`` to build -:class:`~collections.namedtuple` types. - -See :meth:`Snapshot.statistics` for more options. - - -Compute differences -^^^^^^^^^^^^^^^^^^^ - -Take two snapshots and display the differences:: - - import tracemalloc - tracemalloc.start() - # ... start your application ... - - snapshot1 = tracemalloc.take_snapshot() - # ... call the function leaking memory ... - snapshot2 = tracemalloc.take_snapshot() - - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - - print("[ Top 10 differences ]") - for stat in top_stats[:10]: - print(stat) - -Example of output before/after running some tests of the Python test suite:: - - [ Top 10 differences ] - :716: size=8173 KiB (+4428 KiB), count=71332 (+39369), average=117 B - /usr/lib/python3.4/linecache.py:127: size=940 KiB (+940 KiB), count=8106 (+8106), average=119 B - /usr/lib/python3.4/unittest/case.py:571: size=298 KiB (+298 KiB), count=589 (+589), average=519 B - :284: size=1005 KiB (+166 KiB), count=7423 (+1526), average=139 B - /usr/lib/python3.4/mimetypes.py:217: size=112 KiB (+112 KiB), count=1334 (+1334), average=86 B - /usr/lib/python3.4/http/server.py:848: size=96.0 KiB (+96.0 KiB), count=1 (+1), average=96.0 KiB - /usr/lib/python3.4/inspect.py:1465: size=83.5 KiB (+83.5 KiB), count=109 (+109), average=784 B - /usr/lib/python3.4/unittest/mock.py:491: size=77.7 KiB (+77.7 KiB), count=143 (+143), average=557 B - /usr/lib/python3.4/urllib/parse.py:476: size=71.8 KiB (+71.8 KiB), count=969 (+969), average=76 B - /usr/lib/python3.4/contextlib.py:38: size=67.2 KiB (+67.2 KiB), count=126 (+126), average=546 B - -We can see that Python has loaded ``8173 KiB`` of module data (bytecode and -constants), and that this is ``4428 KiB`` more than had been loaded before the -tests, when the previous snapshot was taken. Similarly, the :mod:`linecache` -module has cached ``940 KiB`` of Python source code to format tracebacks, all -of it since the previous snapshot. - -If the system has little free memory, snapshots can be written on disk using -the :meth:`Snapshot.dump` method to analyze the snapshot offline. Then use the -:meth:`Snapshot.load` method reload the snapshot. - - -Get the traceback of a memory block -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Code to display the traceback of the biggest memory block:: - - import tracemalloc - - # Store 25 frames - tracemalloc.start(25) - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('traceback') - - # pick the biggest memory block - stat = top_stats[0] - print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024)) - for line in stat.traceback.format(): - print(line) - -Example of output of the Python test suite (traceback limited to 25 frames):: - - 903 memory blocks: 870.1 KiB - File "", line 716 - File "", line 1036 - File "", line 934 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/doctest.py", line 101 - import pdb - File "", line 284 - File "", line 938 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/test/support/__init__.py", line 1728 - import doctest - File "/usr/lib/python3.4/test/test_pickletools.py", line 21 - support.run_doctest(pickletools) - File "/usr/lib/python3.4/test/regrtest.py", line 1276 - test_runner() - File "/usr/lib/python3.4/test/regrtest.py", line 976 - display_failure=not verbose) - File "/usr/lib/python3.4/test/regrtest.py", line 761 - match_tests=ns.match_tests) - File "/usr/lib/python3.4/test/regrtest.py", line 1563 - main() - File "/usr/lib/python3.4/test/__main__.py", line 3 - regrtest.main_in_temp_cwd() - File "/usr/lib/python3.4/runpy.py", line 73 - exec(code, run_globals) - File "/usr/lib/python3.4/runpy.py", line 160 - "__main__", fname, loader, pkg_name) - -We can see that the most memory was allocated in the :mod:`importlib` module to -load data (bytecode and constants) from modules: ``870.1 KiB``. The traceback is -where the :mod:`importlib` loaded data most recently: on the ``import pdb`` -line of the :mod:`doctest` module. The traceback may change if a new module is -loaded. - - -Pretty top -^^^^^^^^^^ - -Code to display the 10 lines allocating the most memory with a pretty output, -ignoring ```` and ```` files:: - - import linecache - import os - import tracemalloc - - def display_top(snapshot, key_type='lineno', limit=10): - snapshot = snapshot.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - )) - top_stats = snapshot.statistics(key_type) - - print("Top %s lines" % limit) - for index, stat in enumerate(top_stats[:limit], 1): - frame = stat.traceback[0] - # replace "/path/to/module/file.py" with "module/file.py" - filename = os.sep.join(frame.filename.split(os.sep)[-2:]) - print("#%s: %s:%s: %.1f KiB" - % (index, filename, frame.lineno, stat.size / 1024)) - line = linecache.getline(frame.filename, frame.lineno).strip() - if line: - print(' %s' % line) - - other = top_stats[limit:] - if other: - size = sum(stat.size for stat in other) - print("%s other: %.1f KiB" % (len(other), size / 1024)) - total = sum(stat.size for stat in top_stats) - print("Total allocated size: %.1f KiB" % (total / 1024)) - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - display_top(snapshot) - -Example of output of the Python test suite:: - - Top 10 lines - #1: Lib/base64.py:414: 419.8 KiB - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - #2: Lib/base64.py:306: 419.8 KiB - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - #3: collections/__init__.py:368: 293.6 KiB - exec(class_definition, namespace) - #4: Lib/abc.py:133: 115.2 KiB - cls = super().__new__(mcls, name, bases, namespace) - #5: unittest/case.py:574: 103.1 KiB - testMethod() - #6: Lib/linecache.py:127: 95.4 KiB - lines = fp.readlines() - #7: urllib/parse.py:476: 71.8 KiB - for a in _hexdig for b in _hexdig} - #8: :5: 62.0 KiB - #9: Lib/_weakrefset.py:37: 60.0 KiB - self.data = set() - #10: Lib/base64.py:142: 59.8 KiB - _b32tab2 = [a + b for a in _b32tab for b in _b32tab] - 6220 other: 3602.8 KiB - Total allocated size: 5303.1 KiB - -See :meth:`Snapshot.statistics` for more options. - - -API ---- - -Functions -^^^^^^^^^ - -.. function:: clear_traces() - - Clear traces of memory blocks allocated by Python. - - See also :func:`stop`. - - -.. function:: get_object_traceback(obj) - - Get the traceback where the Python object *obj* was allocated. - Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` - module is not tracing memory allocations or did not trace the allocation of - the object. - - See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. - - -.. function:: get_traceback_limit() - - Get the maximum number of frames stored in the traceback of a trace. - - The :mod:`tracemalloc` module must be tracing memory allocations to - get the limit, otherwise an exception is raised. - - The limit is set by the :func:`start` function. - - -.. function:: get_traced_memory() - - Get the current size and peak size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. - - -.. function:: get_tracemalloc_memory() - - Get the memory usage in bytes of the :mod:`tracemalloc` module used to store - traces of memory blocks. - Return an :class:`int`. - - -.. function:: is_tracing() - - ``True`` if the :mod:`tracemalloc` module is tracing Python memory - allocations, ``False`` otherwise. - - See also :func:`start` and :func:`stop` functions. - - -.. function:: start(nframe: int=1) - - Start tracing Python memory allocations: install hooks on Python memory - allocators. Collected tracebacks of traces will be limited to *nframe* - frames. By default, a trace of a memory block only stores the most recent - frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. - - Storing more than ``1`` frame is only useful to compute statistics grouped - by ``'traceback'`` or to compute cumulative statistics: see the - :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. - - Storing more frames increases the memory and CPU overhead of the - :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function - to measure how much memory is used by the :mod:`tracemalloc` module. - - The :envvar:`PYTHONTRACEMALLOC` environment variable - (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` - command line option can be used to start tracing at startup. - - See also :func:`stop`, :func:`is_tracing` and :func:`get_traceback_limit` - functions. - - -.. function:: stop() - - Stop tracing Python memory allocations: uninstall hooks on Python memory - allocators. Also clears all previously collected traces of memory blocks - allocated by Python. - - Call :func:`take_snapshot` function to take a snapshot of traces before - clearing them. - - See also :func:`start`, :func:`is_tracing` and :func:`clear_traces` - functions. - - -.. function:: take_snapshot() - - Take a snapshot of traces of memory blocks allocated by Python. Return a new - :class:`Snapshot` instance. - - The snapshot does not include memory blocks allocated before the - :mod:`tracemalloc` module started to trace memory allocations. - - Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use - the *nframe* parameter of the :func:`start` function to store more frames. - - The :mod:`tracemalloc` module must be tracing memory allocations to take a - snapshot, see the :func:`start` function. - - See also the :func:`get_object_traceback` function. - - -DomainFilter -^^^^^^^^^^^^ - -.. class:: DomainFilter(inclusive: bool, domain: int) - - Filter traces of memory blocks by their address space (domain). - - .. versionadded:: 3.6 - - .. attribute:: inclusive - - If *inclusive* is ``True`` (include), match memory blocks allocated - in the address space :attr:`domain`. - - If *inclusive* is ``False`` (exclude), match memory blocks not allocated - in the address space :attr:`domain`. - - .. attribute:: domain - - Address space of a memory block (``int``). Read-only property. - - -Filter -^^^^^^ - -.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False, domain: int=None) - - Filter on traces of memory blocks. - - See the :func:`fnmatch.fnmatch` function for the syntax of - *filename_pattern*. The ``'.pyc'`` file extension is - replaced with ``'.py'``. - - Examples: - - * ``Filter(True, subprocess.__file__)`` only includes traces of the - :mod:`subprocess` module - * ``Filter(False, tracemalloc.__file__)`` excludes traces of the - :mod:`tracemalloc` module - * ``Filter(False, "")`` excludes empty tracebacks - - - .. versionchanged:: 3.5 - The ``'.pyo'`` file extension is no longer replaced with ``'.py'``. - - .. versionchanged:: 3.6 - Added the :attr:`domain` attribute. - - - .. attribute:: domain - - Address space of a memory block (``int`` or ``None``). - - .. attribute:: inclusive - - If *inclusive* is ``True`` (include), only match memory blocks allocated - in a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in - a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - .. attribute:: lineno - - Line number (``int``) of the filter. If *lineno* is ``None``, the filter - matches any line number. - - .. attribute:: filename_pattern - - Filename pattern of the filter (``str``). Read-only property. - - .. attribute:: all_frames - - If *all_frames* is ``True``, all frames of the traceback are checked. If - *all_frames* is ``False``, only the most recent frame is checked. - - This attribute has no effect if the traceback limit is ``1``. See the - :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit` - attribute. - - -Frame -^^^^^ - -.. class:: Frame - - Frame of a traceback. - - The :class:`Traceback` class is a sequence of :class:`Frame` instances. - - .. attribute:: filename - - Filename (``str``). - - .. attribute:: lineno - - Line number (``int``). - - -Snapshot -^^^^^^^^ - -.. class:: Snapshot - - Snapshot of traces of memory blocks allocated by Python. - - The :func:`take_snapshot` function creates a snapshot instance. - - .. method:: compare_to(old_snapshot: Snapshot, key_type: str, cumulative: bool=False) - - Compute the differences with an old snapshot. Get statistics as a sorted - list of :class:`StatisticDiff` instances grouped by *key_type*. - - See the :meth:`Snapshot.statistics` method for *key_type* and *cumulative* - parameters. - - The result is sorted from the biggest to the smallest by: absolute value - of :attr:`StatisticDiff.size_diff`, :attr:`StatisticDiff.size`, absolute - value of :attr:`StatisticDiff.count_diff`, :attr:`Statistic.count` and - then by :attr:`StatisticDiff.traceback`. - - - .. method:: dump(filename) - - Write the snapshot into a file. - - Use :meth:`load` to reload the snapshot. - - - .. method:: filter_traces(filters) - - Create a new :class:`Snapshot` instance with a filtered :attr:`traces` - sequence, *filters* is a list of :class:`DomainFilter` and - :class:`Filter` instances. If *filters* is an empty list, return a new - :class:`Snapshot` instance with a copy of the traces. - - All inclusive filters are applied at once, a trace is ignored if no - inclusive filters match it. A trace is ignored if at least one exclusive - filter matches it. - - .. versionchanged:: 3.6 - :class:`DomainFilter` instances are now also accepted in *filters*. - - - .. classmethod:: load(filename) - - Load a snapshot from a file. - - See also :meth:`dump`. - - - .. method:: statistics(key_type: str, cumulative: bool=False) - - Get statistics as a sorted list of :class:`Statistic` instances grouped - by *key_type*: - - ===================== ======================== - key_type description - ===================== ======================== - ``'filename'`` filename - ``'lineno'`` filename and line number - ``'traceback'`` traceback - ===================== ======================== - - If *cumulative* is ``True``, cumulate size and count of memory blocks of - all frames of the traceback of a trace, not only the most recent frame. - The cumulative mode can only be used with *key_type* equals to - ``'filename'`` and ``'lineno'``. - - The result is sorted from the biggest to the smallest by: - :attr:`Statistic.size`, :attr:`Statistic.count` and then by - :attr:`Statistic.traceback`. - - - .. attribute:: traceback_limit - - Maximum number of frames stored in the traceback of :attr:`traces`: - result of the :func:`get_traceback_limit` when the snapshot was taken. - - .. attribute:: traces - - Traces of all memory blocks allocated by Python: sequence of - :class:`Trace` instances. - - The sequence has an undefined order. Use the :meth:`Snapshot.statistics` - method to get a sorted list of statistics. - - -Statistic -^^^^^^^^^ - -.. class:: Statistic - - Statistic on memory allocations. - - :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances. - - See also the :class:`StatisticDiff` class. - - .. attribute:: count - - Number of memory blocks (``int``). - - .. attribute:: size - - Total size of memory blocks in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -StatisticDiff -^^^^^^^^^^^^^ - -.. class:: StatisticDiff - - Statistic difference on memory allocations between an old and a new - :class:`Snapshot` instance. - - :func:`Snapshot.compare_to` returns a list of :class:`StatisticDiff` - instances. See also the :class:`Statistic` class. - - .. attribute:: count - - Number of memory blocks in the new snapshot (``int``): ``0`` if - the memory blocks have been released in the new snapshot. - - .. attribute:: count_diff - - Difference of number of memory blocks between the old and the new - snapshots (``int``): ``0`` if the memory blocks have been allocated in - the new snapshot. - - .. attribute:: size - - Total size of memory blocks in bytes in the new snapshot (``int``): - ``0`` if the memory blocks have been released in the new snapshot. - - .. attribute:: size_diff - - Difference of total size of memory blocks in bytes between the old and - the new snapshots (``int``): ``0`` if the memory blocks have been - allocated in the new snapshot. - - .. attribute:: traceback - - Traceback where the memory blocks were allocated, :class:`Traceback` - instance. - - -Trace -^^^^^ - -.. class:: Trace - - Trace of a memory block. - - The :attr:`Snapshot.traces` attribute is a sequence of :class:`Trace` - instances. - - .. attribute:: size - - Size of the memory block in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -Traceback -^^^^^^^^^ - -.. class:: Traceback - - Sequence of :class:`Frame` instances sorted from the most recent frame to - the oldest frame. - - A traceback contains at least ``1`` frame. If the ``tracemalloc`` module - failed to get a frame, the filename ``""`` at line number ``0`` is - used. - - When a snapshot is taken, tracebacks of traces are limited to - :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. - - The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` - instance. - - .. method:: format(limit=None) - - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. If - *limit* is set, only format the *limit* most recent frames. - - Similar to the :func:`traceback.format_tb` function, except that - :meth:`.format` does not include newlines. - - Example:: - - print("Traceback (most recent call first):") - for line in traceback: - print(line) - - Output:: - - Traceback (most recent call first): - File "test.py", line 9 - obj = Object() - File "test.py", line 12 - tb = tracemalloc.get_object_traceback(f()) diff --git a/third_party/python/Doc/library/tty.rst b/third_party/python/Doc/library/tty.rst deleted file mode 100644 index b30bc3c7a..000000000 --- a/third_party/python/Doc/library/tty.rst +++ /dev/null @@ -1,41 +0,0 @@ -:mod:`tty` --- Terminal control functions -========================================= - -.. module:: tty - :platform: Unix - :synopsis: Utility functions that perform common terminal control operations. - -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/tty.py` - --------------- - -The :mod:`tty` module defines functions for putting the tty into cbreak and raw -modes. - -Because it requires the :mod:`termios` module, it will work only on Unix. - -The :mod:`tty` module defines the following functions: - - -.. function:: setraw(fd, when=termios.TCSAFLUSH) - - Change the mode of the file descriptor *fd* to raw. If *when* is omitted, it - defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. - - -.. function:: setcbreak(fd, when=termios.TCSAFLUSH) - - Change the mode of file descriptor *fd* to cbreak. If *when* is omitted, it - defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. - - -.. seealso:: - - Module :mod:`termios` - Low-level terminal control interface. - diff --git a/third_party/python/Doc/library/tulip_coro.dia b/third_party/python/Doc/library/tulip_coro.dia deleted file mode 100644 index 70a33e3c0..000000000 Binary files a/third_party/python/Doc/library/tulip_coro.dia and /dev/null differ diff --git a/third_party/python/Doc/library/tulip_coro.png b/third_party/python/Doc/library/tulip_coro.png deleted file mode 100644 index 36ced8ddb..000000000 Binary files a/third_party/python/Doc/library/tulip_coro.png and /dev/null differ diff --git a/third_party/python/Doc/library/turtle-star.pdf b/third_party/python/Doc/library/turtle-star.pdf deleted file mode 100644 index e354073dd..000000000 Binary files a/third_party/python/Doc/library/turtle-star.pdf and /dev/null differ diff --git a/third_party/python/Doc/library/turtle-star.png b/third_party/python/Doc/library/turtle-star.png deleted file mode 100644 index caf36a3ab..000000000 Binary files a/third_party/python/Doc/library/turtle-star.png and /dev/null differ diff --git a/third_party/python/Doc/library/turtle-star.ps b/third_party/python/Doc/library/turtle-star.ps deleted file mode 100644 index 46362cb9f..000000000 --- a/third_party/python/Doc/library/turtle-star.ps +++ /dev/null @@ -1,447 +0,0 @@ -%!PS-Adobe-3.0 EPSF-3.0 -%%Creator: Tk Canvas Widget -%%For: Alexander Belopolsky -%%Title: Window .4315905424 -%%CreationDate: Tue Nov 9 12:54:06 2010 -%%XBoundingBox: -172 -52 785 845 -%%BoundingBox: 290 290 520 520 -%%Pages: 1 -%%DocumentData: Clean7Bit -%%Orientation: Portrait -%%EndComments - -%%BeginProlog -/CurrentEncoding [ -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/exclam/quotedbl/numbersign/dollar/percent/ampersand/quotesingle -/parenleft/parenright/asterisk/plus/comma/hyphen/period/slash -/zero/one/two/three/four/five/six/seven -/eight/nine/colon/semicolon/less/equal/greater/question -/at/A/B/C/D/E/F/G -/H/I/J/K/L/M/N/O -/P/Q/R/S/T/U/V/W -/X/Y/Z/bracketleft/backslash/bracketright/asciicircum/underscore -/grave/a/b/c/d/e/f/g -/h/i/j/k/l/m/n/o -/p/q/r/s/t/u/v/w -/x/y/z/braceleft/bar/braceright/asciitilde/space -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/space/space/space/space/space/space/space -/space/exclamdown/cent/sterling/currency/yen/brokenbar/section -/dieresis/copyright/ordfeminine/guillemotleft/logicalnot/hyphen/registered/macron -/degree/plusminus/twosuperior/threesuperior/acute/mu/paragraph/periodcentered -/cedilla/onesuperior/ordmasculine/guillemotright/onequarter/onehalf/threequarters/questiondown -/Agrave/Aacute/Acircumflex/Atilde/Adieresis/Aring/AE/Ccedilla -/Egrave/Eacute/Ecircumflex/Edieresis/Igrave/Iacute/Icircumflex/Idieresis -/Eth/Ntilde/Ograve/Oacute/Ocircumflex/Otilde/Odieresis/multiply -/Oslash/Ugrave/Uacute/Ucircumflex/Udieresis/Yacute/Thorn/germandbls -/agrave/aacute/acircumflex/atilde/adieresis/aring/ae/ccedilla -/egrave/eacute/ecircumflex/edieresis/igrave/iacute/icircumflex/idieresis -/eth/ntilde/ograve/oacute/ocircumflex/otilde/odieresis/divide -/oslash/ugrave/uacute/ucircumflex/udieresis/yacute/thorn/ydieresis -] def - -50 dict begin -% This is a standard prolog for Postscript generated by Tk's canvas -% widget. -% RCS: @(#) $Id$ - -% The definitions below just define all of the variables used in -% any of the procedures here. This is needed for obscure reasons -% explained on p. 716 of the Postscript manual (Section H.2.7, -% "Initializing Variables," in the section on Encapsulated Postscript). - -/baseline 0 def -/stipimage 0 def -/height 0 def -/justify 0 def -/lineLength 0 def -/spacing 0 def -/stipple 0 def -/strings 0 def -/xoffset 0 def -/yoffset 0 def -/tmpstip null def - - -/cstringshow { - { - dup type /stringtype eq - { show } { glyphshow } - ifelse - } - forall -} bind def - - - -/cstringwidth { - 0 exch 0 exch - { - dup type /stringtype eq - { stringwidth } { - currentfont /Encoding get exch 1 exch put (\001) stringwidth - } - ifelse - exch 3 1 roll add 3 1 roll add exch - } - forall -} bind def - -% font ISOEncode font -% This procedure changes the encoding of a font from the default -% Postscript encoding to current system encoding. It's typically invoked just -% before invoking "setfont". The body of this procedure comes from -% Section 5.6.1 of the Postscript book. - -/ISOEncode { - dup length dict begin - {1 index /FID ne {def} {pop pop} ifelse} forall - /Encoding CurrentEncoding def - currentdict - end - - % I'm not sure why it's necessary to use "definefont" on this new - % font, but it seems to be important; just use the name "Temporary" - % for the font. - - /Temporary exch definefont -} bind def - -% StrokeClip -% -% This procedure converts the current path into a clip area under -% the assumption of stroking. It's a bit tricky because some Postscript -% interpreters get errors during strokepath for dashed lines. If -% this happens then turn off dashes and try again. - -/StrokeClip { - {strokepath} stopped { - (This Postscript printer gets limitcheck overflows when) = - (stippling dashed lines; lines will be printed solid instead.) = - [] 0 setdash strokepath} if - clip -} bind def - -% desiredSize EvenPixels closestSize -% -% The procedure below is used for stippling. Given the optimal size -% of a dot in a stipple pattern in the current user coordinate system, -% compute the closest size that is an exact multiple of the device's -% pixel size. This allows stipple patterns to be displayed without -% aliasing effects. - -/EvenPixels { - % Compute exact number of device pixels per stipple dot. - dup 0 matrix currentmatrix dtransform - dup mul exch dup mul add sqrt - - % Round to an integer, make sure the number is at least 1, and compute - % user coord distance corresponding to this. - dup round dup 1 lt {pop 1} if - exch div mul -} bind def - -% width height string StippleFill -- -% -% Given a path already set up and a clipping region generated from -% it, this procedure will fill the clipping region with a stipple -% pattern. "String" contains a proper image description of the -% stipple pattern and "width" and "height" give its dimensions. Each -% stipple dot is assumed to be about one unit across in the current -% user coordinate system. This procedure trashes the graphics state. - -/StippleFill { - % The following code is needed to work around a NeWSprint bug. - - /tmpstip 1 index def - - % Change the scaling so that one user unit in user coordinates - % corresponds to the size of one stipple dot. - 1 EvenPixels dup scale - - % Compute the bounding box occupied by the path (which is now - % the clipping region), and round the lower coordinates down - % to the nearest starting point for the stipple pattern. Be - % careful about negative numbers, since the rounding works - % differently on them. - - pathbbox - 4 2 roll - 5 index div dup 0 lt {1 sub} if cvi 5 index mul 4 1 roll - 6 index div dup 0 lt {1 sub} if cvi 6 index mul 3 2 roll - - % Stack now: width height string y1 y2 x1 x2 - % Below is a doubly-nested for loop to iterate across this area - % in units of the stipple pattern size, going up columns then - % across rows, blasting out a stipple-pattern-sized rectangle at - % each position - - 6 index exch { - 2 index 5 index 3 index { - % Stack now: width height string y1 y2 x y - - gsave - 1 index exch translate - 5 index 5 index true matrix tmpstip imagemask - grestore - } for - pop - } for - pop pop pop pop pop -} bind def - -% -- AdjustColor -- -% Given a color value already set for output by the caller, adjusts -% that value to a grayscale or mono value if requested by the CL -% variable. - -/AdjustColor { - CL 2 lt { - currentgray - CL 0 eq { - .5 lt {0} {1} ifelse - } if - setgray - } if -} bind def - -% x y strings spacing xoffset yoffset justify stipple DrawText -- -% This procedure does all of the real work of drawing text. The -% color and font must already have been set by the caller, and the -% following arguments must be on the stack: -% -% x, y - Coordinates at which to draw text. -% strings - An array of strings, one for each line of the text item, -% in order from top to bottom. -% spacing - Spacing between lines. -% xoffset - Horizontal offset for text bbox relative to x and y: 0 for -% nw/w/sw anchor, -0.5 for n/center/s, and -1.0 for ne/e/se. -% yoffset - Vertical offset for text bbox relative to x and y: 0 for -% nw/n/ne anchor, +0.5 for w/center/e, and +1.0 for sw/s/se. -% justify - 0 for left justification, 0.5 for center, 1 for right justify. -% stipple - Boolean value indicating whether or not text is to be -% drawn in stippled fashion. If text is stippled, -% procedure StippleText must have been defined to call -% StippleFill in the right way. -% -% Also, when this procedure is invoked, the color and font must already -% have been set for the text. - -/DrawText { - /stipple exch def - /justify exch def - /yoffset exch def - /xoffset exch def - /spacing exch def - /strings exch def - - % First scan through all of the text to find the widest line. - - /lineLength 0 def - strings { - cstringwidth pop - dup lineLength gt {/lineLength exch def} {pop} ifelse - newpath - } forall - - % Compute the baseline offset and the actual font height. - - 0 0 moveto (TXygqPZ) false charpath - pathbbox dup /baseline exch def - exch pop exch sub /height exch def pop - newpath - - % Translate coordinates first so that the origin is at the upper-left - % corner of the text's bounding box. Remember that x and y for - % positioning are still on the stack. - - translate - lineLength xoffset mul - strings length 1 sub spacing mul height add yoffset mul translate - - % Now use the baseline and justification information to translate so - % that the origin is at the baseline and positioning point for the - % first line of text. - - justify lineLength mul baseline neg translate - - % Iterate over each of the lines to output it. For each line, - % compute its width again so it can be properly justified, then - % display it. - - strings { - dup cstringwidth pop - justify neg mul 0 moveto - stipple { - - - % The text is stippled, so turn it into a path and print - % by calling StippledText, which in turn calls StippleFill. - % Unfortunately, many Postscript interpreters will get - % overflow errors if we try to do the whole string at - % once, so do it a character at a time. - - gsave - /char (X) def - { - dup type /stringtype eq { - % This segment is a string. - { - char 0 3 -1 roll put - currentpoint - gsave - char true charpath clip StippleText - grestore - char stringwidth translate - moveto - } forall - } { - % This segment is glyph name - % Temporary override - currentfont /Encoding get exch 1 exch put - currentpoint - gsave (\001) true charpath clip StippleText - grestore - (\001) stringwidth translate - moveto - } ifelse - } forall - grestore - } {cstringshow} ifelse - 0 spacing neg translate - } forall -} bind def - -%%EndProlog -%%BeginSetup -/CL 2 def -%%EndSetup - -%%Page: 1 1 -save -306.0 396.0 translate -0.9995 0.9995 scale -4 -449 translate --483 898 moveto 475 898 lineto 475 0 lineto -483 0 lineto closepath clip newpath -gsave -grestore -gsave -0 445 moveto -200 445 lineto -3.03844939755837 479.729635533386 lineto -190.97697355474 411.325606868252 lineto -17.7718927978523 511.325606868252 lineto -170.980781421648 382.768084930944 lineto -42.42325948434 535.97697355474 lineto -142.42325948434 362.771892797852 lineto -74.0192308192062 550.710416955034 lineto -108.748866352592 353.748866352592 lineto -108.748866352592 553.748866352592 lineto -74.0192308192064 356.787315750151 lineto -142.42325948434 544.725839907333 lineto -42.4232594843401 371.520759150445 lineto -170.980781421648 524.72964777424 lineto -17.7718927978524 396.172125836932 lineto -190.97697355474 496.172125836933 lineto -3.03844939755834 427.768097171799 lineto -200 462.497732705185 lineto --1.13686837721616e-13 462.497732705185 lineto -196.961550602442 427.768097171799 lineto -9.02302644525972 496.172125836932 lineto -182.228107202148 396.172125836933 lineto -29.0192185783518 524.72964777424 lineto -157.57674051566 371.520759150445 lineto -57.5767405156596 544.725839907332 lineto -125.980769180794 356.787315750151 lineto -91.2511336474073 553.748866352592 lineto -91.2511336474079 353.748866352592 lineto -125.980769180793 550.710416955034 lineto -57.5767405156601 362.771892797852 lineto -157.57674051566 535.97697355474 lineto -29.0192185783522 382.768084930944 lineto -182.228107202148 511.325606868253 lineto -9.02302644525994 411.325606868252 lineto -196.961550602442 479.729635533386 lineto --1.70530256582424e-13 445 lineto -0 445 lineto -1.000 1.000 0.000 setrgbcolor AdjustColor -eofill -grestore -gsave -0 445 moveto -200 445 lineto -3.03844939755837 479.729635533386 lineto -190.97697355474 411.325606868252 lineto -17.7718927978523 511.325606868252 lineto -170.980781421648 382.768084930944 lineto -42.42325948434 535.97697355474 lineto -142.42325948434 362.771892797852 lineto -74.0192308192062 550.710416955034 lineto -108.748866352592 353.748866352592 lineto -108.748866352592 553.748866352592 lineto -74.0192308192064 356.787315750151 lineto -142.42325948434 544.725839907333 lineto -42.4232594843401 371.520759150445 lineto -170.980781421648 524.72964777424 lineto -17.7718927978524 396.172125836932 lineto -190.97697355474 496.172125836933 lineto -3.03844939755834 427.768097171799 lineto -200 462.497732705185 lineto --1.13686837721616e-13 462.497732705185 lineto -196.961550602442 427.768097171799 lineto -9.02302644525972 496.172125836932 lineto -182.228107202148 396.172125836933 lineto -29.0192185783518 524.72964777424 lineto -157.57674051566 371.520759150445 lineto -57.5767405156596 544.725839907332 lineto -125.980769180794 356.787315750151 lineto -91.2511336474073 553.748866352592 lineto -91.2511336474079 353.748866352592 lineto -125.980769180793 550.710416955034 lineto -57.5767405156601 362.771892797852 lineto -157.57674051566 535.97697355474 lineto -29.0192185783522 382.768084930944 lineto -182.228107202148 511.325606868253 lineto -9.02302644525994 411.325606868252 lineto -196.961550602442 479.729635533386 lineto --1.70530256582424e-13 445 lineto -1 setlinecap -1 setlinejoin -1 setlinewidth -[] 0 setdash -1.000 0.000 0.000 setrgbcolor AdjustColor -stroke -grestore -gsave -grestore -gsave --1.70530256582424e-13 445 moveto --9.00000000000019 450 lineto --7.00000000000017 445 lineto --9.00000000000015 440 lineto --1.70530256582424e-13 445 lineto -1.000 1.000 0.000 setrgbcolor AdjustColor -eofill --1.70530256582424e-13 445 moveto --9.00000000000019 450 lineto --7.00000000000017 445 lineto --9.00000000000015 440 lineto --1.70530256582424e-13 445 lineto -1 setlinejoin 1 setlinecap -1 setlinewidth -[] 0 setdash -1.000 0.000 0.000 setrgbcolor AdjustColor -stroke -grestore -restore showpage - -%%Trailer -end -%%EOF - diff --git a/third_party/python/Doc/library/turtle.rst b/third_party/python/Doc/library/turtle.rst deleted file mode 100644 index 8135c9330..000000000 --- a/third_party/python/Doc/library/turtle.rst +++ /dev/null @@ -1,2441 +0,0 @@ -================================= -:mod:`turtle` --- Turtle graphics -================================= - -.. module:: turtle - :synopsis: An educational framework for simple graphics applications - -.. sectionauthor:: Gregor Lingl - -**Source code:** :source:`Lib/turtle.py` - -.. testsetup:: default - - from turtle import * - turtle = Turtle() - --------------- - -Introduction -============ - -Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzig and -Seymour Papert in 1966. - -Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the -command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the -direction it is facing, drawing a line as it moves. Give it the command -``turtle.right(25)``, and it rotates in-place 25 degrees clockwise. - -.. sidebar:: Turtle star - - Turtle can draw intricate shapes using programs that repeat simple - moves. - - .. image:: turtle-star.* - :align: center - - .. literalinclude:: ../includes/turtle-star.py - -By combining together these and similar commands, intricate shapes and pictures -can easily be drawn. - -The :mod:`turtle` module is an extended reimplementation of the same-named -module from the Python standard distribution up to version Python 2.5. - -It tries to keep the merits of the old turtle module and to be (nearly) 100% -compatible with it. This means in the first place to enable the learning -programmer to use all the commands, classes and methods interactively when using -the module from within IDLE run with the ``-n`` switch. - -The turtle module provides turtle graphics primitives, in both object-oriented -and procedure-oriented ways. Because it uses :mod:`tkinter` for the underlying -graphics, it needs a version of Python installed with Tk support. - -The object-oriented interface uses essentially two+two classes: - -1. The :class:`TurtleScreen` class defines graphics windows as a playground for - the drawing turtles. Its constructor needs a :class:`tkinter.Canvas` or a - :class:`ScrolledCanvas` as argument. It should be used when :mod:`turtle` is - used as part of some application. - - The function :func:`Screen` returns a singleton object of a - :class:`TurtleScreen` subclass. This function should be used when - :mod:`turtle` is used as a standalone tool for doing graphics. - As a singleton object, inheriting from its class is not possible. - - All methods of TurtleScreen/Screen also exist as functions, i.e. as part of - the procedure-oriented interface. - -2. :class:`RawTurtle` (alias: :class:`RawPen`) defines Turtle objects which draw - on a :class:`TurtleScreen`. Its constructor needs a Canvas, ScrolledCanvas - or TurtleScreen as argument, so the RawTurtle objects know where to draw. - - Derived from RawTurtle is the subclass :class:`Turtle` (alias: :class:`Pen`), - which draws on "the" :class:`Screen` instance which is automatically - created, if not already present. - - All methods of RawTurtle/Turtle also exist as functions, i.e. part of the - procedure-oriented interface. - -The procedural interface provides functions which are derived from the methods -of the classes :class:`Screen` and :class:`Turtle`. They have the same names as -the corresponding methods. A screen object is automatically created whenever a -function derived from a Screen method is called. An (unnamed) turtle object is -automatically created whenever any of the functions derived from a Turtle method -is called. - -To use multiple turtles on a screen one has to use the object-oriented interface. - -.. note:: - In the following documentation the argument list for functions is given. - Methods, of course, have the additional first argument *self* which is - omitted here. - - -Overview of available Turtle and Screen methods -================================================= - -Turtle methods --------------- - -Turtle motion - Move and draw - | :func:`forward` | :func:`fd` - | :func:`backward` | :func:`bk` | :func:`back` - | :func:`right` | :func:`rt` - | :func:`left` | :func:`lt` - | :func:`goto` | :func:`setpos` | :func:`setposition` - | :func:`setx` - | :func:`sety` - | :func:`setheading` | :func:`seth` - | :func:`home` - | :func:`circle` - | :func:`dot` - | :func:`stamp` - | :func:`clearstamp` - | :func:`clearstamps` - | :func:`undo` - | :func:`speed` - - Tell Turtle's state - | :func:`position` | :func:`pos` - | :func:`towards` - | :func:`xcor` - | :func:`ycor` - | :func:`heading` - | :func:`distance` - - Setting and measurement - | :func:`degrees` - | :func:`radians` - -Pen control - Drawing state - | :func:`pendown` | :func:`pd` | :func:`down` - | :func:`penup` | :func:`pu` | :func:`up` - | :func:`pensize` | :func:`width` - | :func:`pen` - | :func:`isdown` - - Color control - | :func:`color` - | :func:`pencolor` - | :func:`fillcolor` - - Filling - | :func:`filling` - | :func:`begin_fill` - | :func:`end_fill` - - More drawing control - | :func:`reset` - | :func:`clear` - | :func:`write` - -Turtle state - Visibility - | :func:`showturtle` | :func:`st` - | :func:`hideturtle` | :func:`ht` - | :func:`isvisible` - - Appearance - | :func:`shape` - | :func:`resizemode` - | :func:`shapesize` | :func:`turtlesize` - | :func:`shearfactor` - | :func:`settiltangle` - | :func:`tiltangle` - | :func:`tilt` - | :func:`shapetransform` - | :func:`get_shapepoly` - -Using events - | :func:`onclick` - | :func:`onrelease` - | :func:`ondrag` - -Special Turtle methods - | :func:`begin_poly` - | :func:`end_poly` - | :func:`get_poly` - | :func:`clone` - | :func:`getturtle` | :func:`getpen` - | :func:`getscreen` - | :func:`setundobuffer` - | :func:`undobufferentries` - - -Methods of TurtleScreen/Screen ------------------------------- - -Window control - | :func:`bgcolor` - | :func:`bgpic` - | :func:`clear` | :func:`clearscreen` - | :func:`reset` | :func:`resetscreen` - | :func:`screensize` - | :func:`setworldcoordinates` - -Animation control - | :func:`delay` - | :func:`tracer` - | :func:`update` - -Using screen events - | :func:`listen` - | :func:`onkey` | :func:`onkeyrelease` - | :func:`onkeypress` - | :func:`onclick` | :func:`onscreenclick` - | :func:`ontimer` - | :func:`mainloop` | :func:`done` - -Settings and special methods - | :func:`mode` - | :func:`colormode` - | :func:`getcanvas` - | :func:`getshapes` - | :func:`register_shape` | :func:`addshape` - | :func:`turtles` - | :func:`window_height` - | :func:`window_width` - -Input methods - | :func:`textinput` - | :func:`numinput` - -Methods specific to Screen - | :func:`bye` - | :func:`exitonclick` - | :func:`setup` - | :func:`title` - - -Methods of RawTurtle/Turtle and corresponding functions -======================================================= - -Most of the examples in this section refer to a Turtle instance called -``turtle``. - -Turtle motion -------------- - -.. function:: forward(distance) - fd(distance) - - :param distance: a number (integer or float) - - Move the turtle forward by the specified *distance*, in the direction the - turtle is headed. - - .. doctest:: - - >>> turtle.position() - (0.00,0.00) - >>> turtle.forward(25) - >>> turtle.position() - (25.00,0.00) - >>> turtle.forward(-75) - >>> turtle.position() - (-50.00,0.00) - - -.. function:: back(distance) - bk(distance) - backward(distance) - - :param distance: a number - - Move the turtle backward by *distance*, opposite to the direction the - turtle is headed. Do not change the turtle's heading. - - .. doctest:: - :hide: - - >>> turtle.goto(0, 0) - - .. doctest:: - - >>> turtle.position() - (0.00,0.00) - >>> turtle.backward(30) - >>> turtle.position() - (-30.00,0.00) - - -.. function:: right(angle) - rt(angle) - - :param angle: a number (integer or float) - - Turn turtle right by *angle* units. (Units are by default degrees, but - can be set via the :func:`degrees` and :func:`radians` functions.) Angle - orientation depends on the turtle mode, see :func:`mode`. - - .. doctest:: - :hide: - - >>> turtle.setheading(22) - - .. doctest:: - - >>> turtle.heading() - 22.0 - >>> turtle.right(45) - >>> turtle.heading() - 337.0 - - -.. function:: left(angle) - lt(angle) - - :param angle: a number (integer or float) - - Turn turtle left by *angle* units. (Units are by default degrees, but - can be set via the :func:`degrees` and :func:`radians` functions.) Angle - orientation depends on the turtle mode, see :func:`mode`. - - .. doctest:: - :hide: - - >>> turtle.setheading(22) - - .. doctest:: - - >>> turtle.heading() - 22.0 - >>> turtle.left(45) - >>> turtle.heading() - 67.0 - - -.. function:: goto(x, y=None) - setpos(x, y=None) - setposition(x, y=None) - - :param x: a number or a pair/vector of numbers - :param y: a number or ``None`` - - If *y* is ``None``, *x* must be a pair of coordinates or a :class:`Vec2D` - (e.g. as returned by :func:`pos`). - - Move turtle to an absolute position. If the pen is down, draw line. Do - not change the turtle's orientation. - - .. doctest:: - :hide: - - >>> turtle.goto(0, 0) - - .. doctest:: - - >>> tp = turtle.pos() - >>> tp - (0.00,0.00) - >>> turtle.setpos(60,30) - >>> turtle.pos() - (60.00,30.00) - >>> turtle.setpos((20,80)) - >>> turtle.pos() - (20.00,80.00) - >>> turtle.setpos(tp) - >>> turtle.pos() - (0.00,0.00) - - -.. function:: setx(x) - - :param x: a number (integer or float) - - Set the turtle's first coordinate to *x*, leave second coordinate - unchanged. - - .. doctest:: - :hide: - - >>> turtle.goto(0, 240) - - .. doctest:: - - >>> turtle.position() - (0.00,240.00) - >>> turtle.setx(10) - >>> turtle.position() - (10.00,240.00) - - -.. function:: sety(y) - - :param y: a number (integer or float) - - Set the turtle's second coordinate to *y*, leave first coordinate unchanged. - - .. doctest:: - :hide: - - >>> turtle.goto(0, 40) - - .. doctest:: - - >>> turtle.position() - (0.00,40.00) - >>> turtle.sety(-10) - >>> turtle.position() - (0.00,-10.00) - - -.. function:: setheading(to_angle) - seth(to_angle) - - :param to_angle: a number (integer or float) - - Set the orientation of the turtle to *to_angle*. Here are some common - directions in degrees: - - =================== ==================== - standard mode logo mode - =================== ==================== - 0 - east 0 - north - 90 - north 90 - east - 180 - west 180 - south - 270 - south 270 - west - =================== ==================== - - .. doctest:: - - >>> turtle.setheading(90) - >>> turtle.heading() - 90.0 - - -.. function:: home() - - Move turtle to the origin -- coordinates (0,0) -- and set its heading to - its start-orientation (which depends on the mode, see :func:`mode`). - - .. doctest:: - :hide: - - >>> turtle.setheading(90) - >>> turtle.goto(0, -10) - - .. doctest:: - - >>> turtle.heading() - 90.0 - >>> turtle.position() - (0.00,-10.00) - >>> turtle.home() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - - -.. function:: circle(radius, extent=None, steps=None) - - :param radius: a number - :param extent: a number (or ``None``) - :param steps: an integer (or ``None``) - - Draw a circle with given *radius*. The center is *radius* units left of - the turtle; *extent* -- an angle -- determines which part of the circle - is drawn. If *extent* is not given, draw the entire circle. If *extent* - is not a full circle, one endpoint of the arc is the current pen - position. Draw the arc in counterclockwise direction if *radius* is - positive, otherwise in clockwise direction. Finally the direction of the - turtle is changed by the amount of *extent*. - - As the circle is approximated by an inscribed regular polygon, *steps* - determines the number of steps to use. If not given, it will be - calculated automatically. May be used to draw regular polygons. - - .. doctest:: - - >>> turtle.home() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - >>> turtle.circle(50) - >>> turtle.position() - (-0.00,0.00) - >>> turtle.heading() - 0.0 - >>> turtle.circle(120, 180) # draw a semicircle - >>> turtle.position() - (0.00,240.00) - >>> turtle.heading() - 180.0 - - -.. function:: dot(size=None, *color) - - :param size: an integer >= 1 (if given) - :param color: a colorstring or a numeric color tuple - - Draw a circular dot with diameter *size*, using *color*. If *size* is - not given, the maximum of pensize+4 and 2*pensize is used. - - - .. doctest:: - - >>> turtle.home() - >>> turtle.dot() - >>> turtle.fd(50); turtle.dot(20, "blue"); turtle.fd(50) - >>> turtle.position() - (100.00,-0.00) - >>> turtle.heading() - 0.0 - - -.. function:: stamp() - - Stamp a copy of the turtle shape onto the canvas at the current turtle - position. Return a stamp_id for that stamp, which can be used to delete - it by calling ``clearstamp(stamp_id)``. - - .. doctest:: - - >>> turtle.color("blue") - >>> turtle.stamp() - 11 - >>> turtle.fd(50) - - -.. function:: clearstamp(stampid) - - :param stampid: an integer, must be return value of previous - :func:`stamp` call - - Delete stamp with given *stampid*. - - .. doctest:: - - >>> turtle.position() - (150.00,-0.00) - >>> turtle.color("blue") - >>> astamp = turtle.stamp() - >>> turtle.fd(50) - >>> turtle.position() - (200.00,-0.00) - >>> turtle.clearstamp(astamp) - >>> turtle.position() - (200.00,-0.00) - - -.. function:: clearstamps(n=None) - - :param n: an integer (or ``None``) - - Delete all or first/last *n* of turtle's stamps. If *n* is ``None``, delete - all stamps, if *n* > 0 delete first *n* stamps, else if *n* < 0 delete - last *n* stamps. - - .. doctest:: - - >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - >>> turtle.clearstamps(2) - >>> turtle.clearstamps(-2) - >>> turtle.clearstamps() - - -.. function:: undo() - - Undo (repeatedly) the last turtle action(s). Number of available - undo actions is determined by the size of the undobuffer. - - .. doctest:: - - >>> for i in range(4): - ... turtle.fd(50); turtle.lt(80) - ... - >>> for i in range(8): - ... turtle.undo() - - -.. function:: speed(speed=None) - - :param speed: an integer in the range 0..10 or a speedstring (see below) - - Set the turtle's speed to an integer value in the range 0..10. If no - argument is given, return current speed. - - If input is a number greater than 10 or smaller than 0.5, speed is set - to 0. Speedstrings are mapped to speedvalues as follows: - - * "fastest": 0 - * "fast": 10 - * "normal": 6 - * "slow": 3 - * "slowest": 1 - - Speeds from 1 to 10 enforce increasingly faster animation of line drawing - and turtle turning. - - Attention: *speed* = 0 means that *no* animation takes - place. forward/back makes turtle jump and likewise left/right make the - turtle turn instantly. - - .. doctest:: - - >>> turtle.speed() - 3 - >>> turtle.speed('normal') - >>> turtle.speed() - 6 - >>> turtle.speed(9) - >>> turtle.speed() - 9 - - -Tell Turtle's state -------------------- - -.. function:: position() - pos() - - Return the turtle's current location (x,y) (as a :class:`Vec2D` vector). - - .. doctest:: - - >>> turtle.pos() - (440.00,-0.00) - - -.. function:: towards(x, y=None) - - :param x: a number or a pair/vector of numbers or a turtle instance - :param y: a number if *x* is a number, else ``None`` - - Return the angle between the line from turtle position to position specified - by (x,y), the vector or the other turtle. This depends on the turtle's start - orientation which depends on the mode - "standard"/"world" or "logo"). - - .. doctest:: - - >>> turtle.goto(10, 10) - >>> turtle.towards(0,0) - 225.0 - - -.. function:: xcor() - - Return the turtle's x coordinate. - - .. doctest:: - - >>> turtle.home() - >>> turtle.left(50) - >>> turtle.forward(100) - >>> turtle.pos() - (64.28,76.60) - >>> print(round(turtle.xcor(), 5)) - 64.27876 - - -.. function:: ycor() - - Return the turtle's y coordinate. - - .. doctest:: - - >>> turtle.home() - >>> turtle.left(60) - >>> turtle.forward(100) - >>> print(turtle.pos()) - (50.00,86.60) - >>> print(round(turtle.ycor(), 5)) - 86.60254 - - -.. function:: heading() - - Return the turtle's current heading (value depends on the turtle mode, see - :func:`mode`). - - .. doctest:: - - >>> turtle.home() - >>> turtle.left(67) - >>> turtle.heading() - 67.0 - - -.. function:: distance(x, y=None) - - :param x: a number or a pair/vector of numbers or a turtle instance - :param y: a number if *x* is a number, else ``None`` - - Return the distance from the turtle to (x,y), the given vector, or the given - other turtle, in turtle step units. - - .. doctest:: - - >>> turtle.home() - >>> turtle.distance(30,40) - 50.0 - >>> turtle.distance((30,40)) - 50.0 - >>> joe = Turtle() - >>> joe.forward(77) - >>> turtle.distance(joe) - 77.0 - - -Settings for measurement ------------------------- - -.. function:: degrees(fullcircle=360.0) - - :param fullcircle: a number - - Set angle measurement units, i.e. set number of "degrees" for a full circle. - Default value is 360 degrees. - - .. doctest:: - - >>> turtle.home() - >>> turtle.left(90) - >>> turtle.heading() - 90.0 - - Change angle measurement unit to grad (also known as gon, - grade, or gradian and equals 1/100-th of the right angle.) - >>> turtle.degrees(400.0) - >>> turtle.heading() - 100.0 - >>> turtle.degrees(360) - >>> turtle.heading() - 90.0 - - -.. function:: radians() - - Set the angle measurement units to radians. Equivalent to - ``degrees(2*math.pi)``. - - .. doctest:: - - >>> turtle.home() - >>> turtle.left(90) - >>> turtle.heading() - 90.0 - >>> turtle.radians() - >>> turtle.heading() - 1.5707963267948966 - - .. doctest:: - :hide: - - >>> turtle.degrees(360) - - -Pen control ------------ - -Drawing state -~~~~~~~~~~~~~ - -.. function:: pendown() - pd() - down() - - Pull the pen down -- drawing when moving. - - -.. function:: penup() - pu() - up() - - Pull the pen up -- no drawing when moving. - - -.. function:: pensize(width=None) - width(width=None) - - :param width: a positive number - - Set the line thickness to *width* or return it. If resizemode is set to - "auto" and turtleshape is a polygon, that polygon is drawn with the same line - thickness. If no argument is given, the current pensize is returned. - - .. doctest:: - - >>> turtle.pensize() - 1 - >>> turtle.pensize(10) # from here on lines of width 10 are drawn - - -.. function:: pen(pen=None, **pendict) - - :param pen: a dictionary with some or all of the below listed keys - :param pendict: one or more keyword-arguments with the below listed keys as keywords - - Return or set the pen's attributes in a "pen-dictionary" with the following - key/value pairs: - - * "shown": True/False - * "pendown": True/False - * "pencolor": color-string or color-tuple - * "fillcolor": color-string or color-tuple - * "pensize": positive number - * "speed": number in range 0..10 - * "resizemode": "auto" or "user" or "noresize" - * "stretchfactor": (positive number, positive number) - * "outline": positive number - * "tilt": number - - This dictionary can be used as argument for a subsequent call to :func:`pen` - to restore the former pen-state. Moreover one or more of these attributes - can be provided as keyword-arguments. This can be used to set several pen - attributes in one statement. - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> turtle.pen(fillcolor="black", pencolor="red", pensize=10) - >>> sorted(turtle.pen().items()) - [('fillcolor', 'black'), ('outline', 1), ('pencolor', 'red'), - ('pendown', True), ('pensize', 10), ('resizemode', 'noresize'), - ('shearfactor', 0.0), ('shown', True), ('speed', 9), - ('stretchfactor', (1.0, 1.0)), ('tilt', 0.0)] - >>> penstate=turtle.pen() - >>> turtle.color("yellow", "") - >>> turtle.penup() - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', ''), ('outline', 1), ('pencolor', 'yellow')] - >>> turtle.pen(penstate, fillcolor="green") - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', 'green'), ('outline', 1), ('pencolor', 'red')] - -.. function:: isdown() - - Return ``True`` if pen is down, ``False`` if it's up. - - .. doctest:: - - >>> turtle.penup() - >>> turtle.isdown() - False - >>> turtle.pendown() - >>> turtle.isdown() - True - - -Color control -~~~~~~~~~~~~~ - -.. function:: pencolor(*args) - - Return or set the pencolor. - - Four input formats are allowed: - - ``pencolor()`` - Return the current pencolor as color specification string or - as a tuple (see example). May be used as input to another - color/pencolor/fillcolor call. - - ``pencolor(colorstring)`` - Set pencolor to *colorstring*, which is a Tk color specification string, - such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. - - ``pencolor((r, g, b))`` - Set pencolor to the RGB color represented by the tuple of *r*, *g*, and - *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode, where - colormode is either 1.0 or 255 (see :func:`colormode`). - - ``pencolor(r, g, b)`` - Set pencolor to the RGB color represented by *r*, *g*, and *b*. Each of - *r*, *g*, and *b* must be in the range 0..colormode. - - If turtleshape is a polygon, the outline of that polygon is drawn with the - newly set pencolor. - - .. doctest:: - - >>> colormode() - 1.0 - >>> turtle.pencolor() - 'red' - >>> turtle.pencolor("brown") - >>> turtle.pencolor() - 'brown' - >>> tup = (0.2, 0.8, 0.55) - >>> turtle.pencolor(tup) - >>> turtle.pencolor() - (0.2, 0.8, 0.5490196078431373) - >>> colormode(255) - >>> turtle.pencolor() - (51.0, 204.0, 140.0) - >>> turtle.pencolor('#32c18f') - >>> turtle.pencolor() - (50.0, 193.0, 143.0) - - -.. function:: fillcolor(*args) - - Return or set the fillcolor. - - Four input formats are allowed: - - ``fillcolor()`` - Return the current fillcolor as color specification string, possibly - in tuple format (see example). May be used as input to another - color/pencolor/fillcolor call. - - ``fillcolor(colorstring)`` - Set fillcolor to *colorstring*, which is a Tk color specification string, - such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. - - ``fillcolor((r, g, b))`` - Set fillcolor to the RGB color represented by the tuple of *r*, *g*, and - *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode, where - colormode is either 1.0 or 255 (see :func:`colormode`). - - ``fillcolor(r, g, b)`` - Set fillcolor to the RGB color represented by *r*, *g*, and *b*. Each of - *r*, *g*, and *b* must be in the range 0..colormode. - - If turtleshape is a polygon, the interior of that polygon is drawn - with the newly set fillcolor. - - .. doctest:: - - >>> turtle.fillcolor("violet") - >>> turtle.fillcolor() - 'violet' - >>> col = turtle.pencolor() - >>> col - (50.0, 193.0, 143.0) - >>> turtle.fillcolor(col) - >>> turtle.fillcolor() - (50.0, 193.0, 143.0) - >>> turtle.fillcolor('#ffffff') - >>> turtle.fillcolor() - (255.0, 255.0, 255.0) - - -.. function:: color(*args) - - Return or set pencolor and fillcolor. - - Several input formats are allowed. They use 0 to 3 arguments as - follows: - - ``color()`` - Return the current pencolor and the current fillcolor as a pair of color - specification strings or tuples as returned by :func:`pencolor` and - :func:`fillcolor`. - - ``color(colorstring)``, ``color((r,g,b))``, ``color(r,g,b)`` - Inputs as in :func:`pencolor`, set both, fillcolor and pencolor, to the - given value. - - ``color(colorstring1, colorstring2)``, ``color((r1,g1,b1), (r2,g2,b2))`` - Equivalent to ``pencolor(colorstring1)`` and ``fillcolor(colorstring2)`` - and analogously if the other input format is used. - - If turtleshape is a polygon, outline and interior of that polygon is drawn - with the newly set colors. - - .. doctest:: - - >>> turtle.color("red", "green") - >>> turtle.color() - ('red', 'green') - >>> color("#285078", "#a0c8f0") - >>> color() - ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) - - -See also: Screen method :func:`colormode`. - - -Filling -~~~~~~~ - -.. doctest:: - :hide: - - >>> turtle.home() - -.. function:: filling() - - Return fillstate (``True`` if filling, ``False`` else). - - .. doctest:: - - >>> turtle.begin_fill() - >>> if turtle.filling(): - ... turtle.pensize(5) - ... else: - ... turtle.pensize(3) - - - -.. function:: begin_fill() - - To be called just before drawing a shape to be filled. - - -.. function:: end_fill() - - Fill the shape drawn after the last call to :func:`begin_fill`. - - .. doctest:: - - >>> turtle.color("black", "red") - >>> turtle.begin_fill() - >>> turtle.circle(80) - >>> turtle.end_fill() - - -More drawing control -~~~~~~~~~~~~~~~~~~~~ - -.. function:: reset() - - Delete the turtle's drawings from the screen, re-center the turtle and set - variables to the default values. - - .. doctest:: - - >>> turtle.goto(0,-22) - >>> turtle.left(100) - >>> turtle.position() - (0.00,-22.00) - >>> turtle.heading() - 100.0 - >>> turtle.reset() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - - -.. function:: clear() - - Delete the turtle's drawings from the screen. Do not move turtle. State and - position of the turtle as well as drawings of other turtles are not affected. - - -.. function:: write(arg, move=False, align="left", font=("Arial", 8, "normal")) - - :param arg: object to be written to the TurtleScreen - :param move: True/False - :param align: one of the strings "left", "center" or right" - :param font: a triple (fontname, fontsize, fonttype) - - Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or right") and with the given - font. If *move* is true, the pen is moved to the bottom-right corner of the - text. By default, *move* is ``False``. - - >>> turtle.write("Home = ", True, align="center") - >>> turtle.write((0,0), True) - - -Turtle state ------------- - -Visibility -~~~~~~~~~~ - -.. function:: hideturtle() - ht() - - Make the turtle invisible. It's a good idea to do this while you're in the - middle of doing some complex drawing, because hiding the turtle speeds up the - drawing observably. - - .. doctest:: - - >>> turtle.hideturtle() - - -.. function:: showturtle() - st() - - Make the turtle visible. - - .. doctest:: - - >>> turtle.showturtle() - - -.. function:: isvisible() - - Return ``True`` if the Turtle is shown, ``False`` if it's hidden. - - >>> turtle.hideturtle() - >>> turtle.isvisible() - False - >>> turtle.showturtle() - >>> turtle.isvisible() - True - - -Appearance -~~~~~~~~~~ - -.. function:: shape(name=None) - - :param name: a string which is a valid shapename - - Set turtle shape to shape with given *name* or, if name is not given, return - name of current shape. Shape with *name* must exist in the TurtleScreen's - shape dictionary. Initially there are the following polygon shapes: "arrow", - "turtle", "circle", "square", "triangle", "classic". To learn about how to - deal with shapes see Screen method :func:`register_shape`. - - .. doctest:: - - >>> turtle.shape() - 'classic' - >>> turtle.shape("turtle") - >>> turtle.shape() - 'turtle' - - -.. function:: resizemode(rmode=None) - - :param rmode: one of the strings "auto", "user", "noresize" - - Set resizemode to one of the values: "auto", "user", "noresize". If *rmode* - is not given, return current resizemode. Different resizemodes have the - following effects: - - - "auto": adapts the appearance of the turtle corresponding to the value of pensize. - - "user": adapts the appearance of the turtle according to the values of - stretchfactor and outlinewidth (outline), which are set by - :func:`shapesize`. - - "noresize": no adaption of the turtle's appearance takes place. - - resizemode("user") is called by :func:`shapesize` when used with arguments. - - .. doctest:: - - >>> turtle.resizemode() - 'noresize' - >>> turtle.resizemode("auto") - >>> turtle.resizemode() - 'auto' - - -.. function:: shapesize(stretch_wid=None, stretch_len=None, outline=None) - turtlesize(stretch_wid=None, stretch_len=None, outline=None) - - :param stretch_wid: positive number - :param stretch_len: positive number - :param outline: positive number - - Return or set the pen's attributes x/y-stretchfactors and/or outline. Set - resizemode to "user". If and only if resizemode is set to "user", the turtle - will be displayed stretched according to its stretchfactors: *stretch_wid* is - stretchfactor perpendicular to its orientation, *stretch_len* is - stretchfactor in direction of its orientation, *outline* determines the width - of the shapes's outline. - - .. doctest:: - - >>> turtle.shapesize() - (1.0, 1.0, 1) - >>> turtle.resizemode("user") - >>> turtle.shapesize(5, 5, 12) - >>> turtle.shapesize() - (5, 5, 12) - >>> turtle.shapesize(outline=8) - >>> turtle.shapesize() - (5, 5, 8) - - -.. function:: shearfactor(shear=None) - - :param shear: number (optional) - - Set or return the current shearfactor. Shear the turtleshape according to - the given shearfactor shear, which is the tangent of the shear angle. - Do *not* change the turtle's heading (direction of movement). - If shear is not given: return the current shearfactor, i. e. the - tangent of the shear angle, by which lines parallel to the - heading of the turtle are sheared. - - .. doctest:: - - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.shearfactor(0.5) - >>> turtle.shearfactor() - 0.5 - - -.. function:: tilt(angle) - - :param angle: a number - - Rotate the turtleshape by *angle* from its current tilt-angle, but do *not* - change the turtle's heading (direction of movement). - - .. doctest:: - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.tilt(30) - >>> turtle.fd(50) - >>> turtle.tilt(30) - >>> turtle.fd(50) - - -.. function:: settiltangle(angle) - - :param angle: a number - - Rotate the turtleshape to point in the direction specified by *angle*, - regardless of its current tilt-angle. *Do not* change the turtle's heading - (direction of movement). - - .. doctest:: - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.settiltangle(45) - >>> turtle.fd(50) - >>> turtle.settiltangle(-45) - >>> turtle.fd(50) - - .. deprecated:: 3.1 - - -.. function:: tiltangle(angle=None) - - :param angle: a number (optional) - - Set or return the current tilt-angle. If angle is given, rotate the - turtleshape to point in the direction specified by angle, - regardless of its current tilt-angle. Do *not* change the turtle's - heading (direction of movement). - If angle is not given: return the current tilt-angle, i. e. the angle - between the orientation of the turtleshape and the heading of the - turtle (its direction of movement). - - .. doctest:: - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.tilt(45) - >>> turtle.tiltangle() - 45.0 - - -.. function:: shapetransform(t11=None, t12=None, t21=None, t22=None) - - :param t11: a number (optional) - :param t12: a number (optional) - :param t21: a number (optional) - :param t12: a number (optional) - - Set or return the current transformation matrix of the turtle shape. - - If none of the matrix elements are given, return the transformation - matrix as a tuple of 4 elements. - Otherwise set the given elements and transform the turtleshape - according to the matrix consisting of first row t11, t12 and - second row t21, 22. The determinant t11 * t22 - t12 * t21 must not be - zero, otherwise an error is raised. - Modify stretchfactor, shearfactor and tiltangle according to the - given matrix. - - .. doctest:: - - >>> turtle = Turtle() - >>> turtle.shape("square") - >>> turtle.shapesize(4,2) - >>> turtle.shearfactor(-0.5) - >>> turtle.shapetransform() - (4.0, -1.0, -0.0, 2.0) - - -.. function:: get_shapepoly() - - Return the current shape polygon as tuple of coordinate pairs. This - can be used to define a new shape or components of a compound shape. - - .. doctest:: - - >>> turtle.shape("square") - >>> turtle.shapetransform(4, -1, 0, 2) - >>> turtle.get_shapepoly() - ((50, -20), (30, 20), (-50, 20), (-30, -20)) - - -Using events ------------- - -.. function:: onclick(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-click events on this turtle. If *fun* is ``None``, - existing bindings are removed. Example for the anonymous turtle, i.e. the - procedural way: - - .. doctest:: - - >>> def turn(x, y): - ... left(180) - ... - >>> onclick(turn) # Now clicking into the turtle will turn it. - >>> onclick(None) # event-binding will be removed - - -.. function:: onrelease(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-button-release events on this turtle. If *fun* is - ``None``, existing bindings are removed. - - .. doctest:: - - >>> class MyTurtle(Turtle): - ... def glow(self,x,y): - ... self.fillcolor("red") - ... def unglow(self,x,y): - ... self.fillcolor("") - ... - >>> turtle = MyTurtle() - >>> turtle.onclick(turtle.glow) # clicking on turtle turns fillcolor red, - >>> turtle.onrelease(turtle.unglow) # releasing turns it to transparent. - - -.. function:: ondrag(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-move events on this turtle. If *fun* is ``None``, - existing bindings are removed. - - Remark: Every sequence of mouse-move-events on a turtle is preceded by a - mouse-click event on that turtle. - - .. doctest:: - - >>> turtle.ondrag(turtle.goto) - - Subsequently, clicking and dragging the Turtle will move it across - the screen thereby producing handdrawings (if pen is down). - - -Special Turtle methods ----------------------- - -.. function:: begin_poly() - - Start recording the vertices of a polygon. Current turtle position is first - vertex of polygon. - - -.. function:: end_poly() - - Stop recording the vertices of a polygon. Current turtle position is last - vertex of polygon. This will be connected with the first vertex. - - -.. function:: get_poly() - - Return the last recorded polygon. - - .. doctest:: - - >>> turtle.home() - >>> turtle.begin_poly() - >>> turtle.fd(100) - >>> turtle.left(20) - >>> turtle.fd(30) - >>> turtle.left(60) - >>> turtle.fd(50) - >>> turtle.end_poly() - >>> p = turtle.get_poly() - >>> register_shape("myFavouriteShape", p) - - -.. function:: clone() - - Create and return a clone of the turtle with same position, heading and - turtle properties. - - .. doctest:: - - >>> mick = Turtle() - >>> joe = mick.clone() - - -.. function:: getturtle() - getpen() - - Return the Turtle object itself. Only reasonable use: as a function to - return the "anonymous turtle": - - .. doctest:: - - >>> pet = getturtle() - >>> pet.fd(50) - >>> pet - - - -.. function:: getscreen() - - Return the :class:`TurtleScreen` object the turtle is drawing on. - TurtleScreen methods can then be called for that object. - - .. doctest:: - - >>> ts = turtle.getscreen() - >>> ts - - >>> ts.bgcolor("pink") - - -.. function:: setundobuffer(size) - - :param size: an integer or ``None`` - - Set or disable undobuffer. If *size* is an integer an empty undobuffer of - given size is installed. *size* gives the maximum number of turtle actions - that can be undone by the :func:`undo` method/function. If *size* is - ``None``, the undobuffer is disabled. - - .. doctest:: - - >>> turtle.setundobuffer(42) - - -.. function:: undobufferentries() - - Return number of entries in the undobuffer. - - .. doctest:: - - >>> while undobufferentries(): - ... undo() - - - -.. _compoundshapes: - -Compound shapes ---------------- - -To use compound turtle shapes, which consist of several polygons of different -color, you must use the helper class :class:`Shape` explicitly as described -below: - -1. Create an empty Shape object of type "compound". -2. Add as many components to this object as desired, using the - :meth:`addcomponent` method. - - For example: - - .. doctest:: - - >>> s = Shape("compound") - >>> poly1 = ((0,0),(10,-5),(0,10),(-10,-5)) - >>> s.addcomponent(poly1, "red", "blue") - >>> poly2 = ((0,0),(10,-5),(-10,-5)) - >>> s.addcomponent(poly2, "blue", "red") - -3. Now add the Shape to the Screen's shapelist and use it: - - .. doctest:: - - >>> register_shape("myshape", s) - >>> shape("myshape") - - -.. note:: - - The :class:`Shape` class is used internally by the :func:`register_shape` - method in different ways. The application programmer has to deal with the - Shape class *only* when using compound shapes like shown above! - - -Methods of TurtleScreen/Screen and corresponding functions -========================================================== - -Most of the examples in this section refer to a TurtleScreen instance called -``screen``. - -.. doctest:: - :hide: - - >>> screen = Screen() - -Window control --------------- - -.. function:: bgcolor(*args) - - :param args: a color string or three numbers in the range 0..colormode or a - 3-tuple of such numbers - - - Set or return background color of the TurtleScreen. - - .. doctest:: - - >>> screen.bgcolor("orange") - >>> screen.bgcolor() - 'orange' - >>> screen.bgcolor("#800080") - >>> screen.bgcolor() - (128.0, 0.0, 128.0) - - -.. function:: bgpic(picname=None) - - :param picname: a string, name of a gif-file or ``"nopic"``, or ``None`` - - Set background image or return name of current backgroundimage. If *picname* - is a filename, set the corresponding image as background. If *picname* is - ``"nopic"``, delete background image, if present. If *picname* is ``None``, - return the filename of the current backgroundimage. :: - - >>> screen.bgpic() - 'nopic' - >>> screen.bgpic("landscape.gif") - >>> screen.bgpic() - "landscape.gif" - - -.. function:: clear() - clearscreen() - - Delete all drawings and all turtles from the TurtleScreen. Reset the now - empty TurtleScreen to its initial state: white background, no background - image, no event bindings and tracing on. - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``clearscreen``. The global function ``clear`` is a different one - derived from the Turtle method ``clear``. - - -.. function:: reset() - resetscreen() - - Reset all Turtles on the Screen to their initial state. - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``resetscreen``. The global function ``reset`` is another one - derived from the Turtle method ``reset``. - - -.. function:: screensize(canvwidth=None, canvheight=None, bg=None) - - :param canvwidth: positive integer, new width of canvas in pixels - :param canvheight: positive integer, new height of canvas in pixels - :param bg: colorstring or color-tuple, new background color - - If no arguments are given, return current (canvaswidth, canvasheight). Else - resize the canvas the turtles are drawing on. Do not alter the drawing - window. To observe hidden parts of the canvas, use the scrollbars. With this - method, one can make visible those parts of a drawing which were outside the - canvas before. - - >>> screen.screensize() - (400, 300) - >>> screen.screensize(2000,1500) - >>> screen.screensize() - (2000, 1500) - - e.g. to search for an erroneously escaped turtle ;-) - - -.. function:: setworldcoordinates(llx, lly, urx, ury) - - :param llx: a number, x-coordinate of lower left corner of canvas - :param lly: a number, y-coordinate of lower left corner of canvas - :param urx: a number, x-coordinate of upper right corner of canvas - :param ury: a number, y-coordinate of upper right corner of canvas - - Set up user-defined coordinate system and switch to mode "world" if - necessary. This performs a ``screen.reset()``. If mode "world" is already - active, all drawings are redrawn according to the new coordinates. - - **ATTENTION**: in user-defined coordinate systems angles may appear - distorted. - - .. doctest:: - - >>> screen.reset() - >>> screen.setworldcoordinates(-50,-7.5,50,7.5) - >>> for _ in range(72): - ... left(10) - ... - >>> for _ in range(8): - ... left(45); fd(2) # a regular octagon - - .. doctest:: - :hide: - - >>> screen.reset() - >>> for t in turtles(): - ... t.reset() - - -Animation control ------------------ - -.. function:: delay(delay=None) - - :param delay: positive integer - - Set or return the drawing *delay* in milliseconds. (This is approximately - the time interval between two consecutive canvas updates.) The longer the - drawing delay, the slower the animation. - - Optional argument: - - .. doctest:: - - >>> screen.delay() - 10 - >>> screen.delay(5) - >>> screen.delay() - 5 - - -.. function:: tracer(n=None, delay=None) - - :param n: nonnegative integer - :param delay: nonnegative integer - - Turn turtle animation on/off and set delay for update drawings. If - *n* is given, only each n-th regular screen update is really - performed. (Can be used to accelerate the drawing of complex - graphics.) When called without arguments, returns the currently - stored value of n. Second argument sets delay value (see - :func:`delay`). - - .. doctest:: - - >>> screen.tracer(8, 25) - >>> dist = 2 - >>> for i in range(200): - ... fd(dist) - ... rt(90) - ... dist += 2 - - -.. function:: update() - - Perform a TurtleScreen update. To be used when tracer is turned off. - -See also the RawTurtle/Turtle method :func:`speed`. - - -Using screen events -------------------- - -.. function:: listen(xdummy=None, ydummy=None) - - Set focus on TurtleScreen (in order to collect key-events). Dummy arguments - are provided in order to be able to pass :func:`listen` to the onclick method. - - -.. function:: onkey(fun, key) - onkeyrelease(fun, key) - - :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") - - Bind *fun* to key-release event of key. If *fun* is ``None``, event bindings - are removed. Remark: in order to be able to register key-events, TurtleScreen - must have the focus. (See method :func:`listen`.) - - .. doctest:: - - >>> def f(): - ... fd(50) - ... lt(60) - ... - >>> screen.onkey(f, "Up") - >>> screen.listen() - - -.. function:: onkeypress(fun, key=None) - - :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") - - Bind *fun* to key-press event of key if key is given, - or to any key-press-event if no key is given. - Remark: in order to be able to register key-events, TurtleScreen - must have focus. (See method :func:`listen`.) - - .. doctest:: - - >>> def f(): - ... fd(50) - ... - >>> screen.onkey(f, "Up") - >>> screen.listen() - - -.. function:: onclick(fun, btn=1, add=None) - onscreenclick(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-click events on this screen. If *fun* is ``None``, - existing bindings are removed. - - Example for a TurtleScreen instance named ``screen`` and a Turtle instance - named turtle: - - .. doctest:: - - >>> screen.onclick(turtle.goto) # Subsequently clicking into the TurtleScreen will - >>> # make the turtle move to the clicked point. - >>> screen.onclick(None) # remove event binding again - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``onscreenclick``. The global function ``onclick`` is another one - derived from the Turtle method ``onclick``. - - -.. function:: ontimer(fun, t=0) - - :param fun: a function with no arguments - :param t: a number >= 0 - - Install a timer that calls *fun* after *t* milliseconds. - - .. doctest:: - - >>> running = True - >>> def f(): - ... if running: - ... fd(50) - ... lt(60) - ... screen.ontimer(f, 250) - >>> f() ### makes the turtle march around - >>> running = False - - -.. function:: mainloop() - done() - - Starts event loop - calling Tkinter's mainloop function. - Must be the last statement in a turtle graphics program. - Must *not* be used if a script is run from within IDLE in -n mode - (No subprocess) - for interactive use of turtle graphics. :: - - >>> screen.mainloop() - - -Input methods -------------- - -.. function:: textinput(title, prompt) - - :param title: string - :param prompt: string - - Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, prompt is a text mostly describing - what information to input. - Return the string input. If the dialog is canceled, return ``None``. :: - - >>> screen.textinput("NIM", "Name of first player:") - - -.. function:: numinput(title, prompt, default=None, minval=None, maxval=None) - - :param title: string - :param prompt: string - :param default: number (optional) - :param minval: number (optional) - :param maxval: number (optional) - - Pop up a dialog window for input of a number. title is the title of the - dialog window, prompt is a text mostly describing what numerical information - to input. default: default value, minval: minimum value for input, - maxval: maximum value for input - The number input must be in the range minval .. maxval if these are - given. If not, a hint is issued and the dialog remains open for - correction. - Return the number input. If the dialog is canceled, return ``None``. :: - - >>> screen.numinput("Poker", "Your stakes:", 1000, minval=10, maxval=10000) - - -Settings and special methods ----------------------------- - -.. function:: mode(mode=None) - - :param mode: one of the strings "standard", "logo" or "world" - - Set turtle mode ("standard", "logo" or "world") and perform reset. If mode - is not given, current mode is returned. - - Mode "standard" is compatible with old :mod:`turtle`. Mode "logo" is - compatible with most Logo turtle graphics. Mode "world" uses user-defined - "world coordinates". **Attention**: in this mode angles appear distorted if - ``x/y`` unit-ratio doesn't equal 1. - - ============ ========================= =================== - Mode Initial turtle heading positive angles - ============ ========================= =================== - "standard" to the right (east) counterclockwise - "logo" upward (north) clockwise - ============ ========================= =================== - - .. doctest:: - - >>> mode("logo") # resets turtle heading to north - >>> mode() - 'logo' - - -.. function:: colormode(cmode=None) - - :param cmode: one of the values 1.0 or 255 - - Return the colormode or set it to 1.0 or 255. Subsequently *r*, *g*, *b* - values of color triples have to be in the range 0..\ *cmode*. - - .. doctest:: - - >>> screen.colormode(1) - >>> turtle.pencolor(240, 160, 80) - Traceback (most recent call last): - ... - TurtleGraphicsError: bad color sequence: (240, 160, 80) - >>> screen.colormode() - 1.0 - >>> screen.colormode(255) - >>> screen.colormode() - 255 - >>> turtle.pencolor(240,160,80) - - -.. function:: getcanvas() - - Return the Canvas of this TurtleScreen. Useful for insiders who know what to - do with a Tkinter Canvas. - - .. doctest:: - - >>> cv = screen.getcanvas() - >>> cv - - - -.. function:: getshapes() - - Return a list of names of all currently available turtle shapes. - - .. doctest:: - - >>> screen.getshapes() - ['arrow', 'blank', 'circle', ..., 'turtle'] - - -.. function:: register_shape(name, shape=None) - addshape(name, shape=None) - - There are three different ways to call this function: - - (1) *name* is the name of a gif-file and *shape* is ``None``: Install the - corresponding image shape. :: - - >>> screen.register_shape("turtle.gif") - - .. note:: - Image shapes *do not* rotate when turning the turtle, so they do not - display the heading of the turtle! - - (2) *name* is an arbitrary string and *shape* is a tuple of pairs of - coordinates: Install the corresponding polygon shape. - - .. doctest:: - - >>> screen.register_shape("triangle", ((5,-3), (0,5), (-5,-3))) - - (3) *name* is an arbitrary string and shape is a (compound) :class:`Shape` - object: Install the corresponding compound shape. - - Add a turtle shape to TurtleScreen's shapelist. Only thusly registered - shapes can be used by issuing the command ``shape(shapename)``. - - -.. function:: turtles() - - Return the list of turtles on the screen. - - .. doctest:: - - >>> for turtle in screen.turtles(): - ... turtle.color("red") - - -.. function:: window_height() - - Return the height of the turtle window. :: - - >>> screen.window_height() - 480 - - -.. function:: window_width() - - Return the width of the turtle window. :: - - >>> screen.window_width() - 640 - - -.. _screenspecific: - -Methods specific to Screen, not inherited from TurtleScreen ------------------------------------------------------------ - -.. function:: bye() - - Shut the turtlegraphics window. - - -.. function:: exitonclick() - - Bind bye() method to mouse clicks on the Screen. - - - If the value "using_IDLE" in the configuration dictionary is ``False`` - (default value), also enter mainloop. Remark: If IDLE with the ``-n`` switch - (no subprocess) is used, this value should be set to ``True`` in - :file:`turtle.cfg`. In this case IDLE's own mainloop is active also for the - client script. - - -.. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) - - Set the size and position of the main window. Default values of arguments - are stored in the configuration dictionary and can be changed via a - :file:`turtle.cfg` file. - - :param width: if an integer, a size in pixels, if a float, a fraction of the - screen; default is 50% of screen - :param height: if an integer, the height in pixels, if a float, a fraction of - the screen; default is 75% of screen - :param startx: if positive, starting position in pixels from the left - edge of the screen, if negative from the right edge, if ``None``, - center window horizontally - :param starty: if positive, starting position in pixels from the top - edge of the screen, if negative from the bottom edge, if ``None``, - center window vertically - - .. doctest:: - - >>> screen.setup (width=200, height=200, startx=0, starty=0) - >>> # sets window to 200x200 pixels, in upper left of screen - >>> screen.setup(width=.75, height=0.5, startx=None, starty=None) - >>> # sets window to 75% of screen by 50% of screen and centers - - -.. function:: title(titlestring) - - :param titlestring: a string that is shown in the titlebar of the turtle - graphics window - - Set title of turtle window to *titlestring*. - - .. doctest:: - - >>> screen.title("Welcome to the turtle zoo!") - - -Public classes -============== - - -.. class:: RawTurtle(canvas) - RawPen(canvas) - - :param canvas: a :class:`tkinter.Canvas`, a :class:`ScrolledCanvas` or a - :class:`TurtleScreen` - - Create a turtle. The turtle has all methods described above as "methods of - Turtle/RawTurtle". - - -.. class:: Turtle() - - Subclass of RawTurtle, has the same interface but draws on a default - :class:`Screen` object created automatically when needed for the first time. - - -.. class:: TurtleScreen(cv) - - :param cv: a :class:`tkinter.Canvas` - - Provides screen oriented methods like :func:`setbg` etc. that are described - above. - -.. class:: Screen() - - Subclass of TurtleScreen, with :ref:`four methods added `. - - -.. class:: ScrolledCanvas(master) - - :param master: some Tkinter widget to contain the ScrolledCanvas, i.e. - a Tkinter-canvas with scrollbars added - - Used by class Screen, which thus automatically provides a ScrolledCanvas as - playground for the turtles. - -.. class:: Shape(type_, data) - - :param type\_: one of the strings "polygon", "image", "compound" - - Data structure modeling shapes. The pair ``(type_, data)`` must follow this - specification: - - - =========== =========== - *type_* *data* - =========== =========== - "polygon" a polygon-tuple, i.e. a tuple of pairs of coordinates - "image" an image (in this form only used internally!) - "compound" ``None`` (a compound shape has to be constructed using the - :meth:`addcomponent` method) - =========== =========== - - .. method:: addcomponent(poly, fill, outline=None) - - :param poly: a polygon, i.e. a tuple of pairs of numbers - :param fill: a color the *poly* will be filled with - :param outline: a color for the poly's outline (if given) - - Example: - - .. doctest:: - - >>> poly = ((0,0),(10,-5),(0,10),(-10,-5)) - >>> s = Shape("compound") - >>> s.addcomponent(poly, "red", "blue") - >>> # ... add more components and then use register_shape() - - See :ref:`compoundshapes`. - - -.. class:: Vec2D(x, y) - - A two-dimensional vector class, used as a helper class for implementing - turtle graphics. May be useful for turtle graphics programs too. Derived - from tuple, so a vector is a tuple! - - Provides (for *a*, *b* vectors, *k* number): - - * ``a + b`` vector addition - * ``a - b`` vector subtraction - * ``a * b`` inner product - * ``k * a`` and ``a * k`` multiplication with scalar - * ``abs(a)`` absolute value of a - * ``a.rotate(angle)`` rotation - - -Help and configuration -====================== - -How to use help ---------------- - -The public methods of the Screen and Turtle classes are documented extensively -via docstrings. So these can be used as online-help via the Python help -facilities: - -- When using IDLE, tooltips show the signatures and first lines of the - docstrings of typed in function-/method calls. - -- Calling :func:`help` on methods or functions displays the docstrings:: - - >>> help(Screen.bgcolor) - Help on method bgcolor in module turtle: - - bgcolor(self, *args) unbound turtle.Screen method - Set or return backgroundcolor of the TurtleScreen. - - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. - - - >>> screen.bgcolor("orange") - >>> screen.bgcolor() - "orange" - >>> screen.bgcolor(0.5,0,0.5) - >>> screen.bgcolor() - "#800080" - - >>> help(Turtle.penup) - Help on method penup in module turtle: - - penup(self) unbound turtle.Turtle method - Pull the pen up -- no drawing when moving. - - Aliases: penup | pu | up - - No argument - - >>> turtle.penup() - -- The docstrings of the functions which are derived from methods have a modified - form:: - - >>> help(bgcolor) - Help on function bgcolor in module turtle: - - bgcolor(*args) - Set or return backgroundcolor of the TurtleScreen. - - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. - - Example:: - - >>> bgcolor("orange") - >>> bgcolor() - "orange" - >>> bgcolor(0.5,0,0.5) - >>> bgcolor() - "#800080" - - >>> help(penup) - Help on function penup in module turtle: - - penup() - Pull the pen up -- no drawing when moving. - - Aliases: penup | pu | up - - No argument - - Example: - >>> penup() - -These modified docstrings are created automatically together with the function -definitions that are derived from the methods at import time. - - -Translation of docstrings into different languages --------------------------------------------------- - -There is a utility to create a dictionary the keys of which are the method names -and the values of which are the docstrings of the public methods of the classes -Screen and Turtle. - -.. function:: write_docstringdict(filename="turtle_docstringdict") - - :param filename: a string, used as filename - - Create and write docstring-dictionary to a Python script with the given - filename. This function has to be called explicitly (it is not used by the - turtle graphics classes). The docstring dictionary will be written to the - Python script :file:`{filename}.py`. It is intended to serve as a template - for translation of the docstrings into different languages. - -If you (or your students) want to use :mod:`turtle` with online help in your -native language, you have to translate the docstrings and save the resulting -file as e.g. :file:`turtle_docstringdict_german.py`. - -If you have an appropriate entry in your :file:`turtle.cfg` file this dictionary -will be read in at import time and will replace the original English docstrings. - -At the time of this writing there are docstring dictionaries in German and in -Italian. (Requests please to glingl@aon.at.) - - - -How to configure Screen and Turtles ------------------------------------ - -The built-in default configuration mimics the appearance and behaviour of the -old turtle module in order to retain best possible compatibility with it. - -If you want to use a different configuration which better reflects the features -of this module or which better fits to your needs, e.g. for use in a classroom, -you can prepare a configuration file ``turtle.cfg`` which will be read at import -time and modify the configuration according to its settings. - -The built in configuration would correspond to the following turtle.cfg:: - - width = 0.5 - height = 0.75 - leftright = None - topbottom = None - canvwidth = 400 - canvheight = 300 - mode = standard - colormode = 1.0 - delay = 10 - undobuffersize = 1000 - shape = classic - pencolor = black - fillcolor = black - resizemode = noresize - visible = True - language = english - exampleturtle = turtle - examplescreen = screen - title = Python Turtle Graphics - using_IDLE = False - -Short explanation of selected entries: - -- The first four lines correspond to the arguments of the :meth:`Screen.setup` - method. -- Line 5 and 6 correspond to the arguments of the method - :meth:`Screen.screensize`. -- *shape* can be any of the built-in shapes, e.g: arrow, turtle, etc. For more - info try ``help(shape)``. -- If you want to use no fillcolor (i.e. make the turtle transparent), you have - to write ``fillcolor = ""`` (but all nonempty strings must not have quotes in - the cfg-file). -- If you want to reflect the turtle its state, you have to use ``resizemode = - auto``. -- If you set e.g. ``language = italian`` the docstringdict - :file:`turtle_docstringdict_italian.py` will be loaded at import time (if - present on the import path, e.g. in the same directory as :mod:`turtle`. -- The entries *exampleturtle* and *examplescreen* define the names of these - objects as they occur in the docstrings. The transformation of - method-docstrings to function-docstrings will delete these names from the - docstrings. -- *using_IDLE*: Set this to ``True`` if you regularly work with IDLE and its -n - switch ("no subprocess"). This will prevent :func:`exitonclick` to enter the - mainloop. - -There can be a :file:`turtle.cfg` file in the directory where :mod:`turtle` is -stored and an additional one in the current working directory. The latter will -override the settings of the first one. - -The :file:`Lib/turtledemo` directory contains a :file:`turtle.cfg` file. You can -study it as an example and see its effects when running the demos (preferably -not from within the demo-viewer). - - -:mod:`turtledemo` --- Demo scripts -================================== - -.. module:: turtledemo - :synopsis: A viewer for example turtle scripts - -The :mod:`turtledemo` package includes a set of demo scripts. These -scripts can be run and viewed using the supplied demo viewer as follows:: - - python -m turtledemo - -Alternatively, you can run the demo scripts individually. For example, :: - - python -m turtledemo.bytedesign - -The :mod:`turtledemo` package directory contains: - -- A demo viewer :file:`__main__.py` which can be used to view the sourcecode - of the scripts and run them at the same time. -- Multiple scripts demonstrating different features of the :mod:`turtle` - module. Examples can be accessed via the Examples menu. They can also - be run standalone. -- A :file:`turtle.cfg` file which serves as an example of how to write - and use such files. - -The demo scripts are: - -.. tabularcolumns:: |l|L|L| - -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| wikipedia | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ - -Have fun! - - -Changes since Python 2.6 -======================== - -- The methods :meth:`Turtle.tracer`, :meth:`Turtle.window_width` and - :meth:`Turtle.window_height` have been eliminated. - Methods with these names and functionality are now available only - as methods of :class:`Screen`. The functions derived from these remain - available. (In fact already in Python 2.6 these methods were merely - duplications of the corresponding - :class:`TurtleScreen`/:class:`Screen`-methods.) - -- The method :meth:`Turtle.fill` has been eliminated. - The behaviour of :meth:`begin_fill` and :meth:`end_fill` - have changed slightly: now every filling-process must be completed with an - ``end_fill()`` call. - -- A method :meth:`Turtle.filling` has been added. It returns a boolean - value: ``True`` if a filling process is under way, ``False`` otherwise. - This behaviour corresponds to a ``fill()`` call without arguments in - Python 2.6. - -Changes since Python 3.0 -======================== - -- The methods :meth:`Turtle.shearfactor`, :meth:`Turtle.shapetransform` and - :meth:`Turtle.get_shapepoly` have been added. Thus the full range of - regular linear transforms is now available for transforming turtle shapes. - :meth:`Turtle.tiltangle` has been enhanced in functionality: it now can - be used to get or set the tiltangle. :meth:`Turtle.settiltangle` has been - deprecated. - -- The method :meth:`Screen.onkeypress` has been added as a complement to - :meth:`Screen.onkey` which in fact binds actions to the keyrelease event. - Accordingly the latter has got an alias: :meth:`Screen.onkeyrelease`. - -- The method :meth:`Screen.mainloop` has been added. So when working only - with Screen and Turtle objects one must not additionally import - :func:`mainloop` anymore. - -- Two input methods has been added :meth:`Screen.textinput` and - :meth:`Screen.numinput`. These popup input dialogs and return - strings and numbers respectively. - -- Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` - have been added to the :file:`Lib/turtledemo` directory. - - -.. doctest:: - :hide: - - >>> for turtle in turtles(): - ... turtle.reset() - >>> turtle.penup() - >>> turtle.goto(-200,25) - >>> turtle.pendown() - >>> turtle.write("No one expects the Spanish Inquisition!", - ... font=("Arial", 20, "normal")) - >>> turtle.penup() - >>> turtle.goto(-100,-50) - >>> turtle.pendown() - >>> turtle.write("Our two chief Turtles are...", - ... font=("Arial", 16, "normal")) - >>> turtle.penup() - >>> turtle.goto(-450,-75) - >>> turtle.write(str(turtles())) diff --git a/third_party/python/Doc/library/types.rst b/third_party/python/Doc/library/types.rst deleted file mode 100644 index 0c5619c71..000000000 --- a/third_party/python/Doc/library/types.rst +++ /dev/null @@ -1,319 +0,0 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== - -.. module:: types - :synopsis: Names for built-in types. - -**Source code:** :source:`Lib/types.py` - --------------- - -This module defines utility function to assist in dynamic creation of -new types. - -It also defines names for some object types that are used by the standard -Python interpreter, but not exposed as builtins like :class:`int` or -:class:`str` are. - -Finally, it provides some additional type-related utility classes and functions -that are not fundamental enough to be builtins. - - -Dynamic Type Creation ---------------------- - -.. function:: new_class(name, bases=(), kwds=None, exec_body=None) - - Creates a class object dynamically using the appropriate metaclass. - - The first three arguments are the components that make up a class - definition header: the class name, the base classes (in order), the - keyword arguments (such as ``metaclass``). - - The *exec_body* argument is a callback that is used to populate the - freshly created class namespace. It should accept the class namespace - as its sole argument and update the namespace directly with the class - contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: ns``. - - .. versionadded:: 3.3 - -.. function:: prepare_class(name, bases=(), kwds=None) - - Calculates the appropriate metaclass and creates the class namespace. - - The arguments are the components that make up a class definition header: - the class name, the base classes (in order) and the keyword arguments - (such as ``metaclass``). - - The return value is a 3-tuple: ``metaclass, namespace, kwds`` - - *metaclass* is the appropriate metaclass, *namespace* is the - prepared class namespace and *kwds* is an updated copy of the passed - in *kwds* argument with any ``'metaclass'`` entry removed. If no *kwds* - argument is passed in, this will be an empty dict. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - - The default value for the ``namespace`` element of the returned - tuple has changed. Now an insertion-order-preserving mapping is - used when the metaclass does not have a ``__prepare__`` method, - -.. seealso:: - - :ref:`metaclasses` - Full details of the class creation process supported by these functions - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - - -Standard Interpreter Types --------------------------- - -This module provides names for many of the types that are required to -implement a Python interpreter. It deliberately avoids including some of -the types that arise only incidentally during processing such as the -``listiterator`` type. - -Typical use of these names is for :func:`isinstance` or -:func:`issubclass` checks. - -Standard names are defined for the following types: - -.. data:: FunctionType - LambdaType - - The type of user-defined functions and functions created by - :keyword:`lambda` expressions. - - -.. data:: GeneratorType - - The type of :term:`generator`-iterator objects, created by - generator functions. - - -.. data:: CoroutineType - - The type of :term:`coroutine` objects, created by - :keyword:`async def` functions. - - .. versionadded:: 3.5 - - -.. data:: AsyncGeneratorType - - The type of :term:`asynchronous generator`-iterator objects, created by - asynchronous generator functions. - - .. versionadded:: 3.6 - - -.. data:: CodeType - - .. index:: builtin: compile - - The type for code objects such as returned by :func:`compile`. - - -.. data:: MethodType - - The type of methods of user-defined class instances. - - -.. data:: BuiltinFunctionType - BuiltinMethodType - - The type of built-in functions like :func:`len` or :func:`sys.exit`, and - methods of built-in classes. (Here, the term "built-in" means "written in - C".) - - -.. class:: ModuleType(name, doc=None) - - The type of :term:`modules `. Constructor takes the name of the - module to be created and optionally its :term:`docstring`. - - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - -.. data:: TracebackType - - The type of traceback objects such as found in ``sys.exc_info()[2]``. - - -.. data:: FrameType - - The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a - traceback object. - - -.. data:: GetSetDescriptorType - - The type of objects defined in extension modules with ``PyGetSetDef``, such - as ``FrameType.f_locals`` or ``array.array.typecode``. This type is used as - descriptor for object attributes; it has the same purpose as the - :class:`property` type, but for classes defined in extension modules. - - -.. data:: MemberDescriptorType - - The type of objects defined in extension modules with ``PyMemberDef``, such - as ``datetime.timedelta.days``. This type is used as descriptor for simple C - data members which use standard conversion functions; it has the same purpose - as the :class:`property` type, but for classes defined in extension modules. - - .. impl-detail:: - - In other implementations of Python, this type may be identical to - ``GetSetDescriptorType``. - -.. class:: MappingProxyType(mapping) - - Read-only proxy of a mapping. It provides a dynamic view on the mapping's - entries, which means that when the mapping changes, the view reflects these - changes. - - .. versionadded:: 3.3 - - .. describe:: key in proxy - - Return ``True`` if the underlying mapping has a key *key*, else - ``False``. - - .. describe:: proxy[key] - - Return the item of the underlying mapping with key *key*. Raises a - :exc:`KeyError` if *key* is not in the underlying mapping. - - .. describe:: iter(proxy) - - Return an iterator over the keys of the underlying mapping. This is a - shortcut for ``iter(proxy.keys())``. - - .. describe:: len(proxy) - - Return the number of items in the underlying mapping. - - .. method:: copy() - - Return a shallow copy of the underlying mapping. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the underlying mapping, else - *default*. If *default* is not given, it defaults to ``None``, so that - this method never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the underlying mapping's items (``(key, value)`` - pairs). - - .. method:: keys() - - Return a new view of the underlying mapping's keys. - - .. method:: values() - - Return a new view of the underlying mapping's values. - - -Additional Utility Classes and Functions ----------------------------------------- - -.. class:: SimpleNamespace - - A simple :class:`object` subclass that provides attribute access to its - namespace, as well as a meaningful repr. - - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. - - The type is roughly equivalent to the following code:: - - class SimpleNamespace: - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - def __repr__(self): - keys = sorted(self.__dict__) - items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) - return "{}({})".format(type(self).__name__, ", ".join(items)) - - def __eq__(self, other): - return self.__dict__ == other.__dict__ - - ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. - However, for a structured record type use :func:`~collections.namedtuple` - instead. - - .. versionadded:: 3.3 - - -.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) - - Route attribute access on a class to __getattr__. - - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see Enum for an example). - - .. versionadded:: 3.4 - - -Coroutine Utility Functions ---------------------------- - -.. function:: coroutine(gen_func) - - This function transforms a :term:`generator` function into a - :term:`coroutine function` which returns a generator-based coroutine. - The generator-based coroutine is still a :term:`generator iterator`, - but is also considered to be a :term:`coroutine` object and is - :term:`awaitable`. However, it may not necessarily implement - the :meth:`__await__` method. - - If *gen_func* is a generator function, it will be modified in-place. - - If *gen_func* is not a generator function, it will be wrapped. If it - returns an instance of :class:`collections.abc.Generator`, the instance - will be wrapped in an *awaitable* proxy object. All other types - of objects will be returned as is. - - .. versionadded:: 3.5 diff --git a/third_party/python/Doc/library/typing.rst b/third_party/python/Doc/library/typing.rst deleted file mode 100644 index 915c3da33..000000000 --- a/third_party/python/Doc/library/typing.rst +++ /dev/null @@ -1,1091 +0,0 @@ -:mod:`typing` --- Support for type hints -======================================== - -.. module:: typing - :synopsis: Support for type hints (see PEP 484). - -.. versionadded:: 3.5 - -**Source code:** :source:`Lib/typing.py` - -.. note:: - - The typing module has been included in the standard library on a - :term:`provisional basis `. New features might - be added and API may change even between minor releases if deemed - necessary by the core developers. - --------------- - -This module supports type hints as specified by :pep:`484` and :pep:`526`. -The most fundamental support consists of the types :data:`Any`, :data:`Union`, -:data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and -:class:`Generic`. For full specification please see :pep:`484`. For -a simplified introduction to type hints see :pep:`483`. - - -The function below takes and returns a string and is annotated as follows:: - - def greeting(name: str) -> str: - return 'Hello ' + name - -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. - -Type aliases ------------- - -A type alias is defined by assigning the type to the alias. In this example, -``Vector`` and ``List[float]`` will be treated as interchangeable synonyms:: - - from typing import List - Vector = List[float] - - def scale(scalar: float, vector: Vector) -> Vector: - return [scalar * num for num in vector] - - # typechecks; a list of floats qualifies as a Vector. - new_vector = scale(2.0, [1.0, -4.2, 5.4]) - -Type aliases are useful for simplifying complex type signatures. For example:: - - from typing import Dict, Tuple, List - - ConnectionOptions = Dict[str, str] - Address = Tuple[str, int] - Server = Tuple[Address, ConnectionOptions] - - def broadcast_message(message: str, servers: List[Server]) -> None: - ... - - # The static type checker will treat the previous type signature as - # being exactly equivalent to this one. - def broadcast_message( - message: str, - servers: List[Tuple[Tuple[str, int], Dict[str, str]]]) -> None: - ... - -Note that ``None`` as a type hint is a special case and is replaced by -``type(None)``. - -.. _distinct: - -NewType -------- - -Use the :func:`NewType` helper function to create distinct types:: - - from typing import NewType - - UserId = NewType('UserId', int) - some_id = UserId(524313) - -The static type checker will treat the new type as if it were a subclass -of the original type. This is useful in helping catch logical errors:: - - def get_user_name(user_id: UserId) -> str: - ... - - # typechecks - user_a = get_user_name(UserId(42351)) - - # does not typecheck; an int is not a UserId - user_b = get_user_name(-1) - -You may still perform all ``int`` operations on a variable of type ``UserId``, -but the result will always be of type ``int``. This lets you pass in a -``UserId`` wherever an ``int`` might be expected, but will prevent you from -accidentally creating a ``UserId`` in an invalid way:: - - # 'output' is of type 'int', not 'UserId' - output = UserId(23413) + UserId(54341) - -Note that these checks are enforced only by the static type checker. At runtime -the statement ``Derived = NewType('Derived', Base)`` will make ``Derived`` a -function that immediately returns whatever parameter you pass it. That means -the expression ``Derived(some_value)`` does not create a new class or introduce -any overhead beyond that of a regular function call. - -More precisely, the expression ``some_value is Derived(some_value)`` is always -true at runtime. - -This also means that it is not possible to create a subtype of ``Derived`` -since it is an identity function at runtime, not an actual type:: - - from typing import NewType - - UserId = NewType('UserId', int) - - # Fails at runtime and does not typecheck - class AdminUserId(UserId): pass - -However, it is possible to create a :func:`NewType` based on a 'derived' ``NewType``:: - - from typing import NewType - - UserId = NewType('UserId', int) - - ProUserId = NewType('ProUserId', UserId) - -and typechecking for ``ProUserId`` will work as expected. - -See :pep:`484` for more details. - -.. note:: - - Recall that the use of a type alias declares two types to be *equivalent* to - one another. Doing ``Alias = Original`` will make the static type checker - treat ``Alias`` as being *exactly equivalent* to ``Original`` in all cases. - This is useful when you want to simplify complex type signatures. - - In contrast, ``NewType`` declares one type to be a *subtype* of another. - Doing ``Derived = NewType('Derived', Original)`` will make the static type - checker treat ``Derived`` as a *subclass* of ``Original``, which means a - value of type ``Original`` cannot be used in places where a value of type - ``Derived`` is expected. This is useful when you want to prevent logic - errors with minimal runtime cost. - -.. versionadded:: 3.5.2 - -Callable --------- - -Frameworks expecting callback functions of specific signatures might be -type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. - -For example:: - - from typing import Callable - - def feeder(get_next_item: Callable[[], str]) -> None: - # Body - - def async_query(on_success: Callable[[int], None], - on_error: Callable[[int, Exception], None]) -> None: - # Body - -It is possible to declare the return type of a callable without specifying -the call signature by substituting a literal ellipsis -for the list of arguments in the type hint: ``Callable[..., ReturnType]``. - -.. _generics: - -Generics --------- - -Since type information about objects kept in containers cannot be statically -inferred in a generic way, abstract base classes have been extended to support -subscription to denote expected types for container elements. - -:: - - from typing import Mapping, Sequence - - def notify_by_email(employees: Sequence[Employee], - overrides: Mapping[str, str]) -> None: ... - -Generics can be parameterized by using a new factory available in typing -called :class:`TypeVar`. - -:: - - from typing import Sequence, TypeVar - - T = TypeVar('T') # Declare type variable - - def first(l: Sequence[T]) -> T: # Generic function - return l[0] - - -User-defined generic types --------------------------- - -A user-defined class can be defined as a generic class. - -:: - - from typing import TypeVar, Generic - from logging import Logger - - T = TypeVar('T') - - class LoggedVar(Generic[T]): - def __init__(self, value: T, name: str, logger: Logger) -> None: - self.name = name - self.logger = logger - self.value = value - - def set(self, new: T) -> None: - self.log('Set ' + repr(self.value)) - self.value = new - - def get(self) -> T: - self.log('Get ' + repr(self.value)) - return self.value - - def log(self, message: str) -> None: - self.logger.info('%s: %s', self.name, message) - -``Generic[T]`` as a base class defines that the class ``LoggedVar`` takes a -single type parameter ``T`` . This also makes ``T`` valid as a type within the -class body. - -The :class:`Generic` base class uses a metaclass that defines -:meth:`__getitem__` so that ``LoggedVar[t]`` is valid as a type:: - - from typing import Iterable - - def zero_all_vars(vars: Iterable[LoggedVar[int]]) -> None: - for var in vars: - var.set(0) - -A generic type can have any number of type variables, and type variables may -be constrained:: - - from typing import TypeVar, Generic - ... - - T = TypeVar('T') - S = TypeVar('S', int, str) - - class StrangePair(Generic[T, S]): - ... - -Each type variable argument to :class:`Generic` must be distinct. -This is thus invalid:: - - from typing import TypeVar, Generic - ... - - T = TypeVar('T') - - class Pair(Generic[T, T]): # INVALID - ... - -You can use multiple inheritance with :class:`Generic`:: - - from typing import TypeVar, Generic, Sized - - T = TypeVar('T') - - class LinkedList(Sized, Generic[T]): - ... - -When inheriting from generic classes, some type variables could be fixed:: - - from typing import TypeVar, Mapping - - T = TypeVar('T') - - class MyDict(Mapping[str, T]): - ... - -In this case ``MyDict`` has a single parameter, ``T``. - -Using a generic class without specifying type parameters assumes -:data:`Any` for each position. In the following example, ``MyIterable`` is -not generic but implicitly inherits from ``Iterable[Any]``:: - - from typing import Iterable - - class MyIterable(Iterable): # Same as Iterable[Any] - -User defined generic type aliases are also supported. Examples:: - - from typing import TypeVar, Iterable, Tuple, Union - S = TypeVar('S') - Response = Union[Iterable[S], int] - - # Return type here is same as Union[Iterable[str], int] - def response(query: str) -> Response[str]: - ... - - T = TypeVar('T', int, float, complex) - Vec = Iterable[Tuple[T, T]] - - def inproduct(v: Vec[T]) -> T: # Same as Iterable[Tuple[T, T]] - return sum(x*y for x, y in v) - -The metaclass used by :class:`Generic` is a subclass of :class:`abc.ABCMeta`. -A generic class can be an ABC by including abstract methods or properties, -and generic classes can also have ABCs as base classes without a metaclass -conflict. Generic metaclasses are not supported. The outcome of parameterizing -generics is cached, and most types in the typing module are hashable and -comparable for equality. - - -The :data:`Any` type --------------------- - -A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. - -This means that it is possible to perform any operation or method call on a -value of type on :data:`Any` and assign it to any variable:: - - from typing import Any - - a = None # type: Any - a = [] # OK - a = 2 # OK - - s = '' # type: str - s = a # OK - - def foo(item: Any) -> int: - # Typechecks; 'item' could be any type, - # and that type might have a 'bar' method - item.bar() - ... - -Notice that no typechecking is performed when assigning a value of type -:data:`Any` to a more precise type. For example, the static type checker did -not report an error when assigning ``a`` to ``s`` even though ``s`` was -declared to be of type :class:`str` and receives an :class:`int` value at -runtime! - -Furthermore, all functions without a return type or parameter types will -implicitly default to using :data:`Any`:: - - def legacy_parser(text): - ... - return data - - # A static type checker will treat the above - # as having the same signature as: - def legacy_parser(text: Any) -> Any: - ... - return data - -This behavior allows :data:`Any` to be used as an *escape hatch* when you -need to mix dynamically and statically typed code. - -Contrast the behavior of :data:`Any` with the behavior of :class:`object`. -Similar to :data:`Any`, every type is a subtype of :class:`object`. However, -unlike :data:`Any`, the reverse is not true: :class:`object` is *not* a -subtype of every other type. - -That means when the type of a value is :class:`object`, a type checker will -reject almost all operations on it, and assigning it to a variable (or using -it as a return value) of a more specialized type is a type error. For example:: - - def hash_a(item: object) -> int: - # Fails; an object does not have a 'magic' method. - item.magic() - ... - - def hash_b(item: Any) -> int: - # Typechecks - item.magic() - ... - - # Typechecks, since ints and strs are subclasses of object - hash_a(42) - hash_a("foo") - - # Typechecks, since Any is compatible with all types - hash_b(42) - hash_b("foo") - -Use :class:`object` to indicate that a value could be any type in a typesafe -manner. Use :data:`Any` to indicate that a value is dynamically typed. - -Classes, functions, and decorators ----------------------------------- - -The module defines the following classes, functions and decorators: - -.. class:: TypeVar - - Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows:: - - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n - - def longest(x: A, y: A) -> A: - """Return the longest of two strings.""" - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note - that if the arguments are instances of some subclass of :class:`str`, - the return type is still plain :class:`str`. - - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, - :func:`isinstance` and :func:`issubclass` should not be used with types. - - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default type variables are invariant. Alternatively, - a type variable may specify an upper bound using ``bound=``. - This means that an actual type substituted (explicitly or implicitly) - for the type variable must be a subclass of the boundary type, - see :pep:`484`. - -.. class:: Generic - - Abstract base class for generic types. - - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - X = TypeVar('X') - Y = TypeVar('Y') - - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: - try: - return mapping[key] - except KeyError: - return default - -.. class:: Type(Generic[CT_co]) - - A variable annotated with ``C`` may accept a value of type ``C``. In - contrast, a variable annotated with ``Type[C]`` may accept values that are - classes themselves -- specifically, it will accept the *class object* of - ``C``. For example:: - - a = 3 # Has type 'int' - b = int # Has type 'Type[int]' - c = type(a) # Also has type 'Type[int]' - - Note that ``Type[C]`` is covariant:: - - class User: ... - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... - - # Accepts User, BasicUser, ProUser, TeamUser, ... - def make_new_user(user_class: Type[User]) -> User: - # ... - return user_class() - - The fact that ``Type[C]`` is covariant implies that all subclasses of - ``C`` should implement the same constructor signature and class method - signatures as ``C``. The type checker should flag violations of this, - but should also allow constructor calls in subclasses that match the - constructor calls in the indicated base class. How the type checker is - required to handle this particular case may change in future revisions of - :pep:`484`. - - The only legal parameters for :class:`Type` are classes, :data:`Any`, - :ref:`type variables `, and unions of any of these types. - For example:: - - def new_non_team_user(user_class: Type[Union[BaseUser, ProUser]]): ... - - ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent - to ``type``, which is the root of Python's metaclass hierarchy. - - .. versionadded:: 3.5.2 - -.. class:: Iterable(Generic[T_co]) - - A generic version of :class:`collections.abc.Iterable`. - -.. class:: Iterator(Iterable[T_co]) - - A generic version of :class:`collections.abc.Iterator`. - -.. class:: Reversible(Iterable[T_co]) - - A generic version of :class:`collections.abc.Reversible`. - -.. class:: SupportsInt - - An ABC with one abstract method ``__int__``. - -.. class:: SupportsFloat - - An ABC with one abstract method ``__float__``. - -.. class:: SupportsComplex - - An ABC with one abstract method ``__complex__``. - -.. class:: SupportsBytes - - An ABC with one abstract method ``__bytes__``. - -.. class:: SupportsAbs - - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. - -.. class:: SupportsRound - - An ABC with one abstract method ``__round__`` - that is covariant in its return type. - -.. class:: Container(Generic[T_co]) - - A generic version of :class:`collections.abc.Container`. - -.. class:: Hashable - - An alias to :class:`collections.abc.Hashable` - -.. class:: Sized - - An alias to :class:`collections.abc.Sized` - -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) - - A generic version of :class:`collections.abc.Collection` - - .. versionadded:: 3.6 - -.. class:: AbstractSet(Sized, Collection[T_co]) - - A generic version of :class:`collections.abc.Set`. - -.. class:: MutableSet(AbstractSet[T]) - - A generic version of :class:`collections.abc.MutableSet`. - -.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) - - A generic version of :class:`collections.abc.Mapping`. - -.. class:: MutableMapping(Mapping[KT, VT]) - - A generic version of :class:`collections.abc.MutableMapping`. - -.. class:: Sequence(Reversible[T_co], Collection[T_co]) - - A generic version of :class:`collections.abc.Sequence`. - -.. class:: MutableSequence(Sequence[T]) - - A generic version of :class:`collections.abc.MutableSequence`. - -.. class:: ByteString(Sequence[int]) - - A generic version of :class:`collections.abc.ByteString`. - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. - - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - -.. class:: Deque(deque, MutableSequence[T]) - - A generic version of :class:`collections.deque`. - - .. versionadded:: 3.6.1 - -.. class:: List(list, MutableSequence[T]) - - Generic version of :class:`list`. - Useful for annotating return types. To annotate arguments it is preferred - to use abstract collection types such as :class:`Mapping`, :class:`Sequence`, - or :class:`AbstractSet`. - - This type may be used as follows:: - - T = TypeVar('T', int, float) - - def vec2(x: T, y: T) -> List[T]: - return [x, y] - - def keep_positives(vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] - -.. class:: Set(set, MutableSet[T]) - - A generic version of :class:`builtins.set `. - -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) - - A generic version of :class:`builtins.frozenset `. - -.. class:: MappingView(Sized, Iterable[T_co]) - - A generic version of :class:`collections.abc.MappingView`. - -.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) - - A generic version of :class:`collections.abc.KeysView`. - -.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) - - A generic version of :class:`collections.abc.ItemsView`. - -.. class:: ValuesView(MappingView[VT_co]) - - A generic version of :class:`collections.abc.ValuesView`. - -.. class:: Awaitable(Generic[T_co]) - - A generic version of :class:`collections.abc.Awaitable`. - -.. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) - - A generic version of :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: - - from typing import List, Coroutine - c = None # type: Coroutine[List[str], str, int] - ... - x = c.send('hi') # type: List[str] - async def bar() -> None: - x = await c # type: int - -.. class:: AsyncIterable(Generic[T_co]) - - A generic version of :class:`collections.abc.AsyncIterable`. - -.. class:: AsyncIterator(AsyncIterable[T_co]) - - A generic version of :class:`collections.abc.AsyncIterator`. - -.. class:: ContextManager(Generic[T_co]) - - A generic version of :class:`contextlib.AbstractContextManager`. - - .. versionadded:: 3.6 - -.. class:: AsyncContextManager(Generic[T_co]) - - An ABC with async abstract :meth:`__aenter__` and :meth:`__aexit__` - methods. - - .. versionadded:: 3.6 - -.. class:: Dict(dict, MutableMapping[KT, VT]) - - A generic version of :class:`dict`. - The usage of this type is as follows:: - - def get_position_in_index(word_list: Dict[str, int], word: str) -> int: - return word_list[word] - -.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - - A generic version of :class:`collections.defaultdict`. - - .. versionadded:: 3.5.2 - -.. class:: Counter(collections.Counter, Dict[T, int]) - - A generic version of :class:`collections.Counter`. - - .. versionadded:: 3.6.1 - -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) - - A generic version of :class:`collections.ChainMap`. - - .. versionadded:: 3.6.1 - -.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) - - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: - - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' - - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. - - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: - - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 - - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 - -.. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) - - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: - - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded - - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. - - If your generator will only yield values, set the ``SendType`` to - ``None``:: - - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) - - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) - - .. versionadded:: 3.5.4 - -.. class:: Text - - ``Text`` is an alias for ``str``. It is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. - - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: - - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' - - .. versionadded:: 3.5.2 - -.. class:: IO - TextIO - BinaryIO - - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. - -.. class:: Pattern - Match - - These type aliases - correspond to the return types from :func:`re.compile` and - :func:`re.match`. These types (and the corresponding functions) - are generic in ``AnyStr`` and can be made specific by writing - ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or - ``Match[bytes]``. - -.. class:: NamedTuple - - Typed version of namedtuple. - - Usage:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - To give a field a default value, you can assign to it in the class body:: - - class Employee(NamedTuple): - name: str - id: int = 3 - - employee = Employee('Guido') - assert employee.id == 3 - - Fields with a default value must come after any fields without a default. - - The resulting class has two extra attributes: ``_field_types``, - giving a dict mapping field names to types, and ``_field_defaults``, a dict - mapping field names to default values. (The field names are in the - ``_fields`` attribute, which is part of the namedtuple API.) - - ``NamedTuple`` subclasses can also have docstrings and methods:: - - class Employee(NamedTuple): - """Represents an employee.""" - name: str - id: int = 3 - - def __repr__(self) -> str: - return f'' - - Backward-compatible usage:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - - .. versionchanged:: 3.6 - Added support for :pep:`526` variable annotation syntax. - - .. versionchanged:: 3.6.1 - Added support for default values, methods, and docstrings. - -.. function:: NewType(typ) - - A helper function to indicate a distinct types to a typechecker, - see :ref:`distinct`. At runtime it returns a function that returns - its argument. Usage:: - - UserId = NewType('UserId', int) - first_user = UserId(1) - - .. versionadded:: 3.5.2 - -.. function:: cast(typ, val) - - Cast a value to a type. - - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). - -.. function:: get_type_hints(obj[, globals[, locals]]) - - Return a dictionary containing type hints for a function, method, module - or class object. - - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. If necessary, - ``Optional[t]`` is added for function and method annotations if a default - value equal to ``None`` is set. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - -.. decorator:: overload - - The ``@overload`` decorator allows describing functions and methods - that support multiple different combinations of argument types. A series - of ``@overload``-decorated definitions must be followed by exactly one - non-``@overload``-decorated definition (for the same function/method). - The ``@overload``-decorated definitions are for the benefit of the - type checker only, since they will be overwritten by the - non-``@overload``-decorated definition, while the latter is used at - runtime but should be ignored by a type checker. At runtime, calling - a ``@overload``-decorated function directly will raise - ``NotImplementedError``. An example of overload that gives a more - precise type than can be expressed using a union or a type variable:: - - @overload - def process(response: None) -> None: - ... - @overload - def process(response: int) -> Tuple[int, str]: - ... - @overload - def process(response: bytes) -> str: - ... - def process(response): - - - See :pep:`484` for details and comparison with other typing semantics. - -.. decorator:: no_type_check - - Decorator to indicate that annotations are not type hints. - - This works as class or function :term:`decorator`. With a class, it - applies recursively to all methods defined in that class (but not - to methods defined in its superclasses or subclasses). - - This mutates the function(s) in place. - -.. decorator:: no_type_check_decorator - - Decorator to give another decorator the :func:`no_type_check` effect. - - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. - -.. data:: Any - - Special type indicating an unconstrained type. - - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. - -.. data:: NoReturn - - Special type indicating that a function never returns. - For example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise RuntimeError('no way') - - .. versionadded:: 3.6.5 - -.. data:: Union - - Union type; ``Union[X, Y]`` means either X or Y. - - To define a union, use e.g. ``Union[int, str]``. Details: - - * The arguments must be types and there must be at least one. - - * Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - * Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - * Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - * When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - * When a class and its subclass are present, the latter is skipped, e.g.:: - - Union[int, object] == object - - * You cannot subclass or instantiate a union. - - * You cannot write ``Union[X][Y]``. - - * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. - -.. data:: Optional - - Optional type. - - ``Optional[X]`` is equivalent to ``Union[X, None]``. - - Note that this is not the same concept as an optional argument, - which is one that has a default. An optional argument with a - default does not require the ``Optional`` qualifier on its type - annotation just because it is optional. For example:: - - def foo(arg: int = 0) -> None: - ... - - On the other hand, if an explicit value of ``None`` is allowed, the - use of ``Optional`` is appropriate, whether the argument is optional - or not. For example:: - - def foo(arg: Optional[int] = None) -> None: - ... - -.. data:: Tuple - - Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. - - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` - is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. - -.. data:: Callable - - Callable type; ``Callable[[int], str]`` is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or an ellipsis; the return type must be - a single type. - - There is no syntax to indicate optional or keyword arguments; - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` (literal ellipsis) can be used to - type hint a callable taking any number of arguments and returning - ``ReturnType``. A plain :data:`Callable` is equivalent to - ``Callable[..., Any]``, and in turn to - :class:`collections.abc.Callable`. - -.. data:: ClassVar - - Special type construct to mark class variables. - - As introduced in :pep:`526`, a variable annotation wrapped in ClassVar - indicates that a given attribute is intended to be used as a class variable - and should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - :data:`ClassVar` accepts only types and cannot be further subscribed. - - :data:`ClassVar` is not a class itself, and should not - be used with :func:`isinstance` or :func:`issubclass`. - :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker - might flag the following code as an error:: - - enterprise_d = Starship(3000) - enterprise_d.stats = {} # Error, setting class variable on instance - Starship.stats = {} # This is OK - - .. versionadded:: 3.5.3 - -.. data:: AnyStr - - ``AnyStr`` is a type variable defined as - ``AnyStr = TypeVar('AnyStr', str, bytes)``. - - It is meant to be used for functions that may accept any kind of string - without allowing different kinds of strings to mix. For example:: - - def concat(a: AnyStr, b: AnyStr) -> AnyStr: - return a + b - - concat(u"foo", u"bar") # Ok, output has type 'unicode' - concat(b"foo", b"bar") # Ok, output has type 'bytes' - concat(u"foo", b"bar") # Error, cannot mix unicode and bytes - -.. data:: TYPE_CHECKING - - A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. Usage:: - - if TYPE_CHECKING: - import expensive_mod - - def fun(arg: 'expensive_mod.SomeType') -> None: - local_var: expensive_mod.AnotherType = other_fun() - - Note that the first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. - - .. versionadded:: 3.5.2 diff --git a/third_party/python/Doc/library/undoc.rst b/third_party/python/Doc/library/undoc.rst deleted file mode 100644 index 2444080d6..000000000 --- a/third_party/python/Doc/library/undoc.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _undoc: - -******************** -Undocumented Modules -******************** - -Here's a quick listing of modules that are currently undocumented, but that -should be documented. Feel free to contribute documentation for them! (Send -via email to docs@python.org.) - -The idea and original contents for this chapter were taken from a posting by -Fredrik Lundh; the specific contents of this chapter have been substantially -revised. - - -Platform specific modules -========================= - -These modules are used to implement the :mod:`os.path` module, and are not -documented beyond this mention. There's little need to document these. - -:mod:`ntpath` - --- Implementation of :mod:`os.path` on Win32 and Win64 platforms. - -:mod:`posixpath` - --- Implementation of :mod:`os.path` on POSIX. diff --git a/third_party/python/Doc/library/unicodedata.rst b/third_party/python/Doc/library/unicodedata.rst deleted file mode 100644 index 2a9777609..000000000 --- a/third_party/python/Doc/library/unicodedata.rst +++ /dev/null @@ -1,173 +0,0 @@ -:mod:`unicodedata` --- Unicode Database -======================================= - -.. module:: unicodedata - :synopsis: Access the Unicode Database. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - -.. index:: - single: Unicode - single: character - pair: Unicode; database - --------------- - -This module provides access to the Unicode Character Database (UCD) which -defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 9.0.0 -`_. - -The module uses the same names and symbols as defined by Unicode -Standard Annex #44, `"Unicode Character Database" -`_. It defines the -following functions: - - -.. function:: lookup(name) - - Look up character by name. If a character with the given name is found, return - the corresponding character. If not found, :exc:`KeyError` is raised. - - .. versionchanged:: 3.3 - Support for name aliases [#]_ and named sequences [#]_ has been added. - - -.. function:: name(chr[, default]) - - Returns the name assigned to the character *chr* as a string. If no - name is defined, *default* is returned, or, if not given, :exc:`ValueError` is - raised. - - -.. function:: decimal(chr[, default]) - - Returns the decimal value assigned to the character *chr* as integer. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: digit(chr[, default]) - - Returns the digit value assigned to the character *chr* as integer. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: numeric(chr[, default]) - - Returns the numeric value assigned to the character *chr* as float. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: category(chr) - - Returns the general category assigned to the character *chr* as - string. - - -.. function:: bidirectional(chr) - - Returns the bidirectional class assigned to the character *chr* as - string. If no such value is defined, an empty string is returned. - - -.. function:: combining(chr) - - Returns the canonical combining class assigned to the character *chr* - as integer. Returns ``0`` if no combining class is defined. - - -.. function:: east_asian_width(chr) - - Returns the east asian width assigned to the character *chr* as - string. - - -.. function:: mirrored(chr) - - Returns the mirrored property assigned to the character *chr* as - integer. Returns ``1`` if the character has been identified as a "mirrored" - character in bidirectional text, ``0`` otherwise. - - -.. function:: decomposition(chr) - - Returns the character decomposition mapping assigned to the character - *chr* as string. An empty string is returned in case no such mapping is - defined. - - -.. function:: normalize(form, unistr) - - Return the normal form *form* for the Unicode string *unistr*. Valid values for - *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. - - The Unicode standard defines various normalization forms of a Unicode string, - based on the definition of canonical equivalence and compatibility equivalence. - In Unicode, several characters can be expressed in various way. For example, the - character U+00C7 (LATIN CAPITAL LETTER C WITH CEDILLA) can also be expressed as - the sequence U+0043 (LATIN CAPITAL LETTER C) U+0327 (COMBINING CEDILLA). - - For each character, there are two normal forms: normal form C and normal form D. - Normal form D (NFD) is also known as canonical decomposition, and translates - each character into its decomposed form. Normal form C (NFC) first applies a - canonical decomposition, then composes pre-combined characters again. - - In addition to these two forms, there are two additional normal forms based on - compatibility equivalence. In Unicode, certain characters are supported which - normally would be unified with other characters. For example, U+2160 (ROMAN - NUMERAL ONE) is really the same thing as U+0049 (LATIN CAPITAL LETTER I). - However, it is supported in Unicode for compatibility with existing character - sets (e.g. gb2312). - - The normal form KD (NFKD) will apply the compatibility decomposition, i.e. - replace all compatibility characters with their equivalents. The normal form KC - (NFKC) first applies the compatibility decomposition, followed by the canonical - composition. - - Even if two unicode strings are normalized and look the same to - a human reader, if one has combining characters and the other - doesn't, they may not compare equal. - - -In addition, the module exposes the following constant: - -.. data:: unidata_version - - The version of the Unicode database used in this module. - - -.. data:: ucd_3_2_0 - - This is an object that has the same methods as the entire module, but uses the - Unicode database version 3.2 instead, for applications that require this - specific version of the Unicode database (such as IDNA). - -Examples: - - >>> import unicodedata - >>> unicodedata.lookup('LEFT CURLY BRACKET') - '{' - >>> unicodedata.name('/') - 'SOLIDUS' - >>> unicodedata.decimal('9') - 9 - >>> unicodedata.decimal('a') - Traceback (most recent call last): - File "", line 1, in - ValueError: not a decimal - >>> unicodedata.category('A') # 'L'etter, 'u'ppercase - 'Lu' - >>> unicodedata.bidirectional('\u0660') # 'A'rabic, 'N'umber - 'AN' - - -.. rubric:: Footnotes - -.. [#] http://www.unicode.org/Public/9.0.0/ucd/NameAliases.txt - -.. [#] http://www.unicode.org/Public/9.0.0/ucd/NamedSequences.txt diff --git a/third_party/python/Doc/library/unittest.mock-examples.rst b/third_party/python/Doc/library/unittest.mock-examples.rst deleted file mode 100644 index cfba72795..000000000 --- a/third_party/python/Doc/library/unittest.mock-examples.rst +++ /dev/null @@ -1,1266 +0,0 @@ -:mod:`unittest.mock` --- getting started -======================================== - -.. moduleauthor:: Michael Foord -.. currentmodule:: unittest.mock - -.. versionadded:: 3.3 - - -.. _getting-started: - -Using Mock ----------- - -Mock Patching Methods -~~~~~~~~~~~~~~~~~~~~~ - -Common uses for :class:`Mock` objects include: - -* Patching methods -* Recording method calls on objects - -You might want to replace a method on an object to check that -it is called with the correct arguments by another part of the system: - - >>> real = SomeClass() - >>> real.method = MagicMock(name='method') - >>> real.method(3, 4, 5, key='value') - - -Once our mock has been used (``real.method`` in this example) it has methods -and attributes that allow you to make assertions about how it has been used. - -.. note:: - - In most of these examples the :class:`Mock` and :class:`MagicMock` classes - are interchangeable. As the ``MagicMock`` is the more capable class it makes - a sensible one to use by default. - -Once the mock has been called its :attr:`~Mock.called` attribute is set to -``True``. More importantly we can use the :meth:`~Mock.assert_called_with` or -:meth:`~Mock.assert_called_once_with` method to check that it was called with -the correct arguments. - -This example tests that calling ``ProductionClass().method`` results in a call to -the ``something`` method: - - >>> class ProductionClass: - ... def method(self): - ... self.something(1, 2, 3) - ... def something(self, a, b, c): - ... pass - ... - >>> real = ProductionClass() - >>> real.something = MagicMock() - >>> real.method() - >>> real.something.assert_called_once_with(1, 2, 3) - - - -Mock for Method Calls on an Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the last example we patched a method directly on an object to check that it -was called correctly. Another common use case is to pass an object into a -method (or some part of the system under test) and then check that it is used -in the correct way. - -The simple ``ProductionClass`` below has a ``closer`` method. If it is called with -an object then it calls ``close`` on it. - - >>> class ProductionClass: - ... def closer(self, something): - ... something.close() - ... - -So to test it we need to pass in an object with a ``close`` method and check -that it was called correctly. - - >>> real = ProductionClass() - >>> mock = Mock() - >>> real.closer(mock) - >>> mock.close.assert_called_with() - -We don't have to do any work to provide the 'close' method on our mock. -Accessing close creates it. So, if 'close' hasn't already been called then -accessing it in the test will create it, but :meth:`~Mock.assert_called_with` -will raise a failure exception. - - -Mocking Classes -~~~~~~~~~~~~~~~ - -A common use case is to mock out classes instantiated by your code under test. -When you patch a class, then that class is replaced with a mock. Instances -are created by *calling the class*. This means you access the "mock instance" -by looking at the return value of the mocked class. - -In the example below we have a function ``some_function`` that instantiates ``Foo`` -and calls a method on it. The call to :func:`patch` replaces the class ``Foo`` with a -mock. The ``Foo`` instance is the result of calling the mock, so it is configured -by modifying the mock :attr:`~Mock.return_value`. - - >>> def some_function(): - ... instance = module.Foo() - ... return instance.method() - ... - >>> with patch('module.Foo') as mock: - ... instance = mock.return_value - ... instance.method.return_value = 'the result' - ... result = some_function() - ... assert result == 'the result' - - -Naming your mocks -~~~~~~~~~~~~~~~~~ - -It can be useful to give your mocks a name. The name is shown in the repr of -the mock and can be helpful when the mock appears in test failure messages. The -name is also propagated to attributes or methods of the mock: - - >>> mock = MagicMock(name='foo') - >>> mock - - >>> mock.method - - - -Tracking all Calls -~~~~~~~~~~~~~~~~~~ - -Often you want to track more than a single call to a method. The -:attr:`~Mock.mock_calls` attribute records all calls -to child attributes of the mock - and also to their children. - - >>> mock = MagicMock() - >>> mock.method() - - >>> mock.attribute.method(10, x=53) - - >>> mock.mock_calls - [call.method(), call.attribute.method(10, x=53)] - -If you make an assertion about ``mock_calls`` and any unexpected methods -have been called, then the assertion will fail. This is useful because as well -as asserting that the calls you expected have been made, you are also checking -that they were made in the right order and with no additional calls: - -You use the :data:`call` object to construct lists for comparing with -``mock_calls``: - - >>> expected = [call.method(), call.attribute.method(10, x=53)] - >>> mock.mock_calls == expected - True - -However, parameters to calls that return mocks are not recorded, which means it is not -possible to track nested calls where the parameters used to create ancestors are important: - - >>> m = Mock() - >>> m.factory(important=True).deliver() - - >>> m.mock_calls[-1] == call.factory(important=False).deliver() - True - - -Setting Return Values and Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Setting the return values on a mock object is trivially easy: - - >>> mock = Mock() - >>> mock.return_value = 3 - >>> mock() - 3 - -Of course you can do the same for methods on the mock: - - >>> mock = Mock() - >>> mock.method.return_value = 3 - >>> mock.method() - 3 - -The return value can also be set in the constructor: - - >>> mock = Mock(return_value=3) - >>> mock() - 3 - -If you need an attribute setting on your mock, just do it: - - >>> mock = Mock() - >>> mock.x = 3 - >>> mock.x - 3 - -Sometimes you want to mock up a more complex situation, like for example -``mock.connection.cursor().execute("SELECT 1")``. If we wanted this call to -return a list, then we have to configure the result of the nested call. - -We can use :data:`call` to construct the set of calls in a "chained call" like -this for easy assertion afterwards: - - >>> mock = Mock() - >>> cursor = mock.connection.cursor.return_value - >>> cursor.execute.return_value = ['foo'] - >>> mock.connection.cursor().execute("SELECT 1") - ['foo'] - >>> expected = call.connection.cursor().execute("SELECT 1").call_list() - >>> mock.mock_calls - [call.connection.cursor(), call.connection.cursor().execute('SELECT 1')] - >>> mock.mock_calls == expected - True - -It is the call to ``.call_list()`` that turns our call object into a list of -calls representing the chained calls. - - -Raising exceptions with mocks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A useful attribute is :attr:`~Mock.side_effect`. If you set this to an -exception class or instance then the exception will be raised when the mock -is called. - - >>> mock = Mock(side_effect=Exception('Boom!')) - >>> mock() - Traceback (most recent call last): - ... - Exception: Boom! - - -Side effect functions and iterables -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``side_effect`` can also be set to a function or an iterable. The use case for -``side_effect`` as an iterable is where your mock is going to be called several -times, and you want each call to return a different value. When you set -``side_effect`` to an iterable every call to the mock returns the next value -from the iterable: - - >>> mock = MagicMock(side_effect=[4, 5, 6]) - >>> mock() - 4 - >>> mock() - 5 - >>> mock() - 6 - - -For more advanced use cases, like dynamically varying the return values -depending on what the mock is called with, ``side_effect`` can be a function. -The function will be called with the same arguments as the mock. Whatever the -function returns is what the call returns: - - >>> vals = {(1, 2): 1, (2, 3): 2} - >>> def side_effect(*args): - ... return vals[args] - ... - >>> mock = MagicMock(side_effect=side_effect) - >>> mock(1, 2) - 1 - >>> mock(2, 3) - 2 - - -Creating a Mock from an Existing Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One problem with over use of mocking is that it couples your tests to the -implementation of your mocks rather than your real code. Suppose you have a -class that implements ``some_method``. In a test for another class, you -provide a mock of this object that *also* provides ``some_method``. If later -you refactor the first class, so that it no longer has ``some_method`` - then -your tests will continue to pass even though your code is now broken! - -:class:`Mock` allows you to provide an object as a specification for the mock, -using the *spec* keyword argument. Accessing methods / attributes on the -mock that don't exist on your specification object will immediately raise an -attribute error. If you change the implementation of your specification, then -tests that use that class will start failing immediately without you having to -instantiate the class in those tests. - - >>> mock = Mock(spec=SomeClass) - >>> mock.old_method() - Traceback (most recent call last): - ... - AttributeError: object has no attribute 'old_method' - -Using a specification also enables a smarter matching of calls made to the -mock, regardless of whether some parameters were passed as positional or -named arguments:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, 3) - - >>> mock.assert_called_with(a=1, b=2, c=3) - -If you want this smarter matching to also work with method calls on the mock, -you can use :ref:`auto-speccing `. - -If you want a stronger form of specification that prevents the setting -of arbitrary attributes as well as the getting of them then you can use -*spec_set* instead of *spec*. - - - -Patch Decorators ----------------- - -.. note:: - - With :func:`patch` it matters that you patch objects in the namespace where - they are looked up. This is normally straightforward, but for a quick guide - read :ref:`where to patch `. - - -A common need in tests is to patch a class attribute or a module attribute, -for example patching a builtin or patching a class in a module to test that it -is instantiated. Modules and classes are effectively global, so patching on -them has to be undone after the test or the patch will persist into other -tests and cause hard to diagnose problems. - -mock provides three convenient decorators for this: :func:`patch`, :func:`patch.object` and -:func:`patch.dict`. ``patch`` takes a single string, of the form -``package.module.Class.attribute`` to specify the attribute you are patching. It -also optionally takes a value that you want the attribute (or class or -whatever) to be replaced with. 'patch.object' takes an object and the name of -the attribute you would like patched, plus optionally the value to patch it -with. - -``patch.object``: - - >>> original = SomeClass.attribute - >>> @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test(): - ... assert SomeClass.attribute == sentinel.attribute - ... - >>> test() - >>> assert SomeClass.attribute == original - - >>> @patch('package.module.attribute', sentinel.attribute) - ... def test(): - ... from package.module import attribute - ... assert attribute is sentinel.attribute - ... - >>> test() - -If you are patching a module (including :mod:`builtins`) then use :func:`patch` -instead of :func:`patch.object`: - - >>> mock = MagicMock(return_value=sentinel.file_handle) - >>> with patch('builtins.open', mock): - ... handle = open('filename', 'r') - ... - >>> mock.assert_called_with('filename', 'r') - >>> assert handle == sentinel.file_handle, "incorrect file handle returned" - -The module name can be 'dotted', in the form ``package.module`` if needed: - - >>> @patch('package.module.ClassName.attribute', sentinel.attribute) - ... def test(): - ... from package.module import ClassName - ... assert ClassName.attribute == sentinel.attribute - ... - >>> test() - -A nice pattern is to actually decorate test methods themselves: - - >>> class MyTest(unittest.TestCase): - ... @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test_something(self): - ... self.assertEqual(SomeClass.attribute, sentinel.attribute) - ... - >>> original = SomeClass.attribute - >>> MyTest('test_something').test_something() - >>> assert SomeClass.attribute == original - -If you want to patch with a Mock, you can use :func:`patch` with only one argument -(or :func:`patch.object` with two arguments). The mock will be created for you and -passed into the test function / method: - - >>> class MyTest(unittest.TestCase): - ... @patch.object(SomeClass, 'static_method') - ... def test_something(self, mock_method): - ... SomeClass.static_method() - ... mock_method.assert_called_with() - ... - >>> MyTest('test_something').test_something() - -You can stack up multiple patch decorators using this pattern: - - >>> class MyTest(unittest.TestCase): - ... @patch('package.module.ClassName1') - ... @patch('package.module.ClassName2') - ... def test_something(self, MockClass2, MockClass1): - ... self.assertIs(package.module.ClassName1, MockClass1) - ... self.assertIs(package.module.ClassName2, MockClass2) - ... - >>> MyTest('test_something').test_something() - -When you nest patch decorators the mocks are passed in to the decorated -function in the same order they applied (the normal *python* order that -decorators are applied). This means from the bottom up, so in the example -above the mock for ``test_module.ClassName2`` is passed in first. - -There is also :func:`patch.dict` for setting values in a dictionary just -during a scope and restoring the dictionary to its original state when the test -ends: - - >>> foo = {'key': 'value'} - >>> original = foo.copy() - >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == original - -``patch``, ``patch.object`` and ``patch.dict`` can all be used as context managers. - -Where you use :func:`patch` to create a mock for you, you can get a reference to the -mock using the "as" form of the with statement: - - >>> class ProductionClass: - ... def method(self): - ... pass - ... - >>> with patch.object(ProductionClass, 'method') as mock_method: - ... mock_method.return_value = None - ... real = ProductionClass() - ... real.method(1, 2, 3) - ... - >>> mock_method.assert_called_with(1, 2, 3) - - -As an alternative ``patch``, ``patch.object`` and ``patch.dict`` can be used as -class decorators. When used in this way it is the same as applying the -decorator individually to every method whose name starts with "test". - - -.. _further-examples: - -Further Examples ----------------- - - -Here are some more examples for some slightly more advanced scenarios. - - -Mocking chained calls -~~~~~~~~~~~~~~~~~~~~~ - -Mocking chained calls is actually straightforward with mock once you -understand the :attr:`~Mock.return_value` attribute. When a mock is called for -the first time, or you fetch its ``return_value`` before it has been called, a -new :class:`Mock` is created. - -This means that you can see how the object returned from a call to a mocked -object has been used by interrogating the ``return_value`` mock: - - >>> mock = Mock() - >>> mock().foo(a=2, b=3) - - >>> mock.return_value.foo.assert_called_with(a=2, b=3) - -From here it is a simple step to configure and then make assertions about -chained calls. Of course another alternative is writing your code in a more -testable way in the first place... - -So, suppose we have some code that looks a little bit like this: - - >>> class Something: - ... def __init__(self): - ... self.backend = BackendProvider() - ... def method(self): - ... response = self.backend.get_endpoint('foobar').create_call('spam', 'eggs').start_call() - ... # more code - -Assuming that ``BackendProvider`` is already well tested, how do we test -``method()``? Specifically, we want to test that the code section ``# more -code`` uses the response object in the correct way. - -As this chain of calls is made from an instance attribute we can monkey patch -the ``backend`` attribute on a ``Something`` instance. In this particular case -we are only interested in the return value from the final call to -``start_call`` so we don't have much configuration to do. Let's assume the -object it returns is 'file-like', so we'll ensure that our response object -uses the builtin :func:`open` as its ``spec``. - -To do this we create a mock instance as our mock backend and create a mock -response object for it. To set the response as the return value for that final -``start_call`` we could do this:: - - mock_backend.get_endpoint.return_value.create_call.return_value.start_call.return_value = mock_response - -We can do that in a slightly nicer way using the :meth:`~Mock.configure_mock` -method to directly set the return value for us: - - >>> something = Something() - >>> mock_response = Mock(spec=open) - >>> mock_backend = Mock() - >>> config = {'get_endpoint.return_value.create_call.return_value.start_call.return_value': mock_response} - >>> mock_backend.configure_mock(**config) - -With these we monkey patch the "mock backend" in place and can make the real -call: - - >>> something.backend = mock_backend - >>> something.method() - -Using :attr:`~Mock.mock_calls` we can check the chained call with a single -assert. A chained call is several calls in one line of code, so there will be -several entries in ``mock_calls``. We can use :meth:`call.call_list` to create -this list of calls for us: - - >>> chained = call.get_endpoint('foobar').create_call('spam', 'eggs').start_call() - >>> call_list = chained.call_list() - >>> assert mock_backend.mock_calls == call_list - - -Partial mocking -~~~~~~~~~~~~~~~ - -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`date.today` method. - -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real -class (and returning real instances). - -The :func:`patch decorator ` is used here to -mock out the ``date`` class in the module under test. The :attr:`side_effect` -attribute on the mock date class is then set to a lambda function that returns -a real date. When the mock date class is called a real date will be -constructed and returned by ``side_effect``. - - >>> from datetime import date - >>> with patch('mymodule.date') as mock_date: - ... mock_date.today.return_value = date(2010, 10, 8) - ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw) - ... - ... assert mymodule.date.today() == date(2010, 10, 8) - ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) - ... - -Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the -module that *uses* it. See :ref:`where to patch `. - -When ``date.today()`` is called a known date is returned, but calls to the -``date(...)`` constructor still return normal dates. Without this you can find -yourself having to calculate an expected result using exactly the same -algorithm as the code under test, which is a classic testing anti-pattern. - -Calls to the date constructor are recorded in the ``mock_date`` attributes -(``call_count`` and friends) which may also be useful for your tests. - -An alternative way of dealing with mocking dates, or other builtin classes, -is discussed in `this blog entry -`_. - - -Mocking a Generator Method -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A Python generator is a function or method that uses the :keyword:`yield` statement -to return a series of values when iterated over [#]_. - -A generator method / function is called to return the generator object. It is -the generator object that is then iterated over. The protocol method for -iteration is :meth:`~container.__iter__`, so we can -mock this using a :class:`MagicMock`. - -Here's an example class with an "iter" method implemented as a generator: - - >>> class Foo: - ... def iter(self): - ... for i in [1, 2, 3]: - ... yield i - ... - >>> foo = Foo() - >>> list(foo.iter()) - [1, 2, 3] - - -How would we mock this class, and in particular its "iter" method? - -To configure the values returned from the iteration (implicit in the call to -:class:`list`), we need to configure the object returned by the call to ``foo.iter()``. - - >>> mock_foo = MagicMock() - >>> mock_foo.iter.return_value = iter([1, 2, 3]) - >>> list(mock_foo.iter()) - [1, 2, 3] - -.. [#] There are also generator expressions and more `advanced uses - `_ of generators, but we aren't - concerned about them here. A very good introduction to generators and how - powerful they are is: `Generator Tricks for Systems Programmers - `_. - - -Applying the same patch to every test method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want several patches in place for multiple test methods the obvious way -is to apply the patch decorators to every method. This can feel like unnecessary -repetition. For Python 2.6 or more recent you can use :func:`patch` (in all its -various forms) as a class decorator. This applies the patches to all test -methods on the class. A test method is identified by methods whose names start -with ``test``: - - >>> @patch('mymodule.SomeClass') - ... class MyTest(TestCase): - ... - ... def test_one(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) - ... - ... def test_two(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) - ... - ... def not_a_test(self): - ... return 'something' - ... - >>> MyTest('test_one').test_one() - >>> MyTest('test_two').test_two() - >>> MyTest('test_two').not_a_test() - 'something' - -An alternative way of managing patches is to use the :ref:`start-and-stop`. -These allow you to move the patching into your ``setUp`` and ``tearDown`` methods. - - >>> class MyTest(TestCase): - ... def setUp(self): - ... self.patcher = patch('mymodule.foo') - ... self.mock_foo = self.patcher.start() - ... - ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) - ... - ... def tearDown(self): - ... self.patcher.stop() - ... - >>> MyTest('test_foo').run() - -If you use this technique you must ensure that the patching is "undone" by -calling ``stop``. This can be fiddlier than you might think, because if an -exception is raised in the setUp then tearDown is not called. -:meth:`unittest.TestCase.addCleanup` makes this easier: - - >>> class MyTest(TestCase): - ... def setUp(self): - ... patcher = patch('mymodule.foo') - ... self.addCleanup(patcher.stop) - ... self.mock_foo = patcher.start() - ... - ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) - ... - >>> MyTest('test_foo').run() - - -Mocking Unbound Methods -~~~~~~~~~~~~~~~~~~~~~~~ - -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. - -If you pass ``autospec=True`` to patch then it does the patching with a -*real* function object. This function object has the same signature as the one -it is replacing, but delegates to a mock under the hood. You still get your -mock auto-created in exactly the same way as before. What it means though, is -that if you use it to patch out an unbound method on a class the mocked -function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: - - >>> class Foo: - ... def foo(self): - ... pass - ... - >>> with patch.object(Foo, 'foo', autospec=True) as mock_foo: - ... mock_foo.return_value = 'foo' - ... foo = Foo() - ... foo.foo() - ... - 'foo' - >>> mock_foo.assert_called_once_with(foo) - -If we don't use ``autospec=True`` then the unbound method is patched out -with a Mock instance instead, and isn't called with ``self``. - - -Checking multiple calls with mock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -mock has a nice API for making assertions about how your mock objects are used. - - >>> mock = Mock() - >>> mock.foo_bar.return_value = None - >>> mock.foo_bar('baz', spam='eggs') - >>> mock.foo_bar.assert_called_with('baz', spam='eggs') - -If your mock is only being called once you can use the -:meth:`assert_called_once_with` method that also asserts that the -:attr:`call_count` is one. - - >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') - >>> mock.foo_bar() - >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') - Traceback (most recent call last): - ... - AssertionError: Expected to be called once. Called 2 times. - -Both ``assert_called_with`` and ``assert_called_once_with`` make assertions about -the *most recent* call. If your mock is going to be called several times, and -you want to make assertions about *all* those calls you can use -:attr:`~Mock.call_args_list`: - - >>> mock = Mock(return_value=None) - >>> mock(1, 2, 3) - >>> mock(4, 5, 6) - >>> mock() - >>> mock.call_args_list - [call(1, 2, 3), call(4, 5, 6), call()] - -The :data:`call` helper makes it easy to make assertions about these calls. You -can build up a list of expected calls and compare it to ``call_args_list``. This -looks remarkably similar to the repr of the ``call_args_list``: - - >>> expected = [call(1, 2, 3), call(4, 5, 6), call()] - >>> mock.call_args_list == expected - True - - -Coping with mutable arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Another situation is rare, but can bite you, is when your mock is called with -mutable arguments. ``call_args`` and ``call_args_list`` store *references* to the -arguments. If the arguments are mutated by the code under test then you can no -longer make assertions about what the values were when the mock was called. - -Here's some example code that shows the problem. Imagine the following functions -defined in 'mymodule':: - - def frob(val): - pass - - def grob(val): - "First frob and then clear val" - frob(val) - val.clear() - -When we try to test that ``grob`` calls ``frob`` with the correct argument look -what happens: - - >>> with patch('mymodule.frob') as mock_frob: - ... val = {6} - ... mymodule.grob(val) - ... - >>> val - set() - >>> mock_frob.assert_called_with({6}) - Traceback (most recent call last): - ... - AssertionError: Expected: (({6},), {}) - Called with: ((set(),), {}) - -One possibility would be for mock to copy the arguments you pass in. This -could then cause problems if you do assertions that rely on object identity -for equality. - -Here's one solution that uses the :attr:`side_effect` -functionality. If you provide a ``side_effect`` function for a mock then -``side_effect`` will be called with the same args as the mock. This gives us an -opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the -mock methods for doing the assertion. Again a helper function sets this up for -me. - - >>> from copy import deepcopy - >>> from unittest.mock import Mock, patch, DEFAULT - >>> def copy_call_args(mock): - ... new_mock = Mock() - ... def side_effect(*args, **kwargs): - ... args = deepcopy(args) - ... kwargs = deepcopy(kwargs) - ... new_mock(*args, **kwargs) - ... return DEFAULT - ... mock.side_effect = side_effect - ... return new_mock - ... - >>> with patch('mymodule.frob') as mock_frob: - ... new_mock = copy_call_args(mock_frob) - ... val = {6} - ... mymodule.grob(val) - ... - >>> new_mock.assert_called_with({6}) - >>> new_mock.call_args - call({6}) - -``copy_call_args`` is called with the mock that will be called. It returns a new -mock that we do the assertion on. The ``side_effect`` function makes a copy of -the args and calls our ``new_mock`` with the copy. - -.. note:: - - If your mock is only going to be used once there is an easier way of - checking arguments at the point they are called. You can simply do the - checking inside a ``side_effect`` function. - - >>> def side_effect(arg): - ... assert arg == {6} - ... - >>> mock = Mock(side_effect=side_effect) - >>> mock({6}) - >>> mock(set()) - Traceback (most recent call last): - ... - AssertionError - -An alternative approach is to create a subclass of :class:`Mock` or -:class:`MagicMock` that copies (using :func:`copy.deepcopy`) the arguments. -Here's an example implementation: - - >>> from copy import deepcopy - >>> class CopyingMock(MagicMock): - ... def __call__(self, *args, **kwargs): - ... args = deepcopy(args) - ... kwargs = deepcopy(kwargs) - ... return super(CopyingMock, self).__call__(*args, **kwargs) - ... - >>> c = CopyingMock(return_value=None) - >>> arg = set() - >>> c(arg) - >>> arg.add(1) - >>> c.assert_called_with(set()) - >>> c.assert_called_with(arg) - Traceback (most recent call last): - ... - AssertionError: Expected call: mock({1}) - Actual call: mock(set()) - >>> c.foo - - -When you subclass ``Mock`` or ``MagicMock`` all dynamically created attributes, -and the ``return_value`` will use your subclass automatically. That means all -children of a ``CopyingMock`` will also have the type ``CopyingMock``. - - -Nesting Patches -~~~~~~~~~~~~~~~ - -Using patch as a context manager is nice, but if you do multiple patches you -can end up with nested with statements indenting further and further to the -right: - - >>> class MyTest(TestCase): - ... - ... def test_foo(self): - ... with patch('mymodule.Foo') as mock_foo: - ... with patch('mymodule.Bar') as mock_bar: - ... with patch('mymodule.Spam') as mock_spam: - ... assert mymodule.Foo is mock_foo - ... assert mymodule.Bar is mock_bar - ... assert mymodule.Spam is mock_spam - ... - >>> original = mymodule.Foo - >>> MyTest('test_foo').test_foo() - >>> assert mymodule.Foo is original - -With unittest ``cleanup`` functions and the :ref:`start-and-stop` we can -achieve the same effect without the nested indentation. A simple helper -method, ``create_patch``, puts the patch in place and returns the created mock -for us: - - >>> class MyTest(TestCase): - ... - ... def create_patch(self, name): - ... patcher = patch(name) - ... thing = patcher.start() - ... self.addCleanup(patcher.stop) - ... return thing - ... - ... def test_foo(self): - ... mock_foo = self.create_patch('mymodule.Foo') - ... mock_bar = self.create_patch('mymodule.Bar') - ... mock_spam = self.create_patch('mymodule.Spam') - ... - ... assert mymodule.Foo is mock_foo - ... assert mymodule.Bar is mock_bar - ... assert mymodule.Spam is mock_spam - ... - >>> original = mymodule.Foo - >>> MyTest('test_foo').run() - >>> assert mymodule.Foo is original - - -Mocking a dictionary with MagicMock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You may want to mock a dictionary, or other container object, recording all -access to it whilst having it still behave like a dictionary. - -We can do this with :class:`MagicMock`, which will behave like a dictionary, -and using :data:`~Mock.side_effect` to delegate dictionary access to a real -underlying dictionary that is under our control. - -When the :meth:`__getitem__` and :meth:`__setitem__` methods of our ``MagicMock`` are called -(normal dictionary access) then ``side_effect`` is called with the key (and in -the case of ``__setitem__`` the value too). We can also control what is returned. - -After the ``MagicMock`` has been used we can use attributes like -:data:`~Mock.call_args_list` to assert about how the dictionary was used: - - >>> my_dict = {'a': 1, 'b': 2, 'c': 3} - >>> def getitem(name): - ... return my_dict[name] - ... - >>> def setitem(name, val): - ... my_dict[name] = val - ... - >>> mock = MagicMock() - >>> mock.__getitem__.side_effect = getitem - >>> mock.__setitem__.side_effect = setitem - -.. note:: - - An alternative to using ``MagicMock`` is to use ``Mock`` and *only* provide - the magic methods you specifically want: - - >>> mock = Mock() - >>> mock.__getitem__ = Mock(side_effect=getitem) - >>> mock.__setitem__ = Mock(side_effect=setitem) - - A *third* option is to use ``MagicMock`` but passing in ``dict`` as the *spec* - (or *spec_set*) argument so that the ``MagicMock`` created only has - dictionary magic methods available: - - >>> mock = MagicMock(spec_set=dict) - >>> mock.__getitem__.side_effect = getitem - >>> mock.__setitem__.side_effect = setitem - -With these side effect functions in place, the ``mock`` will behave like a normal -dictionary but recording the access. It even raises a :exc:`KeyError` if you try -to access a key that doesn't exist. - - >>> mock['a'] - 1 - >>> mock['c'] - 3 - >>> mock['d'] - Traceback (most recent call last): - ... - KeyError: 'd' - >>> mock['b'] = 'fish' - >>> mock['d'] = 'eggs' - >>> mock['b'] - 'fish' - >>> mock['d'] - 'eggs' - -After it has been used you can make assertions about the access using the normal -mock methods and attributes: - - >>> mock.__getitem__.call_args_list - [call('a'), call('c'), call('d'), call('b'), call('d')] - >>> mock.__setitem__.call_args_list - [call('b', 'fish'), call('d', 'eggs')] - >>> my_dict - {'a': 1, 'c': 3, 'b': 'fish', 'd': 'eggs'} - - -Mock subclasses and their attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are various reasons why you might want to subclass :class:`Mock`. One -reason might be to add helper methods. Here's a silly example: - - >>> class MyMock(MagicMock): - ... def has_been_called(self): - ... return self.called - ... - >>> mymock = MyMock(return_value=None) - >>> mymock - - >>> mymock.has_been_called() - False - >>> mymock() - >>> mymock.has_been_called() - True - -The standard behaviour for ``Mock`` instances is that attributes and the return -value mocks are of the same type as the mock they are accessed on. This ensures -that ``Mock`` attributes are ``Mocks`` and ``MagicMock`` attributes are ``MagicMocks`` -[#]_. So if you're subclassing to add helper methods then they'll also be -available on the attributes and return value mock of instances of your -subclass. - - >>> mymock.foo - - >>> mymock.foo.has_been_called() - False - >>> mymock.foo() - - >>> mymock.foo.has_been_called() - True - -Sometimes this is inconvenient. For example, `one user -`_ is subclassing mock to -created a `Twisted adaptor -`_. -Having this applied to attributes too actually causes errors. - -``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create -these "sub-mocks" for attributes and return values. You can prevent your -subclass being used for attributes by overriding this method. The signature is -that it takes arbitrary keyword arguments (``**kwargs``) which are then passed -onto the mock constructor: - - >>> class Subclass(MagicMock): - ... def _get_child_mock(self, **kwargs): - ... return MagicMock(**kwargs) - ... - >>> mymock = Subclass() - >>> mymock.foo - - >>> assert isinstance(mymock, Subclass) - >>> assert not isinstance(mymock.foo, Subclass) - >>> assert not isinstance(mymock(), Subclass) - -.. [#] An exception to this rule are the non-callable mocks. Attributes use the - callable variant because otherwise non-callable mocks couldn't have callable - methods. - - -Mocking imports with patch.dict -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One situation where mocking can be hard is where you have a local import inside -a function. These are harder to mock because they aren't using an object from -the module namespace that we can patch out. - -Generally local imports are to be avoided. They are sometimes done to prevent -circular dependencies, for which there is *usually* a much better way to solve -the problem (refactor the code) or to prevent "up front costs" by delaying the -import. This can also be solved in better ways than an unconditional local -import (store the module as a class or module attribute and only do the import -on first use). - -That aside there is a way to use ``mock`` to affect the results of an import. -Importing fetches an *object* from the :data:`sys.modules` dictionary. Note that it -fetches an *object*, which need not be a module. Importing a module for the -first time results in a module object being put in `sys.modules`, so usually -when you import something you get a module back. This need not be the case -however. - -This means you can use :func:`patch.dict` to *temporarily* put a mock in place -in :data:`sys.modules`. Any imports whilst this patch is active will fetch the mock. -When the patch is complete (the decorated function exits, the with statement -body is complete or ``patcher.stop()`` is called) then whatever was there -previously will be restored safely. - -Here's an example that mocks out the 'fooble' module. - - >>> mock = Mock() - >>> with patch.dict('sys.modules', {'fooble': mock}): - ... import fooble - ... fooble.blob() - ... - - >>> assert 'fooble' not in sys.modules - >>> mock.blob.assert_called_once_with() - -As you can see the ``import fooble`` succeeds, but on exit there is no 'fooble' -left in :data:`sys.modules`. - -This also works for the ``from module import name`` form: - - >>> mock = Mock() - >>> with patch.dict('sys.modules', {'fooble': mock}): - ... from fooble import blob - ... blob.blip() - ... - - >>> mock.blob.blip.assert_called_once_with() - -With slightly more work you can also mock package imports: - - >>> mock = Mock() - >>> modules = {'package': mock, 'package.module': mock.module} - >>> with patch.dict('sys.modules', modules): - ... from package.module import fooble - ... fooble() - ... - - >>> mock.module.fooble.assert_called_once_with() - - -Tracking order of calls and less verbose call assertions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`Mock` class allows you to track the *order* of method calls on -your mock objects through the :attr:`~Mock.method_calls` attribute. This -doesn't allow you to track the order of calls between separate mock objects, -however we can use :attr:`~Mock.mock_calls` to achieve the same effect. - -Because mocks track calls to child mocks in ``mock_calls``, and accessing an -arbitrary attribute of a mock creates a child mock, we can create our separate -mocks from a parent one. Calls to those child mock will then all be recorded, -in order, in the ``mock_calls`` of the parent: - - >>> manager = Mock() - >>> mock_foo = manager.foo - >>> mock_bar = manager.bar - - >>> mock_foo.something() - - >>> mock_bar.other.thing() - - - >>> manager.mock_calls - [call.foo.something(), call.bar.other.thing()] - -We can then assert about the calls, including the order, by comparing with -the ``mock_calls`` attribute on the manager mock: - - >>> expected_calls = [call.foo.something(), call.bar.other.thing()] - >>> manager.mock_calls == expected_calls - True - -If ``patch`` is creating, and putting in place, your mocks then you can attach -them to a manager mock using the :meth:`~Mock.attach_mock` method. After -attaching calls will be recorded in ``mock_calls`` of the manager. - - >>> manager = MagicMock() - >>> with patch('mymodule.Class1') as MockClass1: - ... with patch('mymodule.Class2') as MockClass2: - ... manager.attach_mock(MockClass1, 'MockClass1') - ... manager.attach_mock(MockClass2, 'MockClass2') - ... MockClass1().foo() - ... MockClass2().bar() - ... - - - >>> manager.mock_calls - [call.MockClass1(), - call.MockClass1().foo(), - call.MockClass2(), - call.MockClass2().bar()] - -If many calls have been made, but you're only interested in a particular -sequence of them then an alternative is to use the -:meth:`~Mock.assert_has_calls` method. This takes a list of calls (constructed -with the :data:`call` object). If that sequence of calls are in -:attr:`~Mock.mock_calls` then the assert succeeds. - - >>> m = MagicMock() - >>> m().foo().bar().baz() - - >>> m.one().two().three() - - >>> calls = call.one().two().three().call_list() - >>> m.assert_has_calls(calls) - -Even though the chained call ``m.one().two().three()`` aren't the only calls that -have been made to the mock, the assert still succeeds. - -Sometimes a mock may have several calls made to it, and you are only interested -in asserting about *some* of those calls. You may not even care about the -order. In this case you can pass ``any_order=True`` to ``assert_has_calls``: - - >>> m = MagicMock() - >>> m(1), m.two(2, 3), m.seven(7), m.fifty('50') - (...) - >>> calls = [call.fifty('50'), call(1), call.seven(7)] - >>> m.assert_has_calls(calls, any_order=True) - - -More complex argument matching -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Using the same basic concept as :data:`ANY` we can implement matchers to do more -complex assertions on objects used as arguments to mocks. - -Suppose we expect some object to be passed to a mock that by default -compares equal based on object identity (which is the Python default for user -defined classes). To use :meth:`~Mock.assert_called_with` we would need to pass -in the exact same object. If we are only interested in some of the attributes -of this object then we can create a matcher that will check these attributes -for us. - -You can see in this example how a 'standard' call to ``assert_called_with`` isn't -sufficient: - - >>> class Foo: - ... def __init__(self, a, b): - ... self.a, self.b = a, b - ... - >>> mock = Mock(return_value=None) - >>> mock(Foo(1, 2)) - >>> mock.assert_called_with(Foo(1, 2)) - Traceback (most recent call last): - ... - AssertionError: Expected: call(<__main__.Foo object at 0x...>) - Actual call: call(<__main__.Foo object at 0x...>) - -A comparison function for our ``Foo`` class might look something like this: - - >>> def compare(self, other): - ... if not type(self) == type(other): - ... return False - ... if self.a != other.a: - ... return False - ... if self.b != other.b: - ... return False - ... return True - ... - -And a matcher object that can use comparison functions like this for its -equality operation would look something like this: - - >>> class Matcher: - ... def __init__(self, compare, some_obj): - ... self.compare = compare - ... self.some_obj = some_obj - ... def __eq__(self, other): - ... return self.compare(self.some_obj, other) - ... - -Putting all this together: - - >>> match_foo = Matcher(compare, Foo(1, 2)) - >>> mock.assert_called_with(match_foo) - -The ``Matcher`` is instantiated with our compare function and the ``Foo`` object -we want to compare against. In ``assert_called_with`` the ``Matcher`` equality -method will be called, which compares the object the mock was called with -against the one we created our matcher with. If they match then -``assert_called_with`` passes, and if they don't an :exc:`AssertionError` is raised: - - >>> match_wrong = Matcher(compare, Foo(3, 4)) - >>> mock.assert_called_with(match_wrong) - Traceback (most recent call last): - ... - AssertionError: Expected: ((,), {}) - Called with: ((,), {}) - -With a bit of tweaking you could have the comparison function raise the -:exc:`AssertionError` directly and provide a more useful failure message. - -As of version 1.5, the Python testing library `PyHamcrest -`_ provides similar functionality, -that may be useful here, in the form of its equality matcher -(`hamcrest.library.integration.match_equality -`_). diff --git a/third_party/python/Doc/library/unittest.mock.rst b/third_party/python/Doc/library/unittest.mock.rst deleted file mode 100644 index 01f5a6454..000000000 --- a/third_party/python/Doc/library/unittest.mock.rst +++ /dev/null @@ -1,2379 +0,0 @@ - -:mod:`unittest.mock` --- mock object library -============================================ - -.. module:: unittest.mock - :synopsis: Mock object library. - -.. moduleauthor:: Michael Foord -.. currentmodule:: unittest.mock - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/unittest/mock.py` - --------------- - -:mod:`unittest.mock` is a library for testing in Python. It allows you to -replace parts of your system under test with mock objects and make assertions -about how they have been used. - -:mod:`unittest.mock` provides a core :class:`Mock` class removing the need to -create a host of stubs throughout your test suite. After performing an -action, you can make assertions about which methods / attributes were used -and arguments they were called with. You can also specify return values and -set needed attributes in the normal way. - -Additionally, mock provides a :func:`patch` decorator that handles patching -module and class level attributes within the scope of a test, along with -:const:`sentinel` for creating unique objects. See the `quick guide`_ for -some examples of how to use :class:`Mock`, :class:`MagicMock` and -:func:`patch`. - -Mock is very easy to use and is designed for use with :mod:`unittest`. Mock -is based on the 'action -> assertion' pattern instead of 'record -> replay' -used by many mocking frameworks. - -There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI `_. - - -Quick Guide ------------ - -:class:`Mock` and :class:`MagicMock` objects create all attributes and -methods as you access them and store details of how they have been used. You -can configure them, to specify return values or limit what attributes are -available, and then make assertions about how they have been used: - - >>> from unittest.mock import MagicMock - >>> thing = ProductionClass() - >>> thing.method = MagicMock(return_value=3) - >>> thing.method(3, 4, 5, key='value') - 3 - >>> thing.method.assert_called_with(3, 4, 5, key='value') - -:attr:`side_effect` allows you to perform side effects, including raising an -exception when a mock is called: - - >>> mock = Mock(side_effect=KeyError('foo')) - >>> mock() - Traceback (most recent call last): - ... - KeyError: 'foo' - - >>> values = {'a': 1, 'b': 2, 'c': 3} - >>> def side_effect(arg): - ... return values[arg] - ... - >>> mock.side_effect = side_effect - >>> mock('a'), mock('b'), mock('c') - (1, 2, 3) - >>> mock.side_effect = [5, 4, 3, 2, 1] - >>> mock(), mock(), mock() - (5, 4, 3) - -Mock has many other ways you can configure it and control its behaviour. For -example the *spec* argument configures the mock to take its specification -from another object. Attempting to access attributes or methods on the mock -that don't exist on the spec will fail with an :exc:`AttributeError`. - -The :func:`patch` decorator / context manager makes it easy to mock classes or -objects in a module under test. The object you specify will be replaced with a -mock (or other object) during the test and restored when the test ends: - - >>> from unittest.mock import patch - >>> @patch('module.ClassName2') - ... @patch('module.ClassName1') - ... def test(MockClass1, MockClass2): - ... module.ClassName1() - ... module.ClassName2() - ... assert MockClass1 is module.ClassName1 - ... assert MockClass2 is module.ClassName2 - ... assert MockClass1.called - ... assert MockClass2.called - ... - >>> test() - -.. note:: - - When you nest patch decorators the mocks are passed in to the decorated - function in the same order they applied (the normal *python* order that - decorators are applied). This means from the bottom up, so in the example - above the mock for ``module.ClassName1`` is passed in first. - - With :func:`patch` it matters that you patch objects in the namespace where they - are looked up. This is normally straightforward, but for a quick guide - read :ref:`where to patch `. - -As well as a decorator :func:`patch` can be used as a context manager in a with -statement: - - >>> with patch.object(ProductionClass, 'method', return_value=None) as mock_method: - ... thing = ProductionClass() - ... thing.method(1, 2, 3) - ... - >>> mock_method.assert_called_once_with(1, 2, 3) - - -There is also :func:`patch.dict` for setting values in a dictionary just -during a scope and restoring the dictionary to its original state when the test -ends: - - >>> foo = {'key': 'value'} - >>> original = foo.copy() - >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == original - -Mock supports the mocking of Python :ref:`magic methods `. The -easiest way of using magic methods is with the :class:`MagicMock` class. It -allows you to do things like: - - >>> mock = MagicMock() - >>> mock.__str__.return_value = 'foobarbaz' - >>> str(mock) - 'foobarbaz' - >>> mock.__str__.assert_called_with() - -Mock allows you to assign functions (or other Mock instances) to magic methods -and they will be called appropriately. The :class:`MagicMock` class is just a Mock -variant that has all of the magic methods pre-created for you (well, all the -useful ones anyway). - -The following is an example of using magic methods with the ordinary Mock -class: - - >>> mock = Mock() - >>> mock.__str__ = Mock(return_value='wheeeeee') - >>> str(mock) - 'wheeeeee' - -For ensuring that the mock objects in your tests have the same api as the -objects they are replacing, you can use :ref:`auto-speccing `. -Auto-speccing can be done through the *autospec* argument to patch, or the -:func:`create_autospec` function. Auto-speccing creates mock objects that -have the same attributes and methods as the objects they are replacing, and -any functions and methods (including constructors) have the same call -signature as the real object. - -This ensures that your mocks will fail in the same way as your production -code if they are used incorrectly: - - >>> from unittest.mock import create_autospec - >>> def function(a, b, c): - ... pass - ... - >>> mock_function = create_autospec(function, return_value='fishy') - >>> mock_function(1, 2, 3) - 'fishy' - >>> mock_function.assert_called_once_with(1, 2, 3) - >>> mock_function('wrong arguments') - Traceback (most recent call last): - ... - TypeError: () takes exactly 3 arguments (1 given) - -:func:`create_autospec` can also be used on classes, where it copies the signature of -the ``__init__`` method, and on callable objects where it copies the signature of -the ``__call__`` method. - - - -The Mock Class --------------- - - -:class:`Mock` is a flexible mock object intended to replace the use of stubs and -test doubles throughout your code. Mocks are callable and create attributes as -new mocks when you access them [#]_. Accessing the same attribute will always -return the same mock. Mocks record how you use them, allowing you to make -assertions about what your code has done to them. - -:class:`MagicMock` is a subclass of :class:`Mock` with all the magic methods -pre-created and ready to use. There are also non-callable variants, useful -when you are mocking out objects that aren't callable: -:class:`NonCallableMock` and :class:`NonCallableMagicMock` - -The :func:`patch` decorators makes it easy to temporarily replace classes -in a particular module with a :class:`Mock` object. By default :func:`patch` will create -a :class:`MagicMock` for you. You can specify an alternative class of :class:`Mock` using -the *new_callable* argument to :func:`patch`. - - -.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) - - Create a new :class:`Mock` object. :class:`Mock` takes several optional arguments - that specify the behaviour of the Mock object: - - * *spec*: This can be either a list of strings or an existing object (a - class or instance) that acts as the specification for the mock object. If - you pass in an object then a list of strings is formed by calling dir on - the object (excluding unsupported magic attributes and methods). - Accessing any attribute not in this list will raise an :exc:`AttributeError`. - - If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This - allows mocks to pass :func:`isinstance` tests. - - * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* - or get an attribute on the mock that isn't on the object passed as - *spec_set* will raise an :exc:`AttributeError`. - - * *side_effect*: A function to be called whenever the Mock is called. See - the :attr:`~Mock.side_effect` attribute. Useful for raising exceptions or - dynamically changing return values. The function is called with the same - arguments as the mock, and unless it returns :data:`DEFAULT`, the return - value of this function is used as the return value. - - Alternatively *side_effect* can be an exception class or instance. In - this case the exception will be raised when the mock is called. - - If *side_effect* is an iterable then each call to the mock will return - the next value from the iterable. - - A *side_effect* can be cleared by setting it to ``None``. - - * *return_value*: The value returned when the mock is called. By default - this is a new Mock (created on first access). See the - :attr:`return_value` attribute. - - * *unsafe*: By default if any attribute starts with *assert* or - *assret* will raise an :exc:`AttributeError`. Passing ``unsafe=True`` - will allow access to these attributes. - - .. versionadded:: 3.5 - - * *wraps*: Item for the mock object to wrap. If *wraps* is not ``None`` then - calling the Mock will pass the call through to the wrapped object - (returning the real result). Attribute access on the mock will return a - Mock object that wraps the corresponding attribute of the wrapped - object (so attempting to access an attribute that doesn't exist will - raise an :exc:`AttributeError`). - - If the mock has an explicit *return_value* set then calls are not passed - to the wrapped object and the *return_value* is returned instead. - - * *name*: If the mock has a name then it will be used in the repr of the - mock. This can be useful for debugging. The name is propagated to child - mocks. - - Mocks can also be called with arbitrary keyword arguments. These will be - used to set attributes on the mock after it is created. See the - :meth:`configure_mock` method for details. - - .. method:: assert_called(*args, **kwargs) - - Assert that the mock was called at least once. - - >>> mock = Mock() - >>> mock.method() - - >>> mock.method.assert_called() - - .. versionadded:: 3.6 - - .. method:: assert_called_once(*args, **kwargs) - - Assert that the mock was called exactly once. - - >>> mock = Mock() - >>> mock.method() - - >>> mock.method.assert_called_once() - >>> mock.method() - - >>> mock.method.assert_called_once() - Traceback (most recent call last): - ... - AssertionError: Expected 'method' to have been called once. Called 2 times. - - .. versionadded:: 3.6 - - - .. method:: assert_called_with(*args, **kwargs) - - This method is a convenient way of asserting that calls are made in a - particular way: - - >>> mock = Mock() - >>> mock.method(1, 2, 3, test='wow') - - >>> mock.method.assert_called_with(1, 2, 3, test='wow') - - .. method:: assert_called_once_with(*args, **kwargs) - - Assert that the mock was called exactly once and that that call was - with the specified arguments. - - >>> mock = Mock(return_value=None) - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('other', bar='values') - >>> mock.assert_called_once_with('other', bar='values') - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - - - .. method:: assert_any_call(*args, **kwargs) - - assert the mock has been called with the specified arguments. - - The assert passes if the mock has *ever* been called, unlike - :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one, and in the case of - :meth:`assert_called_once_with` it must also be the only call. - - >>> mock = Mock(return_value=None) - >>> mock(1, 2, arg='thing') - >>> mock('some', 'thing', 'else') - >>> mock.assert_any_call(1, 2, arg='thing') - - - .. method:: assert_has_calls(calls, any_order=False) - - assert the mock has been called with the specified calls. - The :attr:`mock_calls` list is checked for the calls. - - If *any_order* is false (the default) then the calls must be - sequential. There can be extra calls before or after the - specified calls. - - If *any_order* is true then the calls can be in any order, but - they must all appear in :attr:`mock_calls`. - - >>> mock = Mock(return_value=None) - >>> mock(1) - >>> mock(2) - >>> mock(3) - >>> mock(4) - >>> calls = [call(2), call(3)] - >>> mock.assert_has_calls(calls) - >>> calls = [call(4), call(2), call(3)] - >>> mock.assert_has_calls(calls, any_order=True) - - .. method:: assert_not_called() - - Assert the mock was never called. - - >>> m = Mock() - >>> m.hello.assert_not_called() - >>> obj = m.hello() - >>> m.hello.assert_not_called() - Traceback (most recent call last): - ... - AssertionError: Expected 'hello' to not have been called. Called 1 times. - - .. versionadded:: 3.5 - - - .. method:: reset_mock(*, return_value=False, side_effect=False) - - The reset_mock method resets all the call attributes on a mock object: - - >>> mock = Mock(return_value=None) - >>> mock('hello') - >>> mock.called - True - >>> mock.reset_mock() - >>> mock.called - False - - .. versionchanged:: 3.6 - Added two keyword only argument to the reset_mock function. - - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. - - .. note:: *return_value*, and :attr:`side_effect` are keyword only - argument. - - - .. method:: mock_add_spec(spec, spec_set=False) - - Add a spec to a mock. *spec* can either be an object or a - list of strings. Only attributes on the *spec* can be fetched as - attributes from the mock. - - If *spec_set* is true then only attributes on the spec can be set. - - - .. method:: attach_mock(mock, attribute) - - Attach a mock as an attribute of this one, replacing its name and - parent. Calls to the attached mock will be recorded in the - :attr:`method_calls` and :attr:`mock_calls` attributes of this one. - - - .. method:: configure_mock(**kwargs) - - Set attributes on the mock through keyword arguments. - - Attributes plus return values and side effects can be set on child - mocks using standard dot notation and unpacking a dictionary in the - method call: - - >>> mock = Mock() - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock.configure_mock(**attrs) - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - - The same thing can be achieved in the constructor call to mocks: - - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock = Mock(some_attribute='eggs', **attrs) - >>> mock.some_attribute - 'eggs' - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - - :meth:`configure_mock` exists to make it easier to do configuration - after the mock has been created. - - - .. method:: __dir__() - - :class:`Mock` objects limit the results of ``dir(some_mock)`` to useful results. - For mocks with a *spec* this includes all the permitted attributes - for the mock. - - See :data:`FILTER_DIR` for what this filtering does, and how to - switch it off. - - - .. method:: _get_child_mock(**kw) - - Create the child mocks for attributes and return value. - By default child mocks will be the same type as the parent. - Subclasses of Mock may want to override this to customize the way - child mocks are made. - - For non-callable mocks the callable variant will be used (rather than - any custom subclass). - - - .. attribute:: called - - A boolean representing whether or not the mock object has been called: - - >>> mock = Mock(return_value=None) - >>> mock.called - False - >>> mock() - >>> mock.called - True - - .. attribute:: call_count - - An integer telling you how many times the mock object has been called: - - >>> mock = Mock(return_value=None) - >>> mock.call_count - 0 - >>> mock() - >>> mock() - >>> mock.call_count - 2 - - - .. attribute:: return_value - - Set this to configure the value returned by calling the mock: - - >>> mock = Mock() - >>> mock.return_value = 'fish' - >>> mock() - 'fish' - - The default return value is a mock object and you can configure it in - the normal way: - - >>> mock = Mock() - >>> mock.return_value.attribute = sentinel.Attribute - >>> mock.return_value() - - >>> mock.return_value.assert_called_with() - - :attr:`return_value` can also be set in the constructor: - - >>> mock = Mock(return_value=3) - >>> mock.return_value - 3 - >>> mock() - 3 - - - .. attribute:: side_effect - - This can either be a function to be called when the mock is called, - an iterable or an exception (class or instance) to be raised. - - If you pass in a function it will be called with same arguments as the - mock and unless the function returns the :data:`DEFAULT` singleton the - call to the mock will then return whatever the function returns. If the - function returns :data:`DEFAULT` then the mock will return its normal - value (from the :attr:`return_value`). - - If you pass in an iterable, it is used to retrieve an iterator which - must yield a value on every call. This value can either be an exception - instance to be raised, or a value to be returned from the call to the - mock (:data:`DEFAULT` handling is identical to the function case). - - An example of a mock that raises an exception (to test exception - handling of an API): - - >>> mock = Mock() - >>> mock.side_effect = Exception('Boom!') - >>> mock() - Traceback (most recent call last): - ... - Exception: Boom! - - Using :attr:`side_effect` to return a sequence of values: - - >>> mock = Mock() - >>> mock.side_effect = [3, 2, 1] - >>> mock(), mock(), mock() - (3, 2, 1) - - Using a callable: - - >>> mock = Mock(return_value=3) - >>> def side_effect(*args, **kwargs): - ... return DEFAULT - ... - >>> mock.side_effect = side_effect - >>> mock() - 3 - - :attr:`side_effect` can be set in the constructor. Here's an example that - adds one to the value the mock is called with and returns it: - - >>> side_effect = lambda value: value + 1 - >>> mock = Mock(side_effect=side_effect) - >>> mock(3) - 4 - >>> mock(-8) - -7 - - Setting :attr:`side_effect` to ``None`` clears it: - - >>> m = Mock(side_effect=KeyError, return_value=3) - >>> m() - Traceback (most recent call last): - ... - KeyError - >>> m.side_effect = None - >>> m() - 3 - - - .. attribute:: call_args - - This is either ``None`` (if the mock hasn't been called), or the - arguments that the mock was last called with. This will be in the - form of a tuple: the first member is any ordered arguments the mock - was called with (or an empty tuple) and the second member is any - keyword arguments (or an empty dictionary). - - >>> mock = Mock(return_value=None) - >>> print(mock.call_args) - None - >>> mock() - >>> mock.call_args - call() - >>> mock.call_args == () - True - >>> mock(3, 4) - >>> mock.call_args - call(3, 4) - >>> mock.call_args == ((3, 4),) - True - >>> mock(3, 4, 5, key='fish', next='w00t!') - >>> mock.call_args - call(3, 4, 5, key='fish', next='w00t!') - - :attr:`call_args`, along with members of the lists :attr:`call_args_list`, - :attr:`method_calls` and :attr:`mock_calls` are :data:`call` objects. - These are tuples, so they can be unpacked to get at the individual - arguments and make more complex assertions. See - :ref:`calls as tuples `. - - - .. attribute:: call_args_list - - This is a list of all the calls made to the mock object in sequence - (so the length of the list is the number of times it has been - called). Before any calls have been made it is an empty list. The - :data:`call` object can be used for conveniently constructing lists of - calls to compare with :attr:`call_args_list`. - - >>> mock = Mock(return_value=None) - >>> mock() - >>> mock(3, 4) - >>> mock(key='fish', next='w00t!') - >>> mock.call_args_list - [call(), call(3, 4), call(key='fish', next='w00t!')] - >>> expected = [(), ((3, 4),), ({'key': 'fish', 'next': 'w00t!'},)] - >>> mock.call_args_list == expected - True - - Members of :attr:`call_args_list` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - - .. attribute:: method_calls - - As well as tracking calls to themselves, mocks also track calls to - methods and attributes, and *their* methods and attributes: - - >>> mock = Mock() - >>> mock.method() - - >>> mock.property.method.attribute() - - >>> mock.method_calls - [call.method(), call.property.method.attribute()] - - Members of :attr:`method_calls` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - - .. attribute:: mock_calls - - :attr:`mock_calls` records *all* calls to the mock object, its methods, - magic methods *and* return value mocks. - - >>> mock = MagicMock() - >>> result = mock(1, 2, 3) - >>> mock.first(a=3) - - >>> mock.second() - - >>> int(mock) - 1 - >>> result(1) - - >>> expected = [call(1, 2, 3), call.first(a=3), call.second(), - ... call.__int__(), call()(1)] - >>> mock.mock_calls == expected - True - - Members of :attr:`mock_calls` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - .. note:: - - The way :attr:`mock_calls` are recorded means that where nested - calls are made, the parameters of ancestor calls are not recorded - and so will always compare equal: - - >>> mock = MagicMock() - >>> mock.top(a=3).bottom() - - >>> mock.mock_calls - [call.top(a=3), call.top().bottom()] - >>> mock.mock_calls[-1] == call.top(a=-1).bottom() - True - - .. attribute:: __class__ - - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class - instead. This allows mock objects to pass :func:`isinstance` tests for the - object they are replacing / masquerading as: - - >>> mock = Mock(spec=3) - >>> isinstance(mock, int) - True - - :attr:`__class__` is assignable to, this allows a mock to pass an - :func:`isinstance` check without forcing you to use a spec: - - >>> mock = Mock() - >>> mock.__class__ = dict - >>> isinstance(mock, dict) - True - -.. class:: NonCallableMock(spec=None, wraps=None, name=None, spec_set=None, **kwargs) - - A non-callable version of :class:`Mock`. The constructor parameters have the same - meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* - which have no meaning on a non-callable mock. - -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: - - >>> mock = Mock(spec=SomeClass) - >>> isinstance(mock, SomeClass) - True - >>> mock = Mock(spec_set=SomeClass()) - >>> isinstance(mock, SomeClass) - True - -The :class:`Mock` classes have support for mocking magic methods. See :ref:`magic -methods ` for the full details. - -The mock classes and the :func:`patch` decorators all take arbitrary keyword -arguments for configuration. For the :func:`patch` decorators the keywords are -passed to the constructor of the mock being created. The keyword arguments -are for configuring attributes of the mock: - - >>> m = MagicMock(attribute=3, other='fish') - >>> m.attribute - 3 - >>> m.other - 'fish' - -The return value and side effect of child mocks can be set in the same way, -using dotted notation. As you can't use dotted names directly in a call you -have to create a dictionary and unpack it using ``**``: - - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock = Mock(some_attribute='eggs', **attrs) - >>> mock.some_attribute - 'eggs' - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - -A callable mock which was created with a *spec* (or a *spec_set*) will -introspect the specification object's signature when matching calls to -the mock. Therefore, it can match the actual call's arguments regardless -of whether they were passed positionally or by name:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, c=3) - - >>> mock.assert_called_with(1, 2, 3) - >>> mock.assert_called_with(a=1, b=2, c=3) - -This applies to :meth:`~Mock.assert_called_with`, -:meth:`~Mock.assert_called_once_with`, :meth:`~Mock.assert_has_calls` and -:meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also -apply to method calls on the mock object. - - .. versionchanged:: 3.4 - Added signature introspection on specced and autospecced mock objects. - - -.. class:: PropertyMock(*args, **kwargs) - - A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods - so you can specify a return value when it is fetched. - - Fetching a :class:`PropertyMock` instance from an object calls the mock, with - no args. Setting it calls the mock with the value being set. - - >>> class Foo: - ... @property - ... def foo(self): - ... return 'something' - ... @foo.setter - ... def foo(self, value): - ... pass - ... - >>> with patch('__main__.Foo.foo', new_callable=PropertyMock) as mock_foo: - ... mock_foo.return_value = 'mockity-mock' - ... this_foo = Foo() - ... print(this_foo.foo) - ... this_foo.foo = 6 - ... - mockity-mock - >>> mock_foo.mock_calls - [call(), call(6)] - -Because of the way mock attributes are stored you can't directly attach a -:class:`PropertyMock` to a mock object. Instead you can attach it to the mock type -object:: - - >>> m = MagicMock() - >>> p = PropertyMock(return_value=3) - >>> type(m).foo = p - >>> m.foo - 3 - >>> p.assert_called_once_with() - - -Calling -~~~~~~~ - -Mock objects are callable. The call will return the value set as the -:attr:`~Mock.return_value` attribute. The default return value is a new Mock -object; it is created the first time the return value is accessed (either -explicitly or by calling the Mock) - but it is stored and the same one -returned each time. - -Calls made to the object will be recorded in the attributes -like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. - -If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still -recorded. - -The simplest way to make a mock raise an exception when called is to make -:attr:`~Mock.side_effect` an exception class or instance: - - >>> m = MagicMock(side_effect=IndexError) - >>> m(1, 2, 3) - Traceback (most recent call last): - ... - IndexError - >>> m.mock_calls - [call(1, 2, 3)] - >>> m.side_effect = KeyError('Bang!') - >>> m('two', 'three', 'four') - Traceback (most recent call last): - ... - KeyError: 'Bang!' - >>> m.mock_calls - [call(1, 2, 3), call('two', 'three', 'four')] - -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the -same arguments as the mock. This allows you to vary the return value of the -call dynamically, based on the input: - - >>> def side_effect(value): - ... return value + 1 - ... - >>> m = MagicMock(side_effect=side_effect) - >>> m(1) - 2 - >>> m(2) - 3 - >>> m.mock_calls - [call(1), call(2)] - -If you want the mock to still return the default return value (a new mock), or -any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: - - >>> m = MagicMock() - >>> def side_effect(*args, **kwargs): - ... return m.return_value - ... - >>> m.side_effect = side_effect - >>> m.return_value = 3 - >>> m() - 3 - >>> def side_effect(*args, **kwargs): - ... return DEFAULT - ... - >>> m.side_effect = side_effect - >>> m() - 3 - -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: - - >>> m = MagicMock(return_value=6) - >>> def side_effect(*args, **kwargs): - ... return 3 - ... - >>> m.side_effect = side_effect - >>> m() - 3 - >>> m.side_effect = None - >>> m() - 6 - -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock -will return values from the iterable (until the iterable is exhausted and -a :exc:`StopIteration` is raised): - - >>> m = MagicMock(side_effect=[1, 2, 3]) - >>> m() - 1 - >>> m() - 2 - >>> m() - 3 - >>> m() - Traceback (most recent call last): - ... - StopIteration - -If any members of the iterable are exceptions they will be raised instead of -returned:: - - >>> iterable = (33, ValueError, 66) - >>> m = MagicMock(side_effect=iterable) - >>> m() - 33 - >>> m() - Traceback (most recent call last): - ... - ValueError - >>> m() - 66 - - -.. _deleting-attributes: - -Deleting Attributes -~~~~~~~~~~~~~~~~~~~ - -Mock objects create attributes on demand. This allows them to pretend to be -objects of any type. - -You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an -:exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. - -You "block" attributes by deleting them. Once deleted, accessing an attribute -will raise an :exc:`AttributeError`. - - >>> mock = MagicMock() - >>> hasattr(mock, 'm') - True - >>> del mock.m - >>> hasattr(mock, 'm') - False - >>> del mock.f - >>> mock.f - Traceback (most recent call last): - ... - AttributeError: f - - -Mock names and the name attribute -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since "name" is an argument to the :class:`Mock` constructor, if you want your -mock object to have a "name" attribute you can't just pass it in at creation -time. There are two alternatives. One option is to use -:meth:`~Mock.configure_mock`:: - - >>> mock = MagicMock() - >>> mock.configure_mock(name='my_name') - >>> mock.name - 'my_name' - -A simpler option is to simply set the "name" attribute after mock creation:: - - >>> mock = MagicMock() - >>> mock.name = "foo" - - -Attaching Mocks as Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When you attach a mock as an attribute of another mock (or as the return -value) it becomes a "child" of that mock. Calls to the child are recorded in -the :attr:`~Mock.method_calls` and :attr:`~Mock.mock_calls` attributes of the -parent. This is useful for configuring child mocks and then attaching them to -the parent, or for attaching mocks to a parent that records all calls to the -children and allows you to make assertions about the order of calls between -mocks: - - >>> parent = MagicMock() - >>> child1 = MagicMock(return_value=None) - >>> child2 = MagicMock(return_value=None) - >>> parent.child1 = child1 - >>> parent.child2 = child2 - >>> child1(1) - >>> child2(2) - >>> parent.mock_calls - [call.child1(1), call.child2(2)] - -The exception to this is if the mock has a name. This allows you to prevent -the "parenting" if for some reason you don't want it to happen. - - >>> mock = MagicMock() - >>> not_a_child = MagicMock(name='not-a-child') - >>> mock.attribute = not_a_child - >>> mock.attribute() - - >>> mock.mock_calls - [] - -Mocks created for you by :func:`patch` are automatically given names. To -attach mocks that have names to a parent you use the :meth:`~Mock.attach_mock` -method: - - >>> thing1 = object() - >>> thing2 = object() - >>> parent = MagicMock() - >>> with patch('__main__.thing1', return_value=None) as child1: - ... with patch('__main__.thing2', return_value=None) as child2: - ... parent.attach_mock(child1, 'child1') - ... parent.attach_mock(child2, 'child2') - ... child1('one') - ... child2('two') - ... - >>> parent.mock_calls - [call.child1('one'), call.child2('two')] - - -.. [#] The only exceptions are magic methods and attributes (those that have - leading and trailing double underscores). Mock doesn't create these but - instead raises an :exc:`AttributeError`. This is because the interpreter - will often implicitly request these methods, and gets *very* confused to - get a new Mock object when it expects a magic method. If you need magic - method support see :ref:`magic methods `. - - -The patchers ------------- - -The patch decorators are used for patching objects only within the scope of -the function they decorate. They automatically handle the unpatching for you, -even if exceptions are raised. All of these functions can also be used in with -statements or as class decorators. - - -patch -~~~~~ - -.. note:: - - :func:`patch` is straightforward to use. The key is to do the patching in the - right namespace. See the section `where to patch`_. - -.. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - :func:`patch` acts as a function decorator, class decorator or a context - manager. Inside the body of the function or with statement, the *target* - is patched with a *new* object. When the function/with statement exits - the patch is undone. - - If *new* is omitted, then the target is replaced with a - :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is - omitted, the created mock is passed in as an extra argument to the - decorated function. If :func:`patch` is used as a context manager the created - mock is returned by the context manager. - - *target* should be a string in the form ``'package.module.ClassName'``. The - *target* is imported and the specified object replaced with the *new* - object, so the *target* must be importable from the environment you are - calling :func:`patch` from. The target is imported when the decorated function - is executed, not at decoration time. - - The *spec* and *spec_set* keyword arguments are passed to the :class:`MagicMock` - if patch is creating one for you. - - In addition you can pass ``spec=True`` or ``spec_set=True``, which causes - patch to pass in the object being mocked as the spec/spec_set object. - - *new_callable* allows you to specify a different class, or callable object, - that will be called to create the *new* object. By default :class:`MagicMock` is - used. - - A more powerful form of *spec* is *autospec*. If you set ``autospec=True`` - then the mock will be created with a spec from the object being replaced. - All attributes of the mock will also have the spec of the corresponding - attribute of the object being replaced. Methods and functions being mocked - will have their arguments checked and will raise a :exc:`TypeError` if they are - called with the wrong signature. For mocks - replacing a class, their return value (the 'instance') will have the same - spec as the class. See the :func:`create_autospec` function and - :ref:`auto-speccing`. - - Instead of ``autospec=True`` you can pass ``autospec=some_object`` to use an - arbitrary object as the spec instead of the one being replaced. - - By default :func:`patch` will fail to replace attributes that don't exist. If - you pass in ``create=True``, and the attribute doesn't exist, patch will - create the attribute for you when the patched function is called, and - delete it again afterwards. This is useful for writing tests against - attributes that your production code creates at runtime. It is off by - default because it can be dangerous. With it switched on you can write - passing tests against APIs that don't actually exist! - - .. note:: - - .. versionchanged:: 3.5 - If you are patching builtins in a module then you don't - need to pass ``create=True``, it will be added by default. - - Patch can be used as a :class:`TestCase` class decorator. It works by - decorating each test method in the class. This reduces the boilerplate - code when your test methods share a common patchings set. :func:`patch` finds - tests by looking for method names that start with ``patch.TEST_PREFIX``. - By default this is ``'test'``, which matches the way :mod:`unittest` finds tests. - You can specify an alternative prefix by setting ``patch.TEST_PREFIX``. - - Patch can be used as a context manager, with the with statement. Here the - patching applies to the indented block after the with statement. If you - use "as" then the patched object will be bound to the name after the - "as"; very useful if :func:`patch` is creating a mock object for you. - - :func:`patch` takes arbitrary keyword arguments. These will be passed to - the :class:`Mock` (or *new_callable*) on construction. - - ``patch.dict(...)``, ``patch.multiple(...)`` and ``patch.object(...)`` are - available for alternate use-cases. - -:func:`patch` as function decorator, creating the mock for you and passing it into -the decorated function: - - >>> @patch('__main__.SomeClass') - ... def function(normal_argument, mock_class): - ... print(mock_class is SomeClass) - ... - >>> function(None) - True - -Patching a class replaces the class with a :class:`MagicMock` *instance*. If the -class is instantiated in the code under test then it will be the -:attr:`~Mock.return_value` of the mock that will be used. - -If the class is instantiated multiple times you could use -:attr:`~Mock.side_effect` to return a new mock each time. Alternatively you -can set the *return_value* to be anything you want. - -To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example: - - >>> class Class: - ... def method(self): - ... pass - ... - >>> with patch('__main__.Class') as MockClass: - ... instance = MockClass.return_value - ... instance.method.return_value = 'foo' - ... assert Class() is instance - ... assert Class().method() == 'foo' - ... - -If you use *spec* or *spec_set* and :func:`patch` is replacing a *class*, then the -return value of the created mock will have the same spec. - - >>> Original = Class - >>> patcher = patch('__main__.Class', spec=True) - >>> MockClass = patcher.start() - >>> instance = MockClass() - >>> assert isinstance(instance, Original) - >>> patcher.stop() - -The *new_callable* argument is useful where you want to use an alternative -class to the default :class:`MagicMock` for the created mock. For example, if -you wanted a :class:`NonCallableMock` to be used: - - >>> thing = object() - >>> with patch('__main__.thing', new_callable=NonCallableMock) as mock_thing: - ... assert thing is mock_thing - ... thing() - ... - Traceback (most recent call last): - ... - TypeError: 'NonCallableMock' object is not callable - -Another use case might be to replace an object with an :class:`io.StringIO` instance: - - >>> from io import StringIO - >>> def foo(): - ... print('Something') - ... - >>> @patch('sys.stdout', new_callable=StringIO) - ... def test(mock_stdout): - ... foo() - ... assert mock_stdout.getvalue() == 'Something\n' - ... - >>> test() - -When :func:`patch` is creating a mock for you, it is common that the first thing -you need to do is to configure the mock. Some of that configuration can be done -in the call to patch. Any arbitrary keywords you pass into the call will be -used to set attributes on the created mock: - - >>> patcher = patch('__main__.thing', first='one', second='two') - >>> mock_thing = patcher.start() - >>> mock_thing.first - 'one' - >>> mock_thing.second - 'two' - -As well as attributes on the created mock attributes, like the -:attr:`~Mock.return_value` and :attr:`~Mock.side_effect`, of child mocks can -also be configured. These aren't syntactically valid to pass in directly as -keyword arguments, but a dictionary with these as keys can still be expanded -into a :func:`patch` call using ``**``: - - >>> config = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> patcher = patch('__main__.thing', **config) - >>> mock_thing = patcher.start() - >>> mock_thing.method() - 3 - >>> mock_thing.other() - Traceback (most recent call last): - ... - KeyError - - -patch.object -~~~~~~~~~~~~ - -.. function:: patch.object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - patch the named member (*attribute*) on an object (*target*) with a mock - object. - - :func:`patch.object` can be used as a decorator, class decorator or a context - manager. Arguments *new*, *spec*, *create*, *spec_set*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. Like :func:`patch`, - :func:`patch.object` takes arbitrary keyword arguments for configuring the mock - object it creates. - - When used as a class decorator :func:`patch.object` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -You can either call :func:`patch.object` with three arguments or two arguments. The -three argument form takes the object to be patched, the attribute name and the -object to replace the attribute with. - -When calling with the two argument form you omit the replacement object, and a -mock is created for you and passed in as an extra argument to the decorated -function: - - >>> @patch.object(SomeClass, 'class_method') - ... def test(mock_method): - ... SomeClass.class_method(3) - ... mock_method.assert_called_with(3) - ... - >>> test() - -*spec*, *create* and the other arguments to :func:`patch.object` have the same -meaning as they do for :func:`patch`. - - -patch.dict -~~~~~~~~~~ - -.. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) - - Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. - - *in_dict* can be a dictionary or a mapping like container. If it is a - mapping then it must at least support getting, setting and deleting items - plus iterating over keys. - - *in_dict* can also be a string specifying the name of the dictionary, which - will then be fetched by importing it. - - *values* can be a dictionary of values to set in the dictionary. *values* - can also be an iterable of ``(key, value)`` pairs. - - If *clear* is true then the dictionary will be cleared before the new - values are set. - - :func:`patch.dict` can also be called with arbitrary keyword arguments to set - values in the dictionary. - - :func:`patch.dict` can be used as a context manager, decorator or class - decorator. When used as a class decorator :func:`patch.dict` honours - ``patch.TEST_PREFIX`` for choosing which methods to wrap. - -:func:`patch.dict` can be used to add members to a dictionary, or simply let a test -change a dictionary, and ensure the dictionary is restored when the test -ends. - - >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == {} - - >>> import os - >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): - ... print(os.environ['newkey']) - ... - newvalue - >>> assert 'newkey' not in os.environ - -Keywords can be used in the :func:`patch.dict` call to set values in the dictionary: - - >>> mymodule = MagicMock() - >>> mymodule.function.return_value = 'fish' - >>> with patch.dict('sys.modules', mymodule=mymodule): - ... import mymodule - ... mymodule.function('some', 'args') - ... - 'fish' - -:func:`patch.dict` can be used with dictionary like objects that aren't actually -dictionaries. At the very minimum they must support item getting, setting, -deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. - - >>> class Container: - ... def __init__(self): - ... self.values = {} - ... def __getitem__(self, name): - ... return self.values[name] - ... def __setitem__(self, name, value): - ... self.values[name] = value - ... def __delitem__(self, name): - ... del self.values[name] - ... def __iter__(self): - ... return iter(self.values) - ... - >>> thing = Container() - >>> thing['one'] = 1 - >>> with patch.dict(thing, one=2, two=3): - ... assert thing['one'] == 2 - ... assert thing['two'] == 3 - ... - >>> assert thing['one'] == 1 - >>> assert list(thing) == ['one'] - - -patch.multiple -~~~~~~~~~~~~~~ - -.. function:: patch.multiple(target, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - Perform multiple patches in a single call. It takes the object to be - patched (either as an object or a string to fetch the object by importing) - and keyword arguments for the patches:: - - with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'): - ... - - Use :data:`DEFAULT` as the value if you want :func:`patch.multiple` to create - mocks for you. In this case the created mocks are passed into a decorated - function by keyword, and a dictionary is returned when :func:`patch.multiple` is - used as a context manager. - - :func:`patch.multiple` can be used as a decorator, class decorator or a context - manager. The arguments *spec*, *spec_set*, *create*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. These arguments will - be applied to *all* patches done by :func:`patch.multiple`. - - When used as a class decorator :func:`patch.multiple` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -If you want :func:`patch.multiple` to create mocks for you, then you can use -:data:`DEFAULT` as the value. If you use :func:`patch.multiple` as a decorator -then the created mocks are passed into the decorated function by keyword. - - >>> thing = object() - >>> other = object() - - >>> @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(thing, other): - ... assert isinstance(thing, MagicMock) - ... assert isinstance(other, MagicMock) - ... - >>> test_function() - -:func:`patch.multiple` can be nested with other ``patch`` decorators, but put arguments -passed by keyword *after* any of the standard arguments created by :func:`patch`: - - >>> @patch('sys.exit') - ... @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(mock_exit, other, thing): - ... assert 'other' in repr(other) - ... assert 'thing' in repr(thing) - ... assert 'exit' in repr(mock_exit) - ... - >>> test_function() - -If :func:`patch.multiple` is used as a context manager, the value returned by the -context manger is a dictionary where created mocks are keyed by name: - - >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values: - ... assert 'other' in repr(values['other']) - ... assert 'thing' in repr(values['thing']) - ... assert values['thing'] is thing - ... assert values['other'] is other - ... - - -.. _start-and-stop: - -patch methods: start and stop -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do -patching in ``setUp`` methods or where you want to do multiple patches without -nesting decorators or with statements. - -To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as -normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. - -If you are using :func:`patch` to create a mock for you then it will be returned by -the call to ``patcher.start``. - - >>> patcher = patch('package.module.ClassName') - >>> from package import module - >>> original = module.ClassName - >>> new_mock = patcher.start() - >>> assert module.ClassName is not original - >>> assert module.ClassName is new_mock - >>> patcher.stop() - >>> assert module.ClassName is original - >>> assert module.ClassName is not new_mock - - -A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`: - - >>> class MyTest(TestCase): - ... def setUp(self): - ... self.patcher1 = patch('package.module.Class1') - ... self.patcher2 = patch('package.module.Class2') - ... self.MockClass1 = self.patcher1.start() - ... self.MockClass2 = self.patcher2.start() - ... - ... def tearDown(self): - ... self.patcher1.stop() - ... self.patcher2.stop() - ... - ... def test_something(self): - ... assert package.module.Class1 is self.MockClass1 - ... assert package.module.Class2 is self.MockClass2 - ... - >>> MyTest('test_something').run() - -.. caution:: - - If you use this technique you must ensure that the patching is "undone" by - calling ``stop``. This can be fiddlier than you might think, because if an - exception is raised in the ``setUp`` then ``tearDown`` is not called. - :meth:`unittest.TestCase.addCleanup` makes this easier: - - >>> class MyTest(TestCase): - ... def setUp(self): - ... patcher = patch('package.module.Class') - ... self.MockClass = patcher.start() - ... self.addCleanup(patcher.stop) - ... - ... def test_something(self): - ... assert package.module.Class is self.MockClass - ... - - As an added bonus you no longer need to keep a reference to the ``patcher`` - object. - -It is also possible to stop all patches which have been started by using -:func:`patch.stopall`. - -.. function:: patch.stopall - - Stop all active patches. Only stops patches started with ``start``. - - -.. _patch-builtins: - -patch builtins -~~~~~~~~~~~~~~ -You can patch any builtins within a module. The following example patches -builtin :func:`ord`: - - >>> @patch('__main__.ord') - ... def test(mock_ord): - ... mock_ord.return_value = 101 - ... print(ord('c')) - ... - >>> test() - 101 - - -TEST_PREFIX -~~~~~~~~~~~ - -All of the patchers can be used as class decorators. When used in this way -they wrap every test method on the class. The patchers recognise methods that -start with ``'test'`` as being test methods. This is the same way that the -:class:`unittest.TestLoader` finds test methods by default. - -It is possible that you want to use a different prefix for your tests. You can -inform the patchers of the different prefix by setting ``patch.TEST_PREFIX``: - - >>> patch.TEST_PREFIX = 'foo' - >>> value = 3 - >>> - >>> @patch('__main__.value', 'not three') - ... class Thing: - ... def foo_one(self): - ... print(value) - ... def foo_two(self): - ... print(value) - ... - >>> - >>> Thing().foo_one() - not three - >>> Thing().foo_two() - not three - >>> value - 3 - - -Nesting Patch Decorators -~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to perform multiple patches then you can simply stack up the -decorators. - -You can stack up multiple patch decorators using this pattern: - - >>> @patch.object(SomeClass, 'class_method') - ... @patch.object(SomeClass, 'static_method') - ... def test(mock1, mock2): - ... assert SomeClass.static_method is mock1 - ... assert SomeClass.class_method is mock2 - ... SomeClass.static_method('foo') - ... SomeClass.class_method('bar') - ... return mock1, mock2 - ... - >>> mock1, mock2 = test() - >>> mock1.assert_called_once_with('foo') - >>> mock2.assert_called_once_with('bar') - - -Note that the decorators are applied from the bottom upwards. This is the -standard way that Python applies decorators. The order of the created mocks -passed into your test function matches this order. - - -.. _where-to-patch: - -Where to patch -~~~~~~~~~~~~~~ - -:func:`patch` works by (temporarily) changing the object that a *name* points to with -another one. There can be many names pointing to any individual object, so -for patching to work you must ensure that you patch the name used by the system -under test. - -The basic principle is that you patch where an object is *looked up*, which -is not necessarily the same place as where it is defined. A couple of -examples will help to clarify this. - -Imagine we have a project that we want to test with the following structure:: - - a.py - -> Defines SomeClass - - b.py - -> from a import SomeClass - -> some_function instantiates SomeClass - -Now we want to test ``some_function`` but we want to mock out ``SomeClass`` using -:func:`patch`. The problem is that when we import module b, which we will have to -do then it imports ``SomeClass`` from module a. If we use :func:`patch` to mock out -``a.SomeClass`` then it will have no effect on our test; module b already has a -reference to the *real* ``SomeClass`` and it looks like our patching had no -effect. - -The key is to patch out ``SomeClass`` where it is used (or where it is looked up -). In this case ``some_function`` will actually look up ``SomeClass`` in module b, -where we have imported it. The patching should look like:: - - @patch('b.SomeClass') - -However, consider the alternative scenario where instead of ``from a import -SomeClass`` module b does ``import a`` and ``some_function`` uses ``a.SomeClass``. Both -of these import forms are common. In this case the class we want to patch is -being looked up in the module and so we have to patch ``a.SomeClass`` instead:: - - @patch('a.SomeClass') - - -Patching Descriptors and Proxy Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Both patch_ and patch.object_ correctly patch and restore descriptors: class -methods, static methods and properties. You should patch these on the *class* -rather than an instance. They also work with *some* objects -that proxy attribute access, like the `django settings object -`_. - - -MagicMock and magic method support ----------------------------------- - -.. _magic-methods: - -Mocking Magic Methods -~~~~~~~~~~~~~~~~~~~~~ - -:class:`Mock` supports mocking the Python protocol methods, also known as -"magic methods". This allows mock objects to replace containers or other -objects that implement Python protocols. - -Because magic methods are looked up differently from normal methods [#]_, this -support has been specially implemented. This means that only specific magic -methods are supported. The supported list includes *almost* all of them. If -there are any missing that you need please let us know. - -You mock magic methods by setting the method you are interested in to a function -or a mock instance. If you are using a function then it *must* take ``self`` as -the first argument [#]_. - - >>> def __str__(self): - ... return 'fooble' - ... - >>> mock = Mock() - >>> mock.__str__ = __str__ - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__str__ = Mock() - >>> mock.__str__.return_value = 'fooble' - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__iter__ = Mock(return_value=iter([])) - >>> list(mock) - [] - -One use case for this is for mocking objects used as context managers in a -:keyword:`with` statement: - - >>> mock = Mock() - >>> mock.__enter__ = Mock(return_value='foo') - >>> mock.__exit__ = Mock(return_value=False) - >>> with mock as m: - ... assert m == 'foo' - ... - >>> mock.__enter__.assert_called_with() - >>> mock.__exit__.assert_called_with(None, None, None) - -Calls to magic methods do not appear in :attr:`~Mock.method_calls`, but they -are recorded in :attr:`~Mock.mock_calls`. - -.. note:: - - If you use the *spec* keyword argument to create a mock then attempting to - set a magic method that isn't in the spec will raise an :exc:`AttributeError`. - -The full list of supported magic methods is: - -* ``__hash__``, ``__sizeof__``, ``__repr__`` and ``__str__`` -* ``__dir__``, ``__format__`` and ``__subclasses__`` -* ``__floor__``, ``__trunc__`` and ``__ceil__`` -* Comparisons: ``__lt__``, ``__gt__``, ``__le__``, ``__ge__``, - ``__eq__`` and ``__ne__`` -* Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, - ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` - and ``__missing__`` -* Context manager: ``__enter__`` and ``__exit__`` -* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` -* The numeric methods (including right hand and in-place variants): - ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, - ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``, - ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__`` -* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__`` - and ``__index__`` -* Descriptor methods: ``__get__``, ``__set__`` and ``__delete__`` -* Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, - ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` - - -The following methods exist but are *not* supported as they are either in use -by mock, can't be set dynamically, or can cause problems: - -* ``__getattr__``, ``__setattr__``, ``__init__`` and ``__new__`` -* ``__prepare__``, ``__instancecheck__``, ``__subclasscheck__``, ``__del__`` - - - -Magic Mock -~~~~~~~~~~ - -There are two ``MagicMock`` variants: :class:`MagicMock` and :class:`NonCallableMagicMock`. - - -.. class:: MagicMock(*args, **kw) - - ``MagicMock`` is a subclass of :class:`Mock` with default implementations - of most of the magic methods. You can use ``MagicMock`` without having to - configure the magic methods yourself. - - The constructor parameters have the same meaning as for :class:`Mock`. - - If you use the *spec* or *spec_set* arguments then *only* magic methods - that exist in the spec will be created. - - -.. class:: NonCallableMagicMock(*args, **kw) - - A non-callable version of :class:`MagicMock`. - - The constructor parameters have the same meaning as for - :class:`MagicMock`, with the exception of *return_value* and - *side_effect* which have no meaning on a non-callable mock. - -The magic methods are setup with :class:`MagicMock` objects, so you can configure them -and use them in the usual way: - - >>> mock = MagicMock() - >>> mock[3] = 'fish' - >>> mock.__setitem__.assert_called_with(3, 'fish') - >>> mock.__getitem__.return_value = 'result' - >>> mock[2] - 'result' - -By default many of the protocol methods are required to return objects of a -specific type. These methods are preconfigured with a default return value, so -that they can be used without you having to do anything if you aren't interested -in the return value. You can still *set* the return value manually if you want -to change the default. - -Methods and their defaults: - -* ``__lt__``: NotImplemented -* ``__gt__``: NotImplemented -* ``__le__``: NotImplemented -* ``__ge__``: NotImplemented -* ``__int__``: 1 -* ``__contains__``: False -* ``__len__``: 0 -* ``__iter__``: iter([]) -* ``__exit__``: False -* ``__complex__``: 1j -* ``__float__``: 1.0 -* ``__bool__``: True -* ``__index__``: 1 -* ``__hash__``: default hash for the mock -* ``__str__``: default str for the mock -* ``__sizeof__``: default sizeof for the mock - -For example: - - >>> mock = MagicMock() - >>> int(mock) - 1 - >>> len(mock) - 0 - >>> list(mock) - [] - >>> object() in mock - False - -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. -They do the default equality comparison on identity, using the -:attr:`~Mock.side_effect` attribute, unless you change their return value to -return something else:: - - >>> MagicMock() == 3 - False - >>> MagicMock() != 3 - True - >>> mock = MagicMock() - >>> mock.__eq__.return_value = True - >>> mock == 3 - True - -The return value of :meth:`MagicMock.__iter__` can be any iterable object and isn't -required to be an iterator: - - >>> mock = MagicMock() - >>> mock.__iter__.return_value = ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - -If the return value *is* an iterator, then iterating over it once will consume -it and subsequent iterations will result in an empty list: - - >>> mock.__iter__.return_value = iter(['a', 'b', 'c']) - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - [] - -``MagicMock`` has all of the supported magic methods configured except for some -of the obscure and obsolete ones. You can still set these up if you want. - -Magic methods that are supported but not setup by default in ``MagicMock`` are: - -* ``__subclasses__`` -* ``__dir__`` -* ``__format__`` -* ``__get__``, ``__set__`` and ``__delete__`` -* ``__reversed__`` and ``__missing__`` -* ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, - ``__getstate__`` and ``__setstate__`` -* ``__getformat__`` and ``__setformat__`` - - - -.. [#] Magic methods *should* be looked up on the class rather than the - instance. Different versions of Python are inconsistent about applying this - rule. The supported protocol methods should work with all supported versions - of Python. -.. [#] The function is basically hooked up to the class, but each ``Mock`` - instance is kept isolated from the others. - - -Helpers -------- - -sentinel -~~~~~~~~ - -.. data:: sentinel - - The ``sentinel`` object provides a convenient way of providing unique - objects for your tests. - - Attributes are created on demand when you access them by name. Accessing - the same attribute will always return the same object. The objects - returned have a sensible repr so that test failure messages are readable. - - The ``sentinel`` attributes don't preserve their identity when they are - :mod:`copied ` or :mod:`pickled `. - -Sometimes when testing you need to test that a specific object is passed as an -argument to another method, or returned. It can be common to create named -sentinel objects to test this. :data:`sentinel` provides a convenient way of -creating and testing the identity of objects like this. - -In this example we monkey patch ``method`` to return ``sentinel.some_object``: - - >>> real = ProductionClass() - >>> real.method = Mock(name="method") - >>> real.method.return_value = sentinel.some_object - >>> result = real.method() - >>> assert result is sentinel.some_object - >>> sentinel.some_object - sentinel.some_object - - -DEFAULT -~~~~~~~ - - -.. data:: DEFAULT - - The :data:`DEFAULT` object is a pre-created sentinel (actually - ``sentinel.DEFAULT``). It can be used by :attr:`~Mock.side_effect` - functions to indicate that the normal return value should be used. - - -call -~~~~ - -.. function:: call(*args, **kwargs) - - :func:`call` is a helper object for making simpler assertions, for comparing with - :attr:`~Mock.call_args`, :attr:`~Mock.call_args_list`, - :attr:`~Mock.mock_calls` and :attr:`~Mock.method_calls`. :func:`call` can also be - used with :meth:`~Mock.assert_has_calls`. - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, a='foo', b='bar') - >>> m() - >>> m.call_args_list == [call(1, 2, a='foo', b='bar'), call()] - True - -.. method:: call.call_list() - - For a call object that represents multiple calls, :meth:`call_list` - returns a list of all the intermediate calls as well as the - final call. - -``call_list`` is particularly useful for making assertions on "chained calls". A -chained call is multiple calls on a single line of code. This results in -multiple entries in :attr:`~Mock.mock_calls` on a mock. Manually constructing -the sequence of calls can be tedious. - -:meth:`~call.call_list` can construct the sequence of calls from the same -chained call: - - >>> m = MagicMock() - >>> m(1).method(arg='foo').other('bar')(2.0) - - >>> kall = call(1).method(arg='foo').other('bar')(2.0) - >>> kall.call_list() - [call(1), - call().method(arg='foo'), - call().method().other('bar'), - call().method().other()(2.0)] - >>> m.mock_calls == kall.call_list() - True - -.. _calls-as-tuples: - -A ``call`` object is either a tuple of (positional args, keyword args) or -(name, positional args, keyword args) depending on how it was constructed. When -you construct them yourself this isn't particularly interesting, but the ``call`` -objects that are in the :attr:`Mock.call_args`, :attr:`Mock.call_args_list` and -:attr:`Mock.mock_calls` attributes can be introspected to get at the individual -arguments they contain. - -The ``call`` objects in :attr:`Mock.call_args` and :attr:`Mock.call_args_list` -are two-tuples of (positional args, keyword args) whereas the ``call`` objects -in :attr:`Mock.mock_calls`, along with ones you construct yourself, are -three-tuples of (name, positional args, keyword args). - -You can use their "tupleness" to pull out the individual arguments for more -complex introspection and assertions. The positional arguments are a tuple -(an empty tuple if there are no positional arguments) and the keyword -arguments are a dictionary: - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, 3, arg='one', arg2='two') - >>> kall = m.call_args - >>> args, kwargs = kall - >>> args - (1, 2, 3) - >>> kwargs - {'arg2': 'two', 'arg': 'one'} - >>> args is kall[0] - True - >>> kwargs is kall[1] - True - - >>> m = MagicMock() - >>> m.foo(4, 5, 6, arg='two', arg2='three') - - >>> kall = m.mock_calls[0] - >>> name, args, kwargs = kall - >>> name - 'foo' - >>> args - (4, 5, 6) - >>> kwargs - {'arg2': 'three', 'arg': 'two'} - >>> name is m.mock_calls[0][0] - True - - -create_autospec -~~~~~~~~~~~~~~~ - -.. function:: create_autospec(spec, spec_set=False, instance=False, **kwargs) - - Create a mock object using another object as a spec. Attributes on the - mock will use the corresponding attribute on the *spec* object as their - spec. - - Functions or methods being mocked will have their arguments checked to - ensure that they are called with the correct signature. - - If *spec_set* is ``True`` then attempting to set attributes that don't exist - on the spec object will raise an :exc:`AttributeError`. - - If a class is used as a spec then the return value of the mock (the - instance of the class) will have the same spec. You can use a class as the - spec for an instance object by passing ``instance=True``. The returned mock - will only be callable if instances of the mock are callable. - - :func:`create_autospec` also takes arbitrary keyword arguments that are passed to - the constructor of the created mock. - -See :ref:`auto-speccing` for examples of how to use auto-speccing with -:func:`create_autospec` and the *autospec* argument to :func:`patch`. - - -ANY -~~~ - -.. data:: ANY - -Sometimes you may need to make assertions about *some* of the arguments in a -call to mock, but either not care about some of the arguments or want to pull -them individually out of :attr:`~Mock.call_args` and make more complex -assertions on them. - -To ignore certain arguments you can pass in objects that compare equal to -*everything*. Calls to :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with` will then succeed no matter what was -passed in. - - >>> mock = Mock(return_value=None) - >>> mock('foo', bar=object()) - >>> mock.assert_called_once_with('foo', bar=ANY) - -:data:`ANY` can also be used in comparisons with call lists like -:attr:`~Mock.mock_calls`: - - >>> m = MagicMock(return_value=None) - >>> m(1) - >>> m(1, 2) - >>> m(object()) - >>> m.mock_calls == [call(1), call(1, 2), ANY] - True - - - -FILTER_DIR -~~~~~~~~~~ - -.. data:: FILTER_DIR - -:data:`FILTER_DIR` is a module level variable that controls the way mock objects -respond to :func:`dir` (only for Python 2.6 or more recent). The default is ``True``, -which uses the filtering described below, to only show useful members. If you -dislike this filtering, or need to switch it off for diagnostic purposes, then -set ``mock.FILTER_DIR = False``. - -With filtering on, ``dir(some_mock)`` shows only useful attributes and will -include any dynamically created attributes that wouldn't normally be shown. -If the mock was created with a *spec* (or *autospec* of course) then all the -attributes from the original are shown, even if they haven't been accessed -yet: - - >>> dir(Mock()) - ['assert_any_call', - 'assert_called_once_with', - 'assert_called_with', - 'assert_has_calls', - 'attach_mock', - ... - >>> from urllib import request - >>> dir(Mock(spec=request)) - ['AbstractBasicAuthHandler', - 'AbstractDigestAuthHandler', - 'AbstractHTTPHandler', - 'BaseHandler', - ... - -Many of the not-very-useful (private to :class:`Mock` rather than the thing being -mocked) underscore and double underscore prefixed attributes have been -filtered from the result of calling :func:`dir` on a :class:`Mock`. If you dislike this -behaviour you can switch it off by setting the module level switch -:data:`FILTER_DIR`: - - >>> from unittest import mock - >>> mock.FILTER_DIR = False - >>> dir(mock.Mock()) - ['_NonCallableMock__get_return_value', - '_NonCallableMock__get_side_effect', - '_NonCallableMock__return_value_doc', - '_NonCallableMock__set_return_value', - '_NonCallableMock__set_side_effect', - '__call__', - '__class__', - ... - -Alternatively you can just use ``vars(my_mock)`` (instance members) and -``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:data:`mock.FILTER_DIR`. - - -mock_open -~~~~~~~~~ - -.. function:: mock_open(mock=None, read_data=None) - - A helper function to create a mock to replace the use of :func:`open`. It works - for :func:`open` called directly or used as a context manager. - - The *mock* argument is the mock object to configure. If ``None`` (the - default) then a :class:`MagicMock` will be created for you, with the API limited - to methods or attributes available on standard file handles. - - *read_data* is a string for the :meth:`~io.IOBase.read`, - :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods - of the file handle to return. Calls to those methods will take data from - *read_data* until it is depleted. The mock of these methods is pretty - simplistic: every time the *mock* is called, the *read_data* is rewound to - the start. If you need more control over the data that you are feeding to - the tested code you will need to customize this mock for yourself. When that - is insufficient, one of the in-memory filesystem packages on `PyPI - `_ can offer a realistic filesystem for testing. - - .. versionchanged:: 3.4 - Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather - than returning it on each call. - - .. versionchanged:: 3.5 - *read_data* is now reset on each call to the *mock*. - -Using :func:`open` as a context manager is a great way to ensure your file handles -are closed properly and is becoming common:: - - with open('/some/path', 'w') as f: - f.write('something') - -The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`__enter__` and -:meth:`__exit__` called). - -Mocking context managers with a :class:`MagicMock` is common enough and fiddly -enough that a helper function is useful. - - >>> m = mock_open() - >>> with patch('__main__.open', m): - ... with open('foo', 'w') as h: - ... h.write('some stuff') - ... - >>> m.mock_calls - [call('foo', 'w'), - call().__enter__(), - call().write('some stuff'), - call().__exit__(None, None, None)] - >>> m.assert_called_once_with('foo', 'w') - >>> handle = m() - >>> handle.write.assert_called_once_with('some stuff') - -And for reading files: - - >>> with patch('__main__.open', mock_open(read_data='bibble')) as m: - ... with open('foo') as h: - ... result = h.read() - ... - >>> m.assert_called_once_with('foo') - >>> assert result == 'bibble' - - -.. _auto-speccing: - -Autospeccing -~~~~~~~~~~~~ - -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the -api of mocks to the api of an original object (the spec), but it is recursive -(implemented lazily) so that attributes of mocks only have the same api as -the attributes of the spec. In addition mocked functions / methods have the -same call signature as the original so they raise a :exc:`TypeError` if they are -called incorrectly. - -Before I explain how auto-speccing works, here's why it is needed. - -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) - -Your tests can pass silently and incorrectly because of the typo. - -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. - -Note that this is another reason why you need integration tests as well as -unit tests. Testing everything in isolation is all fine and dandy, but if you -don't test how your units are "wired together" there is still lots of room -for bugs that tests might have caught. - -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access -attributes on the mock that exist on the real class: - - >>> from urllib import request - >>> mock = Mock(spec=request.Request) - >>> mock.assret_called_with - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - -The spec only applies to the mock itself, so we still have the same issue -with any methods on the mock: - -.. code-block:: pycon - - >>> mock.has_data() - - >>> mock.has_data.assret_called_with() - -Auto-speccing solves this problem. You can either pass ``autospec=True`` to -:func:`patch` / :func:`patch.object` or use the :func:`create_autospec` function to create a -mock with a spec. If you use the ``autospec=True`` argument to :func:`patch` then the -object that is being replaced will be used as the spec object. Because the -speccing is done "lazily" (the spec is created as attributes on the mock are -accessed) you can use it with very complex or deeply nested objects (like -modules that import modules that import modules) without a big performance -hit. - -Here's an example of it in use: - - >>> from urllib import request - >>> patcher = patch('__main__.request', autospec=True) - >>> mock_request = patcher.start() - >>> request is mock_request - True - >>> mock_request.Request - - -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two -arguments in the constructor (one of which is *self*). Here's what happens if -we try to call it incorrectly: - - >>> req = request.Request() - Traceback (most recent call last): - ... - TypeError: () takes at least 2 arguments (1 given) - -The spec also applies to instantiated classes (i.e. the return value of -specced mocks): - - >>> req = request.Request('foo') - >>> req - - -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place -any typos in our asserts will raise the correct error: - - >>> req.add_header('spam', 'eggs') - - >>> req.add_header.assret_called_with - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - >>> req.add_header.assert_called_with('spam', 'eggs') - -In many cases you will just be able to add ``autospec=True`` to your existing -:func:`patch` calls and then be protected against bugs due to typos and api -changes. - -As well as using *autospec* through :func:`patch` there is a -:func:`create_autospec` for creating autospecced mocks directly: - - >>> from urllib import request - >>> mock_request = create_autospec(request) - >>> mock_request.Request('foo', 'bar') - - -This isn't without caveats and limitations however, which is why it is not -the default behaviour. In order to know what attributes are available on the -spec object, autospec has to introspect (access attributes) the spec. As you -traverse attributes on the mock a corresponding traversal of the original -object is happening under the hood. If any of your specced objects have -properties or descriptors that can trigger code execution then you may not be -able to use autospec. On the other hand it is much better to design your -objects so that introspection is safe [#]_. - -A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. -*autospec* can't know about any dynamically created attributes and restricts -the api to visible attributes. - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -There are a few different ways of resolving this problem. The easiest, but -not necessarily the least annoying, way is to simply set the required -attributes on the mock after creation. Just because *autospec* doesn't allow -you to fetch attributes that don't exist on the spec it doesn't prevent you -setting them: - - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a = 33 - ... - -There is a more aggressive version of both *spec* and *autospec* that *does* -prevent you setting non-existent attributes. This is useful if you want to -ensure your code only *sets* valid attributes too, but obviously it prevents -this particular scenario: - - >>> with patch('__main__.Something', autospec=True, spec_set=True): - ... thing = Something() - ... thing.a = 33 - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via -class attributes (shared between instances of course) is faster too. e.g. - -.. code-block:: python - - class Something: - a = 33 - -This brings up another issue. It is relatively common to provide a default -value of ``None`` for members that will later be an object of a different type. -``None`` would be useless as a spec because it wouldn't let you access *any* -attributes or methods on it. As ``None`` is *never* going to be useful as a -spec, and probably indicates a member that will normally of some other type, -autospec doesn't use a spec for members that are set to ``None``. These will -just be ordinary mocks (well - MagicMocks): - - >>> class Something: - ... member = None - ... - >>> mock = create_autospec(Something) - >>> mock.member.foo.bar.baz() - - -If modifying your production classes to add defaults isn't to your liking -then there are more options. One of these is simply to use an instance as the -spec rather than the class. The other is to create a subclass of the -production class and add the defaults to the subclass without affecting the -production class. Both of these require you to use an alternative object as -the spec. Thankfully :func:`patch` supports this - you can simply pass the -alternative object as the *autospec* argument: - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> class SomethingForTest(Something): - ... a = 33 - ... - >>> p = patch('__main__.Something', autospec=SomethingForTest) - >>> mock = p.start() - >>> mock.a - - - -.. [#] This only applies to classes or already instantiated objects. Calling - a mocked class to create a mock instance *does not* create a real instance. - It is only attribute lookups - along with calls to :func:`dir` - that are done. - diff --git a/third_party/python/Doc/library/unittest.rst b/third_party/python/Doc/library/unittest.rst deleted file mode 100644 index 3a8af0c52..000000000 --- a/third_party/python/Doc/library/unittest.rst +++ /dev/null @@ -1,2295 +0,0 @@ -:mod:`unittest` --- Unit testing framework -========================================== - -.. module:: unittest - :synopsis: Unit testing framework for Python. - -.. moduleauthor:: Steve Purcell -.. sectionauthor:: Steve Purcell -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/unittest/__init__.py` - --------------- - -(If you are already familiar with the basic concepts of testing, you might want -to skip to :ref:`the list of assert methods `.) - -The :mod:`unittest` unit testing framework was originally inspired by JUnit -and has a similar flavor as major unit testing frameworks in other -languages. It supports test automation, sharing of setup and shutdown code -for tests, aggregation of tests into collections, and independence of the -tests from the reporting framework. - -To achieve this, :mod:`unittest` supports some important concepts in an -object-oriented way: - -test fixture - A :dfn:`test fixture` represents the preparation needed to perform one or more - tests, and any associate cleanup actions. This may involve, for example, - creating temporary or proxy databases, directories, or starting a server - process. - -test case - A :dfn:`test case` is the individual unit of testing. It checks for a specific - response to a particular set of inputs. :mod:`unittest` provides a base class, - :class:`TestCase`, which may be used to create new test cases. - -test suite - A :dfn:`test suite` is a collection of test cases, test suites, or both. It is - used to aggregate tests that should be executed together. - -test runner - A :dfn:`test runner` is a component which orchestrates the execution of tests - and provides the outcome to the user. The runner may use a graphical interface, - a textual interface, or return a special value to indicate the results of - executing the tests. - - -.. seealso:: - - Module :mod:`doctest` - Another test-support module with a very different flavor. - - `Simple Smalltalk Testing: With Patterns `_ - Kent Beck's original paper on testing frameworks using the pattern shared - by :mod:`unittest`. - - `Nose `_ and `pytest `_ - Third-party unittest frameworks with a lighter-weight syntax for writing - tests. For example, ``assert func(10) == 42``. - - `The Python Testing Tools Taxonomy `_ - An extensive list of Python testing tools including functional testing - frameworks and mock object libraries. - - `Testing in Python Mailing List `_ - A special-interest-group for discussion of testing, and testing tools, - in Python. - - The script :file:`Tools/unittestgui/unittestgui.py` in the Python source distribution is - a GUI tool for test discovery and execution. This is intended largely for ease of use - for those new to unit testing. For production environments it is - recommended that tests be driven by a continuous integration system such as - `Buildbot `_, `Jenkins `_ - or `Hudson `_. - - -.. _unittest-minimal-example: - -Basic example -------------- - -The :mod:`unittest` module provides a rich set of tools for constructing and -running tests. This section demonstrates that a small subset of the tools -suffice to meet the needs of most users. - -Here is a short script to test three string methods:: - - import unittest - - class TestStringMethods(unittest.TestCase): - - def test_upper(self): - self.assertEqual('foo'.upper(), 'FOO') - - def test_isupper(self): - self.assertTrue('FOO'.isupper()) - self.assertFalse('Foo'.isupper()) - - def test_split(self): - s = 'hello world' - self.assertEqual(s.split(), ['hello', 'world']) - # check that s.split fails when the separator is not a string - with self.assertRaises(TypeError): - s.split(2) - - if __name__ == '__main__': - unittest.main() - - -A testcase is created by subclassing :class:`unittest.TestCase`. The three -individual tests are defined with methods whose names start with the letters -``test``. This naming convention informs the test runner about which methods -represent tests. - -The crux of each test is a call to :meth:`~TestCase.assertEqual` to check for an -expected result; :meth:`~TestCase.assertTrue` or :meth:`~TestCase.assertFalse` -to verify a condition; or :meth:`~TestCase.assertRaises` to verify that a -specific exception gets raised. These methods are used instead of the -:keyword:`assert` statement so the test runner can accumulate all test results -and produce a report. - -The :meth:`~TestCase.setUp` and :meth:`~TestCase.tearDown` methods allow you -to define instructions that will be executed before and after each test method. -They are covered in more detail in the section :ref:`organizing-tests`. - -The final block shows a simple way to run the tests. :func:`unittest.main` -provides a command-line interface to the test script. When run from the command -line, the above script produces an output that looks like this:: - - ... - ---------------------------------------------------------------------- - Ran 3 tests in 0.000s - - OK - -Passing the ``-v`` option to your test script will instruct :func:`unittest.main` -to enable a higher level of verbosity, and produce the following output:: - - test_isupper (__main__.TestStringMethods) ... ok - test_split (__main__.TestStringMethods) ... ok - test_upper (__main__.TestStringMethods) ... ok - - ---------------------------------------------------------------------- - Ran 3 tests in 0.001s - - OK - -The above examples show the most commonly used :mod:`unittest` features which -are sufficient to meet many everyday testing needs. The remainder of the -documentation explores the full feature set from first principles. - - -.. _unittest-command-line-interface: - -Command-Line Interface ----------------------- - -The unittest module can be used from the command line to run tests from -modules, classes or even individual test methods:: - - python -m unittest test_module1 test_module2 - python -m unittest test_module.TestClass - python -m unittest test_module.TestClass.test_method - -You can pass in a list with any combination of module names, and fully -qualified class or method names. - -Test modules can be specified by file path as well:: - - python -m unittest tests/test_something.py - -This allows you to use the shell filename completion to specify the test module. -The file specified must still be importable as a module. The path is converted -to a module name by removing the '.py' and converting path separators into '.'. -If you want to execute a test file that isn't importable as a module you should -execute the file directly instead. - -You can run tests with more detail (higher verbosity) by passing in the -v flag:: - - python -m unittest -v test_module - -When executed without arguments :ref:`unittest-test-discovery` is started:: - - python -m unittest - -For a list of all the command-line options:: - - python -m unittest -h - -.. versionchanged:: 3.2 - In earlier versions it was only possible to run individual test methods and - not modules or classes. - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -:program:`unittest` supports these command-line options: - -.. program:: unittest - -.. cmdoption:: -b, --buffer - - The standard output and standard error streams are buffered during the test - run. Output during a passing test is discarded. Output is echoed normally - on test fail or error and is added to the failure messages. - -.. cmdoption:: -c, --catch - - :kbd:`Control-C` during the test run waits for the current test to end and then - reports all the results so far. A second :kbd:`Control-C` raises the normal - :exc:`KeyboardInterrupt` exception. - - See `Signal Handling`_ for the functions that provide this functionality. - -.. cmdoption:: -f, --failfast - - Stop the test run on the first error or failure. - -.. cmdoption:: --locals - - Show local variables in tracebacks. - -.. versionadded:: 3.2 - The command-line options ``-b``, ``-c`` and ``-f`` were added. - -.. versionadded:: 3.5 - The command-line option ``--locals``. - -The command line can also be used for test discovery, for running all of the -tests in a project or just a subset. - - -.. _unittest-test-discovery: - -Test Discovery --------------- - -.. versionadded:: 3.2 - -Unittest supports simple test discovery. In order to be compatible with test -discovery, all of the test files must be :ref:`modules ` or -:ref:`packages ` (including :term:`namespace packages -`) importable from the top-level directory of -the project (this means that their filenames must be valid :ref:`identifiers -`). - -Test discovery is implemented in :meth:`TestLoader.discover`, but can also be -used from the command line. The basic command-line usage is:: - - cd project_directory - python -m unittest discover - -.. note:: - - As a shortcut, ``python -m unittest`` is the equivalent of - ``python -m unittest discover``. If you want to pass arguments to test - discovery the ``discover`` sub-command must be used explicitly. - -The ``discover`` sub-command has the following options: - -.. program:: unittest discover - -.. cmdoption:: -v, --verbose - - Verbose output - -.. cmdoption:: -s, --start-directory directory - - Directory to start discovery (``.`` default) - -.. cmdoption:: -p, --pattern pattern - - Pattern to match test files (``test*.py`` default) - -.. cmdoption:: -t, --top-level-directory directory - - Top level directory of project (defaults to start directory) - -The :option:`-s`, :option:`-p`, and :option:`-t` options can be passed in -as positional arguments in that order. The following two command lines -are equivalent:: - - python -m unittest discover -s project_directory -p "*_test.py" - python -m unittest discover project_directory "*_test.py" - -As well as being a path it is possible to pass a package name, for example -``myproject.subpackage.test``, as the start directory. The package name you -supply will then be imported and its location on the filesystem will be used -as the start directory. - -.. caution:: - - Test discovery loads tests by importing them. Once test discovery has found - all the test files from the start directory you specify it turns the paths - into package names to import. For example :file:`foo/bar/baz.py` will be - imported as ``foo.bar.baz``. - - If you have a package installed globally and attempt test discovery on - a different copy of the package then the import *could* happen from the - wrong place. If this happens test discovery will warn you and exit. - - If you supply the start directory as a package name rather than a - path to a directory then discover assumes that whichever location it - imports from is the location you intended, so you will not get the - warning. - -Test modules and packages can customize test loading and discovery by through -the `load_tests protocol`_. - -.. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages `. - - -.. _organizing-tests: - -Organizing test code --------------------- - -The basic building blocks of unit testing are :dfn:`test cases` --- single -scenarios that must be set up and checked for correctness. In :mod:`unittest`, -test cases are represented by :class:`unittest.TestCase` instances. -To make your own test cases you must write subclasses of -:class:`TestCase` or use :class:`FunctionTestCase`. - -The testing code of a :class:`TestCase` instance should be entirely self -contained, such that it can be run either in isolation or in arbitrary -combination with any number of other test cases. - -The simplest :class:`TestCase` subclass will simply implement a test method -(i.e. a method whose name starts with ``test``) in order to perform specific -testing code:: - - import unittest - - class DefaultWidgetSizeTestCase(unittest.TestCase): - def test_default_widget_size(self): - widget = Widget('The widget') - self.assertEqual(widget.size(), (50, 50)) - -Note that in order to test something, we use one of the :meth:`assert\*` -methods provided by the :class:`TestCase` base class. If the test fails, an -exception will be raised with an explanatory message, and :mod:`unittest` -will identify the test case as a :dfn:`failure`. Any other exceptions will be -treated as :dfn:`errors`. - -Tests can be numerous, and their set-up can be repetitive. Luckily, we -can factor out set-up code by implementing a method called -:meth:`~TestCase.setUp`, which the testing framework will automatically -call for every single test we run:: - - import unittest - - class WidgetTestCase(unittest.TestCase): - def setUp(self): - self.widget = Widget('The widget') - - def test_default_widget_size(self): - self.assertEqual(self.widget.size(), (50,50), - 'incorrect default size') - - def test_widget_resize(self): - self.widget.resize(100,150) - self.assertEqual(self.widget.size(), (100,150), - 'wrong size after resize') - -.. note:: - The order in which the various tests will be run is determined - by sorting the test method names with respect to the built-in - ordering for strings. - -If the :meth:`~TestCase.setUp` method raises an exception while the test is -running, the framework will consider the test to have suffered an error, and -the test method will not be executed. - -Similarly, we can provide a :meth:`~TestCase.tearDown` method that tidies up -after the test method has been run:: - - import unittest - - class WidgetTestCase(unittest.TestCase): - def setUp(self): - self.widget = Widget('The widget') - - def tearDown(self): - self.widget.dispose() - -If :meth:`~TestCase.setUp` succeeded, :meth:`~TestCase.tearDown` will be -run whether the test method succeeded or not. - -Such a working environment for the testing code is called a -:dfn:`test fixture`. A new TestCase instance is created as a unique -test fixture used to execute each individual test method. Thus -:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__` -will be called once per test. - -It is recommended that you use TestCase implementations to group tests together -according to the features they test. :mod:`unittest` provides a mechanism for -this: the :dfn:`test suite`, represented by :mod:`unittest`'s -:class:`TestSuite` class. In most cases, calling :func:`unittest.main` will do -the right thing and collect all the module's test cases for you and execute -them. - -However, should you want to customize the building of your test suite, -you can do it yourself:: - - def suite(): - suite = unittest.TestSuite() - suite.addTest(WidgetTestCase('test_default_widget_size')) - suite.addTest(WidgetTestCase('test_widget_resize')) - return suite - - if __name__ == '__main__': - runner = unittest.TextTestRunner() - runner.run(suite()) - -You can place the definitions of test cases and test suites in the same modules -as the code they are to test (such as :file:`widget.py`), but there are several -advantages to placing the test code in a separate module, such as -:file:`test_widget.py`: - -* The test module can be run standalone from the command line. - -* The test code can more easily be separated from shipped code. - -* There is less temptation to change test code to fit the code it tests without - a good reason. - -* Test code should be modified much less frequently than the code it tests. - -* Tested code can be refactored more easily. - -* Tests for modules written in C must be in separate modules anyway, so why not - be consistent? - -* If the testing strategy changes, there is no need to change the source code. - - -.. _legacy-unit-tests: - -Re-using old test code ----------------------- - -Some users will find that they have existing test code that they would like to -run from :mod:`unittest`, without converting every old test function to a -:class:`TestCase` subclass. - -For this reason, :mod:`unittest` provides a :class:`FunctionTestCase` class. -This subclass of :class:`TestCase` can be used to wrap an existing test -function. Set-up and tear-down functions can also be provided. - -Given the following test function:: - - def testSomething(): - something = makeSomething() - assert something.name is not None - # ... - -one can create an equivalent test case instance as follows, with optional -set-up and tear-down methods:: - - testcase = unittest.FunctionTestCase(testSomething, - setUp=makeSomethingDB, - tearDown=deleteSomethingDB) - -.. note:: - - Even though :class:`FunctionTestCase` can be used to quickly convert an - existing test base over to a :mod:`unittest`\ -based system, this approach is - not recommended. Taking the time to set up proper :class:`TestCase` - subclasses will make future test refactorings infinitely easier. - -In some cases, the existing tests may have been written using the :mod:`doctest` -module. If so, :mod:`doctest` provides a :class:`DocTestSuite` class that can -automatically build :class:`unittest.TestSuite` instances from the existing -:mod:`doctest`\ -based tests. - - -.. _unittest-skipping: - -Skipping tests and expected failures ------------------------------------- - -.. versionadded:: 3.1 - -Unittest supports skipping individual test methods and even whole classes of -tests. In addition, it supports marking a test as an "expected failure," a test -that is broken and will fail, but shouldn't be counted as a failure on a -:class:`TestResult`. - -Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants. - -Basic skipping looks like this:: - - class MyTestCase(unittest.TestCase): - - @unittest.skip("demonstrating skipping") - def test_nothing(self): - self.fail("shouldn't happen") - - @unittest.skipIf(mylib.__version__ < (1, 3), - "not supported in this library version") - def test_format(self): - # Tests that work for only a certain version of the library. - pass - - @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") - def test_windows_support(self): - # windows specific testing code - pass - -This is the output of running the example above in verbose mode:: - - test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' - test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' - test_windows_support (__main__.MyTestCase) ... skipped 'requires Windows' - - ---------------------------------------------------------------------- - Ran 3 tests in 0.005s - - OK (skipped=3) - -Classes can be skipped just like methods:: - - @unittest.skip("showing class skipping") - class MySkippedTestCase(unittest.TestCase): - def test_not_run(self): - pass - -:meth:`TestCase.setUp` can also skip the test. This is useful when a resource -that needs to be set up is not available. - -Expected failures use the :func:`expectedFailure` decorator. :: - - class ExpectedFailureTestCase(unittest.TestCase): - @unittest.expectedFailure - def test_fail(self): - self.assertEqual(1, 0, "broken") - -It's easy to roll your own skipping decorators by making a decorator that calls -:func:`skip` on the test when it wants it to be skipped. This decorator skips -the test unless the passed object has a certain attribute:: - - def skipUnlessHasattr(obj, attr): - if hasattr(obj, attr): - return lambda func: func - return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) - -The following decorators implement test skipping and expected failures: - -.. decorator:: skip(reason) - - Unconditionally skip the decorated test. *reason* should describe why the - test is being skipped. - -.. decorator:: skipIf(condition, reason) - - Skip the decorated test if *condition* is true. - -.. decorator:: skipUnless(condition, reason) - - Skip the decorated test unless *condition* is true. - -.. decorator:: expectedFailure - - Mark the test as an expected failure. If the test fails when run, the test - is not counted as a failure. - -.. exception:: SkipTest(reason) - - This exception is raised to skip a test. - - Usually you can use :meth:`TestCase.skipTest` or one of the skipping - decorators instead of raising this directly. - -Skipped tests will not have :meth:`~TestCase.setUp` or :meth:`~TestCase.tearDown` run around them. -Skipped classes will not have :meth:`~TestCase.setUpClass` or :meth:`~TestCase.tearDownClass` run. -Skipped modules will not have :func:`setUpModule` or :func:`tearDownModule` run. - - -.. _subtests: - -Distinguishing test iterations using subtests ---------------------------------------------- - -.. versionadded:: 3.4 - -When there are very small differences among your tests, for -instance some parameters, unittest allows you to distinguish them inside -the body of a test method using the :meth:`~TestCase.subTest` context manager. - -For example, the following test:: - - class NumbersTest(unittest.TestCase): - - def test_even(self): - """ - Test that numbers between 0 and 5 are all even. - """ - for i in range(0, 6): - with self.subTest(i=i): - self.assertEqual(i % 2, 0) - -will produce the following output:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=1) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=3) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=5) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - -Without using a subtest, execution would stop after the first failure, -and the error would be less easy to diagnose because the value of ``i`` -wouldn't be displayed:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - -.. _unittest-contents: - -Classes and functions ---------------------- - -This section describes in depth the API of :mod:`unittest`. - - -.. _testcase-objects: - -Test cases -~~~~~~~~~~ - -.. class:: TestCase(methodName='runTest') - - Instances of the :class:`TestCase` class represent the logical test units - in the :mod:`unittest` universe. This class is intended to be used as a base - class, with specific tests being implemented by concrete subclasses. This class - implements the interface needed by the test runner to allow it to drive the - tests, and methods that the test code can use to check for and report various - kinds of failure. - - Each instance of :class:`TestCase` will run a single base method: the method - named *methodName*. - In most uses of :class:`TestCase`, you will neither change - the *methodName* nor reimplement the default ``runTest()`` method. - - .. versionchanged:: 3.2 - :class:`TestCase` can be instantiated successfully without providing a - *methodName*. This makes it easier to experiment with :class:`TestCase` - from the interactive interpreter. - - :class:`TestCase` instances provide three groups of methods: one group used - to run the test, another used by the test implementation to check conditions - and report failures, and some inquiry methods allowing information about the - test itself to be gathered. - - Methods in the first group (running the test) are: - - .. method:: setUp() - - Method called to prepare the test fixture. This is called immediately - before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, - any exception raised by this method will be considered an error rather than - a test failure. The default implementation does nothing. - - - .. method:: tearDown() - - Method called immediately after the test method has been called and the - result recorded. This is called even if the test method raised an - exception, so the implementation in subclasses may need to be particularly - careful about checking internal state. Any exception, other than - :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be - considered an additional error rather than a test failure (thus increasing - the total number of reported errors). This method will only be called if - the :meth:`setUp` succeeds, regardless of the outcome of the test method. - The default implementation does nothing. - - - .. method:: setUpClass() - - A class method called before tests in an individual class are run. - ``setUpClass`` is called with the class as the only argument - and must be decorated as a :func:`classmethod`:: - - @classmethod - def setUpClass(cls): - ... - - See `Class and Module Fixtures`_ for more details. - - .. versionadded:: 3.2 - - - .. method:: tearDownClass() - - A class method called after tests in an individual class have run. - ``tearDownClass`` is called with the class as the only argument - and must be decorated as a :meth:`classmethod`:: - - @classmethod - def tearDownClass(cls): - ... - - See `Class and Module Fixtures`_ for more details. - - .. versionadded:: 3.2 - - - .. method:: run(result=None) - - Run the test, collecting the result into the :class:`TestResult` object - passed as *result*. If *result* is omitted or ``None``, a temporary - result object is created (by calling the :meth:`defaultTestResult` - method) and used. The result object is returned to :meth:`run`'s - caller. - - The same effect may be had by simply calling the :class:`TestCase` - instance. - - .. versionchanged:: 3.3 - Previous versions of ``run`` did not return the result. Neither did - calling an instance. - - .. method:: skipTest(reason) - - Calling this during a test method or :meth:`setUp` skips the current - test. See :ref:`unittest-skipping` for more information. - - .. versionadded:: 3.1 - - - .. method:: subTest(msg=None, **params) - - Return a context manager which executes the enclosed code block as a - subtest. *msg* and *params* are optional, arbitrary values which are - displayed whenever a subtest fails, allowing you to identify them - clearly. - - A test case can contain any number of subtest declarations, and - they can be arbitrarily nested. - - See :ref:`subtests` for more information. - - .. versionadded:: 3.4 - - - .. method:: debug() - - Run the test without collecting the result. This allows exceptions raised - by the test to be propagated to the caller, and can be used to support - running tests under a debugger. - - .. _assert-methods: - - The :class:`TestCase` class provides several assert methods to check for and - report failures. The following table lists the most commonly used methods - (see the tables below for more assert methods): - - +-----------------------------------------+-----------------------------+---------------+ - | Method | Checks that | New in | - +=========================================+=============================+===============+ - | :meth:`assertEqual(a, b) | ``a == b`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotEqual(a, b) | ``a != b`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertTrue(x) | ``bool(x) is True`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertFalse(x) | ``bool(x) is False`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIs(a, b) | ``a is b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNot(a, b) | ``a is not b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNone(x) | ``x is None`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNotNone(x) | ``x is not None`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIn(a, b) | ``a in b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotIn(a, b) | ``a not in b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsInstance(a, b) | ``isinstance(a, b)`` | 3.2 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotIsInstance(a, b) | ``not isinstance(a, b)`` | 3.2 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - - All the assert methods accept a *msg* argument that, if specified, is used - as the error message on failure (see also :data:`longMessage`). - Note that the *msg* keyword argument can be passed to :meth:`assertRaises`, - :meth:`assertRaisesRegex`, :meth:`assertWarns`, :meth:`assertWarnsRegex` - only when they are used as a context manager. - - .. method:: assertEqual(first, second, msg=None) - - Test that *first* and *second* are equal. If the values do not - compare equal, the test will fail. - - In addition, if *first* and *second* are the exact same type and one of - list, tuple, dict, set, frozenset or str or any type that a subclass - registers with :meth:`addTypeEqualityFunc` the type-specific equality - function will be called in order to generate a more useful default - error message (see also the :ref:`list of type-specific methods - `). - - .. versionchanged:: 3.1 - Added the automatic calling of type-specific equality function. - - .. versionchanged:: 3.2 - :meth:`assertMultiLineEqual` added as the default type equality - function for comparing strings. - - - .. method:: assertNotEqual(first, second, msg=None) - - Test that *first* and *second* are not equal. If the values do - compare equal, the test will fail. - - .. method:: assertTrue(expr, msg=None) - assertFalse(expr, msg=None) - - Test that *expr* is true (or false). - - Note that this is equivalent to ``bool(expr) is True`` and not to ``expr - is True`` (use ``assertIs(expr, True)`` for the latter). This method - should also be avoided when more specific methods are available (e.g. - ``assertEqual(a, b)`` instead of ``assertTrue(a == b)``), because they - provide a better error message in case of failure. - - - .. method:: assertIs(first, second, msg=None) - assertIsNot(first, second, msg=None) - - Test that *first* and *second* evaluate (or don't evaluate) to the - same object. - - .. versionadded:: 3.1 - - - .. method:: assertIsNone(expr, msg=None) - assertIsNotNone(expr, msg=None) - - Test that *expr* is (or is not) ``None``. - - .. versionadded:: 3.1 - - - .. method:: assertIn(first, second, msg=None) - assertNotIn(first, second, msg=None) - - Test that *first* is (or is not) in *second*. - - .. versionadded:: 3.1 - - - .. method:: assertIsInstance(obj, cls, msg=None) - assertNotIsInstance(obj, cls, msg=None) - - Test that *obj* is (or is not) an instance of *cls* (which can be a - class or a tuple of classes, as supported by :func:`isinstance`). - To check for the exact type, use :func:`assertIs(type(obj), cls) `. - - .. versionadded:: 3.2 - - - - It is also possible to check the production of exceptions, warnings, and - log messages using the following methods: - - +---------------------------------------------------------+--------------------------------------+------------+ - | Method | Checks that | New in | - +=========================================================+======================================+============+ - | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | - | ` | | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegex(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | - | ` | and the message matches regex *r* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarns(warn, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | and the message matches regex *r* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 | - | ` | with minimum *level* | | - +---------------------------------------------------------+--------------------------------------+------------+ - - .. method:: assertRaises(exception, callable, *args, **kwds) - assertRaises(exception, *, msg=None) - - Test that an exception is raised when *callable* is called with any - positional or keyword arguments that are also passed to - :meth:`assertRaises`. The test passes if *exception* is raised, is an - error if another exception is raised, or fails if no exception is raised. - To catch any of a group of exceptions, a tuple containing the exception - classes may be passed as *exception*. - - If only the *exception* and possibly the *msg* arguments are given, - return a context manager so that the code under test can be written - inline rather than as a function:: - - with self.assertRaises(SomeException): - do_something() - - When used as a context manager, :meth:`assertRaises` accepts the - additional keyword argument *msg*. - - The context manager will store the caught exception object in its - :attr:`exception` attribute. This can be useful if the intention - is to perform additional checks on the exception raised:: - - with self.assertRaises(SomeException) as cm: - do_something() - - the_exception = cm.exception - self.assertEqual(the_exception.error_code, 3) - - .. versionchanged:: 3.1 - Added the ability to use :meth:`assertRaises` as a context manager. - - .. versionchanged:: 3.2 - Added the :attr:`exception` attribute. - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertRaisesRegex(exception, regex, callable, *args, **kwds) - assertRaisesRegex(exception, regex, *, msg=None) - - Like :meth:`assertRaises` but also tests that *regex* matches - on the string representation of the raised exception. *regex* may be - a regular expression object or a string containing a regular expression - suitable for use by :func:`re.search`. Examples:: - - self.assertRaisesRegex(ValueError, "invalid literal for.*XYZ'$", - int, 'XYZ') - - or:: - - with self.assertRaisesRegex(ValueError, 'literal'): - int('XYZ') - - .. versionadded:: 3.1 - under the name ``assertRaisesRegexp``. - - .. versionchanged:: 3.2 - Renamed to :meth:`assertRaisesRegex`. - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertWarns(warning, callable, *args, **kwds) - assertWarns(warning, *, msg=None) - - Test that a warning is triggered when *callable* is called with any - positional or keyword arguments that are also passed to - :meth:`assertWarns`. The test passes if *warning* is triggered and - fails if it isn't. Any exception is an error. - To catch any of a group of warnings, a tuple containing the warning - classes may be passed as *warnings*. - - If only the *warning* and possibly the *msg* arguments are given, - return a context manager so that the code under test can be written - inline rather than as a function:: - - with self.assertWarns(SomeWarning): - do_something() - - When used as a context manager, :meth:`assertWarns` accepts the - additional keyword argument *msg*. - - The context manager will store the caught warning object in its - :attr:`warning` attribute, and the source line which triggered the - warnings in the :attr:`filename` and :attr:`lineno` attributes. - This can be useful if the intention is to perform additional checks - on the warning caught:: - - with self.assertWarns(SomeWarning) as cm: - do_something() - - self.assertIn('myfile.py', cm.filename) - self.assertEqual(320, cm.lineno) - - This method works regardless of the warning filters in place when it - is called. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertWarnsRegex(warning, regex, callable, *args, **kwds) - assertWarnsRegex(warning, regex, *, msg=None) - - Like :meth:`assertWarns` but also tests that *regex* matches on the - message of the triggered warning. *regex* may be a regular expression - object or a string containing a regular expression suitable for use - by :func:`re.search`. Example:: - - self.assertWarnsRegex(DeprecationWarning, - r'legacy_function\(\) is deprecated', - legacy_function, 'XYZ') - - or:: - - with self.assertWarnsRegex(RuntimeWarning, 'unsafe frobnicating'): - frobnicate('/etc/passwd') - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - .. method:: assertLogs(logger=None, level=None) - - A context manager to test that at least one message is logged on - the *logger* or one of its children, with at least the given - *level*. - - If given, *logger* should be a :class:`logging.Logger` object or a - :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages. - - If given, *level* should be either a numeric logging level or - its string equivalent (for example either ``"ERROR"`` or - :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. - - The test passes if at least one message emitted inside the ``with`` - block matches the *logger* and *level* conditions, otherwise it fails. - - The object returned by the context manager is a recording helper - which keeps tracks of the matching log messages. It has two - attributes: - - .. attribute:: records - - A list of :class:`logging.LogRecord` objects of the matching - log messages. - - .. attribute:: output - - A list of :class:`str` objects with the formatted output of - matching messages. - - Example:: - - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - - .. versionadded:: 3.4 - - - There are also other methods used to perform more specific checks, such as: - - +---------------------------------------+--------------------------------+--------------+ - | Method | Checks that | New in | - +=======================================+================================+==============+ - | :meth:`assertAlmostEqual(a, b) | ``round(a-b, 7) == 0`` | | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotAlmostEqual(a, b) | ``round(a-b, 7) != 0`` | | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertGreater(a, b) | ``a > b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertGreaterEqual(a, b) | ``a >= b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertLess(a, b) | ``a < b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertLessEqual(a, b) | ``a <= b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegex(s, r) | ``r.search(s)`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | - | ` | elements in the same number, | | - | | regardless of their order | | - +---------------------------------------+--------------------------------+--------------+ - - - .. method:: assertAlmostEqual(first, second, places=7, msg=None, delta=None) - assertNotAlmostEqual(first, second, places=7, msg=None, delta=None) - - Test that *first* and *second* are approximately (or not approximately) - equal by computing the difference, rounding to the given number of - decimal *places* (default 7), and comparing to zero. Note that these - methods round the values to the given number of *decimal places* (i.e. - like the :func:`round` function) and not *significant digits*. - - If *delta* is supplied instead of *places* then the difference - between *first* and *second* must be less or equal to (or greater than) *delta*. - - Supplying both *delta* and *places* raises a :exc:`TypeError`. - - .. versionchanged:: 3.2 - :meth:`assertAlmostEqual` automatically considers almost equal objects - that compare equal. :meth:`assertNotAlmostEqual` automatically fails - if the objects compare equal. Added the *delta* keyword argument. - - - .. method:: assertGreater(first, second, msg=None) - assertGreaterEqual(first, second, msg=None) - assertLess(first, second, msg=None) - assertLessEqual(first, second, msg=None) - - Test that *first* is respectively >, >=, < or <= than *second* depending - on the method name. If not, the test will fail:: - - >>> self.assertGreaterEqual(3, 4) - AssertionError: "3" unexpectedly not greater than or equal to "4" - - .. versionadded:: 3.1 - - - .. method:: assertRegex(text, regex, msg=None) - assertNotRegex(text, regex, msg=None) - - Test that a *regex* search matches (or does not match) *text*. In case - of failure, the error message will include the pattern and the *text* (or - the pattern and the part of *text* that unexpectedly matched). *regex* - may be a regular expression object or a string containing a regular - expression suitable for use by :func:`re.search`. - - .. versionadded:: 3.1 - under the name ``assertRegexpMatches``. - .. versionchanged:: 3.2 - The method ``assertRegexpMatches()`` has been renamed to - :meth:`.assertRegex`. - .. versionadded:: 3.2 - :meth:`.assertNotRegex`. - .. versionadded:: 3.5 - The name ``assertNotRegexpMatches`` is a deprecated alias - for :meth:`.assertNotRegex`. - - - .. method:: assertCountEqual(first, second, msg=None) - - Test that sequence *first* contains the same elements as *second*, - regardless of their order. When they don't, an error message listing the - differences between the sequences will be generated. - - Duplicate elements are *not* ignored when comparing *first* and - *second*. It verifies whether each element has the same count in both - sequences. Equivalent to: - ``assertEqual(Counter(list(first)), Counter(list(second)))`` - but works with sequences of unhashable objects as well. - - .. versionadded:: 3.2 - - - .. _type-specific-methods: - - The :meth:`assertEqual` method dispatches the equality check for objects of - the same type to different type-specific methods. These methods are already - implemented for most of the built-in types, but it's also possible to - register new methods using :meth:`addTypeEqualityFunc`: - - .. method:: addTypeEqualityFunc(typeobj, function) - - Registers a type-specific method called by :meth:`assertEqual` to check - if two objects of exactly the same *typeobj* (not subclasses) compare - equal. *function* must take two positional arguments and a third msg=None - keyword argument just as :meth:`assertEqual` does. It must raise - :data:`self.failureException(msg) ` when inequality - between the first two parameters is detected -- possibly providing useful - information and explaining the inequalities in details in the error - message. - - .. versionadded:: 3.1 - - The list of type-specific methods automatically used by - :meth:`~TestCase.assertEqual` are summarized in the following table. Note - that it's usually not necessary to invoke these methods directly. - - +-----------------------------------------+-----------------------------+--------------+ - | Method | Used to compare | New in | - +=========================================+=============================+==============+ - | :meth:`assertMultiLineEqual(a, b) | strings | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertSequenceEqual(a, b) | sequences | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertListEqual(a, b) | lists | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertTupleEqual(a, b) | tuples | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertSetEqual(a, b) | sets or frozensets | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertDictEqual(a, b) | dicts | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - - - - .. method:: assertMultiLineEqual(first, second, msg=None) - - Test that the multiline string *first* is equal to the string *second*. - When not equal a diff of the two strings highlighting the differences - will be included in the error message. This method is used by default - when comparing strings with :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertSequenceEqual(first, second, msg=None, seq_type=None) - - Tests that two sequences are equal. If a *seq_type* is supplied, both - *first* and *second* must be instances of *seq_type* or a failure will - be raised. If the sequences are different an error message is - constructed that shows the difference between the two. - - This method is not called directly by :meth:`assertEqual`, but - it's used to implement :meth:`assertListEqual` and - :meth:`assertTupleEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertListEqual(first, second, msg=None) - assertTupleEqual(first, second, msg=None) - - Tests that two lists or tuples are equal. If not, an error message is - constructed that shows only the differences between the two. An error - is also raised if either of the parameters are of the wrong type. - These methods are used by default when comparing lists or tuples with - :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertSetEqual(first, second, msg=None) - - Tests that two sets are equal. If not, an error message is constructed - that lists the differences between the sets. This method is used by - default when comparing sets or frozensets with :meth:`assertEqual`. - - Fails if either of *first* or *second* does not have a :meth:`set.difference` - method. - - .. versionadded:: 3.1 - - - .. method:: assertDictEqual(first, second, msg=None) - - Test that two dictionaries are equal. If not, an error message is - constructed that shows the differences in the dictionaries. This - method will be used by default to compare dictionaries in - calls to :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - - .. _other-methods-and-attrs: - - Finally the :class:`TestCase` provides the following methods and attributes: - - - .. method:: fail(msg=None) - - Signals a test failure unconditionally, with *msg* or ``None`` for - the error message. - - - .. attribute:: failureException - - This class attribute gives the exception raised by the test method. If a - test framework needs to use a specialized exception, possibly to carry - additional information, it must subclass this exception in order to "play - fair" with the framework. The initial value of this attribute is - :exc:`AssertionError`. - - - .. attribute:: longMessage - - This class attribute determines what happens when a custom failure message - is passed as the msg argument to an assertXYY call that fails. - ``True`` is the default value. In this case, the custom message is appended - to the end of the standard failure message. - When set to ``False``, the custom message replaces the standard message. - - The class setting can be overridden in individual test methods by assigning - an instance attribute, self.longMessage, to ``True`` or ``False`` before - calling the assert methods. - - The class setting gets reset before each test call. - - .. versionadded:: 3.1 - - - .. attribute:: maxDiff - - This attribute controls the maximum length of diffs output by assert - methods that report diffs on failure. It defaults to 80*8 characters. - Assert methods affected by this attribute are - :meth:`assertSequenceEqual` (including all the sequence comparison - methods that delegate to it), :meth:`assertDictEqual` and - :meth:`assertMultiLineEqual`. - - Setting ``maxDiff`` to ``None`` means that there is no maximum length of - diffs. - - .. versionadded:: 3.2 - - - Testing frameworks can use the following methods to collect information on - the test: - - - .. method:: countTestCases() - - Return the number of tests represented by this test object. For - :class:`TestCase` instances, this will always be ``1``. - - - .. method:: defaultTestResult() - - Return an instance of the test result class that should be used for this - test case class (if no other result instance is provided to the - :meth:`run` method). - - For :class:`TestCase` instances, this will always be an instance of - :class:`TestResult`; subclasses of :class:`TestCase` should override this - as necessary. - - - .. method:: id() - - Return a string identifying the specific test case. This is usually the - full name of the test method, including the module and class name. - - - .. method:: shortDescription() - - Returns a description of the test, or ``None`` if no description - has been provided. The default implementation of this method - returns the first line of the test method's docstring, if available, - or ``None``. - - .. versionchanged:: 3.1 - In 3.1 this was changed to add the test name to the short description - even in the presence of a docstring. This caused compatibility issues - with unittest extensions and adding the test name was moved to the - :class:`TextTestResult` in Python 3.2. - - - .. method:: addCleanup(function, *args, **kwargs) - - Add a function to be called after :meth:`tearDown` to cleanup resources - used during the test. Functions will be called in reverse order to the - order they are added (:abbr:`LIFO (last-in, first-out)`). They - are called with any arguments and keyword arguments passed into - :meth:`addCleanup` when they are added. - - If :meth:`setUp` fails, meaning that :meth:`tearDown` is not called, - then any cleanup functions added will still be called. - - .. versionadded:: 3.1 - - - .. method:: doCleanups() - - This method is called unconditionally after :meth:`tearDown`, or - after :meth:`setUp` if :meth:`setUp` raises an exception. - - It is responsible for calling all the cleanup functions added by - :meth:`addCleanup`. If you need cleanup functions to be called - *prior* to :meth:`tearDown` then you can call :meth:`doCleanups` - yourself. - - :meth:`doCleanups` pops methods off the stack of cleanup - functions one at a time, so it can be called at any time. - - .. versionadded:: 3.1 - - -.. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None) - - This class implements the portion of the :class:`TestCase` interface which - allows the test runner to drive the test, but does not provide the methods - which test code can use to check and report errors. This is used to create - test cases using legacy test code, allowing it to be integrated into a - :mod:`unittest`-based test framework. - - -.. _deprecated-aliases: - -Deprecated aliases -################## - -For historical reasons, some of the :class:`TestCase` methods had one or more -aliases that are now deprecated. The following table lists the correct names -along with their deprecated aliases: - - ============================== ====================== ======================= - Method Name Deprecated alias Deprecated alias - ============================== ====================== ======================= - :meth:`.assertEqual` failUnlessEqual assertEquals - :meth:`.assertNotEqual` failIfEqual assertNotEquals - :meth:`.assertTrue` failUnless assert\_ - :meth:`.assertFalse` failIf - :meth:`.assertRaises` failUnlessRaises - :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals - :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals - :meth:`.assertRegex` assertRegexpMatches - :meth:`.assertNotRegex` assertNotRegexpMatches - :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ======================= - - .. deprecated:: 3.1 - the fail* aliases listed in the second column. - .. deprecated:: 3.2 - the assert* aliases listed in the third column. - .. deprecated:: 3.2 - ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. - .. deprecated:: 3.5 - the ``assertNotRegexpMatches`` name in favor of :meth:`.assertNotRegex`. - -.. _testsuite-objects: - -Grouping tests -~~~~~~~~~~~~~~ - -.. class:: TestSuite(tests=()) - - This class represents an aggregation of individual test cases and test suites. - The class presents the interface needed by the test runner to allow it to be run - as any other test case. Running a :class:`TestSuite` instance is the same as - iterating over the suite, running each test individually. - - If *tests* is given, it must be an iterable of individual test cases or other - test suites that will be used to build the suite initially. Additional methods - are provided to add test cases and suites to the collection later on. - - :class:`TestSuite` objects behave much like :class:`TestCase` objects, except - they do not actually implement a test. Instead, they are used to aggregate - tests into groups of tests that should be run together. Some additional - methods are available to add tests to :class:`TestSuite` instances: - - - .. method:: TestSuite.addTest(test) - - Add a :class:`TestCase` or :class:`TestSuite` to the suite. - - - .. method:: TestSuite.addTests(tests) - - Add all the tests from an iterable of :class:`TestCase` and :class:`TestSuite` - instances to this test suite. - - This is equivalent to iterating over *tests*, calling :meth:`addTest` for - each element. - - :class:`TestSuite` shares the following methods with :class:`TestCase`: - - - .. method:: run(result) - - Run the tests associated with this suite, collecting the result into the - test result object passed as *result*. Note that unlike - :meth:`TestCase.run`, :meth:`TestSuite.run` requires the result object to - be passed in. - - - .. method:: debug() - - Run the tests associated with this suite without collecting the - result. This allows exceptions raised by the test to be propagated to the - caller and can be used to support running tests under a debugger. - - - .. method:: countTestCases() - - Return the number of tests represented by this test object, including all - individual tests and sub-suites. - - - .. method:: __iter__() - - Tests grouped by a :class:`TestSuite` are always accessed by iteration. - Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note - that this method may be called several times on a single suite (for - example when counting tests or comparing for equality) so the tests - returned by repeated iterations before :meth:`TestSuite.run` must be the - same for each call iteration. After :meth:`TestSuite.run`, callers should - not rely on the tests returned by this method unless the caller uses a - subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve - test references. - - .. versionchanged:: 3.2 - In earlier versions the :class:`TestSuite` accessed tests directly rather - than through iteration, so overriding :meth:`__iter__` wasn't sufficient - for providing tests. - - .. versionchanged:: 3.4 - In earlier versions the :class:`TestSuite` held references to each - :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore - that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. - - In the typical usage of a :class:`TestSuite` object, the :meth:`run` method - is invoked by a :class:`TestRunner` rather than by the end-user test harness. - - -Loading and running tests -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: TestLoader() - - The :class:`TestLoader` class is used to create test suites from classes and - modules. Normally, there is no need to create an instance of this class; the - :mod:`unittest` module provides an instance that can be shared as - :data:`unittest.defaultTestLoader`. Using a subclass or instance, however, - allows customization of some configurable properties. - - :class:`TestLoader` objects have the following attributes: - - - .. attribute:: errors - - A list of the non-fatal errors encountered while loading tests. Not reset - by the loader at any point. Fatal errors are signalled by the relevant - a method raising an exception to the caller. Non-fatal errors are also - indicated by a synthetic test that will raise the original error when - run. - - .. versionadded:: 3.5 - - - :class:`TestLoader` objects have the following methods: - - - .. method:: loadTestsFromTestCase(testCaseClass) - - Return a suite of all test cases contained in the :class:`TestCase`\ -derived - :class:`testCaseClass`. - - A test case instance is created for each method named by - :meth:`getTestCaseNames`. By default these are the method names - beginning with ``test``. If :meth:`getTestCaseNames` returns no - methods, but the :meth:`runTest` method is implemented, a single test - case is created for that method instead. - - - .. method:: loadTestsFromModule(module, pattern=None) - - Return a suite of all test cases contained in the given module. This - method searches *module* for classes derived from :class:`TestCase` and - creates an instance of the class for each test method defined for the - class. - - .. note:: - - While using a hierarchy of :class:`TestCase`\ -derived classes can be - convenient in sharing fixtures and helper functions, defining test - methods on base classes that are not intended to be instantiated - directly does not play well with this method. Doing so, however, can - be useful when the fixtures are different and defined in subclasses. - - If a module provides a ``load_tests`` function it will be called to - load the tests. This allows modules to customize test loading. - This is the `load_tests protocol`_. The *pattern* argument is passed as - the third argument to ``load_tests``. - - .. versionchanged:: 3.2 - Support for ``load_tests`` added. - - .. versionchanged:: 3.5 - The undocumented and unofficial *use_load_tests* default argument is - deprecated and ignored, although it is still accepted for backward - compatibility. The method also now accepts a keyword-only argument - *pattern* which is passed to ``load_tests`` as the third argument. - - - .. method:: loadTestsFromName(name, module=None) - - Return a suite of all test cases given a string specifier. - - The specifier *name* is a "dotted name" that may resolve either to a - module, a test case class, a test method within a test case class, a - :class:`TestSuite` instance, or a callable object which returns a - :class:`TestCase` or :class:`TestSuite` instance. These checks are - applied in the order listed here; that is, a method on a possible test - case class will be picked up as "a test method within a test case class", - rather than "a callable object". - - For example, if you have a module :mod:`SampleTests` containing a - :class:`TestCase`\ -derived class :class:`SampleTestCase` with three test - methods (:meth:`test_one`, :meth:`test_two`, and :meth:`test_three`), the - specifier ``'SampleTests.SampleTestCase'`` would cause this method to - return a suite which will run all three test methods. Using the specifier - ``'SampleTests.SampleTestCase.test_two'`` would cause it to return a test - suite which will run only the :meth:`test_two` test method. The specifier - can refer to modules and packages which have not been imported; they will - be imported as a side-effect. - - The method optionally resolves *name* relative to the given *module*. - - .. versionchanged:: 3.5 - If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing - *name* then a synthetic test that raises that error when run will be - returned. These errors are included in the errors accumulated by - self.errors. - - - .. method:: loadTestsFromNames(names, module=None) - - Similar to :meth:`loadTestsFromName`, but takes a sequence of names rather - than a single name. The return value is a test suite which supports all - the tests defined for each name. - - - .. method:: getTestCaseNames(testCaseClass) - - Return a sorted sequence of method names found within *testCaseClass*; - this should be a subclass of :class:`TestCase`. - - - .. method:: discover(start_dir, pattern='test*.py', top_level_dir=None) - - Find all the test modules by recursing into subdirectories from the - specified start directory, and return a TestSuite object containing them. - Only test files that match *pattern* will be loaded. (Using shell style - pattern matching.) Only module names that are importable (i.e. are valid - Python identifiers) will be loaded. - - All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. - - If importing a module fails, for example due to a syntax error, then - this will be recorded as a single error and discovery will continue. If - the import failure is due to :exc:`SkipTest` being raised, it will be - recorded as a skip instead of an error. - - If a package (a directory containing a file named :file:`__init__.py`) is - found, the package will be checked for a ``load_tests`` function. If this - exists then it will be called - ``package.load_tests(loader, tests, pattern)``. Test discovery takes care - to ensure that a package is only checked for tests once during an - invocation, even if the load_tests function itself calls - ``loader.discover``. - - If ``load_tests`` exists then discovery does *not* recurse into the - package, ``load_tests`` is responsible for loading all tests in the - package. - - The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. - - *start_dir* can be a dotted module name as well as a directory. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - Discovery works for :term:`namespace packages `. - Paths are sorted before being imported so that execution order is - the same even if the underlying file system's ordering is not - dependent on file name. - - .. versionchanged:: 3.5 - Found packages are now checked for ``load_tests`` regardless of - whether their path matches *pattern*, because it is impossible for - a package name to match the default pattern. - - - The following attributes of a :class:`TestLoader` can be configured either by - subclassing or assignment on an instance: - - - .. attribute:: testMethodPrefix - - String giving the prefix of method names which will be interpreted as test - methods. The default value is ``'test'``. - - This affects :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` - methods. - - - .. attribute:: sortTestMethodsUsing - - Function to be used to compare method names when sorting them in - :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` methods. - - - .. attribute:: suiteClass - - Callable object that constructs a test suite from a list of tests. No - methods on the resulting object are needed. The default value is the - :class:`TestSuite` class. - - This affects all the :meth:`loadTestsFrom\*` methods. - - -.. class:: TestResult - - This class is used to compile information about which tests have succeeded - and which have failed. - - A :class:`TestResult` object stores the results of a set of tests. The - :class:`TestCase` and :class:`TestSuite` classes ensure that results are - properly recorded; test authors do not need to worry about recording the - outcome of tests. - - Testing frameworks built on top of :mod:`unittest` may want access to the - :class:`TestResult` object generated by running a set of tests for reporting - purposes; a :class:`TestResult` instance is returned by the - :meth:`TestRunner.run` method for this purpose. - - :class:`TestResult` instances have the following attributes that will be of - interest when inspecting the results of running a set of tests: - - - .. attribute:: errors - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents a test which raised an - unexpected exception. - - .. attribute:: failures - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. - - .. attribute:: skipped - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding the reason for skipping the test. - - .. versionadded:: 3.1 - - .. attribute:: expectedFailures - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents an expected failure - of the test case. - - .. attribute:: unexpectedSuccesses - - A list containing :class:`TestCase` instances that were marked as expected - failures, but succeeded. - - .. attribute:: shouldStop - - Set to ``True`` when the execution of tests should stop by :meth:`stop`. - - .. attribute:: testsRun - - The total number of tests run so far. - - .. attribute:: buffer - - If set to true, ``sys.stdout`` and ``sys.stderr`` will be buffered in between - :meth:`startTest` and :meth:`stopTest` being called. Collected output will - only be echoed onto the real ``sys.stdout`` and ``sys.stderr`` if the test - fails or errors. Any output is also attached to the failure / error message. - - .. versionadded:: 3.2 - - .. attribute:: failfast - - If set to true :meth:`stop` will be called on the first failure or error, - halting the test run. - - .. versionadded:: 3.2 - - .. attribute:: tb_locals - - If set to true then local variables will be shown in tracebacks. - - .. versionadded:: 3.5 - - .. method:: wasSuccessful() - - Return ``True`` if all tests run so far have passed, otherwise returns - ``False``. - - .. versionchanged:: 3.4 - Returns ``False`` if there were any :attr:`unexpectedSuccesses` - from tests marked with the :func:`expectedFailure` decorator. - - .. method:: stop() - - This method can be called to signal that the set of tests being run should - be aborted by setting the :attr:`shouldStop` attribute to ``True``. - :class:`TestRunner` objects should respect this flag and return without - running any additional tests. - - For example, this feature is used by the :class:`TextTestRunner` class to - stop the test framework when the user signals an interrupt from the - keyboard. Interactive tools which provide :class:`TestRunner` - implementations can use this in a similar manner. - - The following methods of the :class:`TestResult` class are used to maintain - the internal data structures, and may be extended in subclasses to support - additional reporting requirements. This is particularly useful in building - tools which support interactive reporting while tests are being run. - - - .. method:: startTest(test) - - Called when the test case *test* is about to be run. - - .. method:: stopTest(test) - - Called after the test case *test* has been executed, regardless of the - outcome. - - .. method:: startTestRun() - - Called once before any tests are executed. - - .. versionadded:: 3.1 - - - .. method:: stopTestRun() - - Called once after all tests are executed. - - .. versionadded:: 3.1 - - - .. method:: addError(test, err) - - Called when the test case *test* raises an unexpected exception. *err* is a - tuple of the form returned by :func:`sys.exc_info`: ``(type, value, - traceback)``. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`errors` attribute, where *formatted_err* is a - formatted traceback derived from *err*. - - - .. method:: addFailure(test, err) - - Called when the test case *test* signals a failure. *err* is a tuple of - the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`failures` attribute, where *formatted_err* is a - formatted traceback derived from *err*. - - - .. method:: addSuccess(test) - - Called when the test case *test* succeeds. - - The default implementation does nothing. - - - .. method:: addSkip(test, reason) - - Called when the test case *test* is skipped. *reason* is the reason the - test gave for skipping. - - The default implementation appends a tuple ``(test, reason)`` to the - instance's :attr:`skipped` attribute. - - - .. method:: addExpectedFailure(test, err) - - Called when the test case *test* fails, but was marked with the - :func:`expectedFailure` decorator. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`expectedFailures` attribute, where *formatted_err* - is a formatted traceback derived from *err*. - - - .. method:: addUnexpectedSuccess(test) - - Called when the test case *test* was marked with the - :func:`expectedFailure` decorator, but succeeded. - - The default implementation appends the test to the instance's - :attr:`unexpectedSuccesses` attribute. - - - .. method:: addSubTest(test, subtest, outcome) - - Called when a subtest finishes. *test* is the test case - corresponding to the test method. *subtest* is a custom - :class:`TestCase` instance describing the subtest. - - If *outcome* is :const:`None`, the subtest succeeded. Otherwise, - it failed with an exception where *outcome* is a tuple of the form - returned by :func:`sys.exc_info`: ``(type, value, traceback)``. - - The default implementation does nothing when the outcome is a - success, and records subtest failures as normal failures. - - .. versionadded:: 3.4 - - -.. class:: TextTestResult(stream, descriptions, verbosity) - - A concrete implementation of :class:`TestResult` used by the - :class:`TextTestRunner`. - - .. versionadded:: 3.2 - This class was previously named ``_TextTestResult``. The old name still - exists as an alias but is deprecated. - - -.. data:: defaultTestLoader - - Instance of the :class:`TestLoader` class intended to be shared. If no - customization of the :class:`TestLoader` is needed, this instance can be used - instead of repeatedly creating new instances. - - -.. class:: TextTestRunner(stream=None, descriptions=True, verbosity=1, failfast=False, \ - buffer=False, resultclass=None, warnings=None, *, tb_locals=False) - - A basic test runner implementation that outputs results to a stream. If *stream* - is ``None``, the default, :data:`sys.stderr` is used as the output stream. This class - has a few configurable parameters, but is essentially very simple. Graphical - applications which run test suites should provide alternate implementations. Such - implementations should accept ``**kwargs`` as the interface to construct runners - changes when features are added to unittest. - - By default this runner shows :exc:`DeprecationWarning`, - :exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and - :exc:`ImportWarning` even if they are :ref:`ignored by default - `. Deprecation warnings caused by :ref:`deprecated unittest - methods ` are also special-cased and, when the warning - filters are ``'default'`` or ``'always'``, they will appear only once - per-module, in order to avoid too many warning messages. This behavior can - be overridden using Python's :option:`!-Wd` or :option:`!-Wa` options - (see :ref:`Warning control `) and leaving - *warnings* to ``None``. - - .. versionchanged:: 3.2 - Added the ``warnings`` argument. - - .. versionchanged:: 3.2 - The default stream is set to :data:`sys.stderr` at instantiation time rather - than import time. - - .. versionchanged:: 3.5 - Added the tb_locals parameter. - - .. method:: _makeResult() - - This method returns the instance of ``TestResult`` used by :meth:`run`. - It is not intended to be called directly, but can be overridden in - subclasses to provide a custom ``TestResult``. - - ``_makeResult()`` instantiates the class or callable passed in the - ``TextTestRunner`` constructor as the ``resultclass`` argument. It - defaults to :class:`TextTestResult` if no ``resultclass`` is provided. - The result class is instantiated with the following arguments:: - - stream, descriptions, verbosity - - .. method:: run(test) - - This method is the main public interface to the `TextTestRunner`. This - method takes a :class:`TestSuite` or :class:`TestCase` instance. A - :class:`TestResult` is created by calling - :func:`_makeResult` and the test(s) are run and the - results printed to stdout. - - -.. function:: main(module='__main__', defaultTest=None, argv=None, testRunner=None, \ - testLoader=unittest.defaultTestLoader, exit=True, verbosity=1, \ - failfast=None, catchbreak=None, buffer=None, warnings=None) - - A command-line program that loads a set of tests from *module* and runs them; - this is primarily for making test modules conveniently executable. - The simplest use for this function is to include the following line at the - end of a test script:: - - if __name__ == '__main__': - unittest.main() - - You can run tests with more detailed information by passing in the verbosity - argument:: - - if __name__ == '__main__': - unittest.main(verbosity=2) - - The *defaultTest* argument is either the name of a single test or an - iterable of test names to run if no test names are specified via *argv*. If - not specified or ``None`` and no test names are provided via *argv*, all - tests found in *module* are run. - - The *argv* argument can be a list of options passed to the program, with the - first element being the program name. If not specified or ``None``, - the values of :data:`sys.argv` are used. - - The *testRunner* argument can either be a test runner class or an already - created instance of it. By default ``main`` calls :func:`sys.exit` with - an exit code indicating success or failure of the tests run. - - The *testLoader* argument has to be a :class:`TestLoader` instance, - and defaults to :data:`defaultTestLoader`. - - ``main`` supports being used from the interactive interpreter by passing in the - argument ``exit=False``. This displays the result on standard output without - calling :func:`sys.exit`:: - - >>> from unittest import main - >>> main(module='test_module', exit=False) - - The *failfast*, *catchbreak* and *buffer* parameters have the same - effect as the same-name `command-line options`_. - - The *warnings* argument specifies the :ref:`warning filter ` - that should be used while running the tests. If it's not specified, it will - remain ``None`` if a :option:`!-W` option is passed to :program:`python` - (see :ref:`Warning control `), - otherwise it will be set to ``'default'``. - - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. - - .. versionchanged:: 3.1 - The *exit* parameter was added. - - .. versionchanged:: 3.2 - The *verbosity*, *failfast*, *catchbreak*, *buffer* - and *warnings* parameters were added. - - .. versionchanged:: 3.4 - The *defaultTest* parameter was changed to also accept an iterable of - test names. - - -load_tests Protocol -################### - -.. versionadded:: 3.2 - -Modules or packages can customize how tests are loaded from them during normal -test runs or test discovery by implementing a function called ``load_tests``. - -If a test module defines ``load_tests`` it will be called by -:meth:`TestLoader.loadTestsFromModule` with the following arguments:: - - load_tests(loader, standard_tests, pattern) - -where *pattern* is passed straight through from ``loadTestsFromModule``. It -defaults to ``None``. - -It should return a :class:`TestSuite`. - -*loader* is the instance of :class:`TestLoader` doing the loading. -*standard_tests* are the tests that would be loaded by default from the -module. It is common for test modules to only want to add or remove tests -from the standard set of tests. -The third argument is used when loading packages as part of test discovery. - -A typical ``load_tests`` function that loads tests from a specific set of -:class:`TestCase` classes may look like:: - - test_cases = (TestCase1, TestCase2, TestCase3) - - def load_tests(loader, tests, pattern): - suite = TestSuite() - for test_class in test_cases: - tests = loader.loadTestsFromTestCase(test_class) - suite.addTests(tests) - return suite - -If discovery is started in a directory containing a package, either from the -command line or by calling :meth:`TestLoader.discover`, then the package -:file:`__init__.py` will be checked for ``load_tests``. If that function does -not exist, discovery will recurse into the package as though it were just -another directory. Otherwise, discovery of the package's tests will be left up -to ``load_tests`` which is called with the following arguments:: - - load_tests(loader, standard_tests, pattern) - -This should return a :class:`TestSuite` representing all the tests -from the package. (``standard_tests`` will only contain tests -collected from :file:`__init__.py`.) - -Because the pattern is passed into ``load_tests`` the package is free to -continue (and potentially modify) test discovery. A 'do nothing' -``load_tests`` function for a test package would look like:: - - def load_tests(loader, standard_tests, pattern): - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - package_tests = loader.discover(start_dir=this_dir, pattern=pattern) - standard_tests.addTests(package_tests) - return standard_tests - -.. versionchanged:: 3.5 - Discovery no longer checks package names for matching *pattern* due to the - impossibility of package names matching the default pattern. - - - -Class and Module Fixtures -------------------------- - -Class and module level fixtures are implemented in :class:`TestSuite`. When -the test suite encounters a test from a new class then :meth:`tearDownClass` -from the previous class (if there is one) is called, followed by -:meth:`setUpClass` from the new class. - -Similarly if a test is from a different module from the previous test then -``tearDownModule`` from the previous module is run, followed by -``setUpModule`` from the new module. - -After all the tests have run the final ``tearDownClass`` and -``tearDownModule`` are run. - -Note that shared fixtures do not play well with [potential] features like test -parallelization and they break test isolation. They should be used with care. - -The default ordering of tests created by the unittest test loaders is to group -all tests from the same modules and classes together. This will lead to -``setUpClass`` / ``setUpModule`` (etc) being called exactly once per class and -module. If you randomize the order, so that tests from different modules and -classes are adjacent to each other, then these shared fixture functions may be -called multiple times in a single test run. - -Shared fixtures are not intended to work with suites with non-standard -ordering. A ``BaseTestSuite`` still exists for frameworks that don't want to -support shared fixtures. - -If there are any exceptions raised during one of the shared fixture functions -the test is reported as an error. Because there is no corresponding test -instance an ``_ErrorHolder`` object (that has the same interface as a -:class:`TestCase`) is created to represent the error. If you are just using -the standard unittest test runner then this detail doesn't matter, but if you -are a framework author it may be relevant. - - -setUpClass and tearDownClass -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These must be implemented as class methods:: - - import unittest - - class Test(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._connection = createExpensiveConnectionObject() - - @classmethod - def tearDownClass(cls): - cls._connection.destroy() - -If you want the ``setUpClass`` and ``tearDownClass`` on base classes called -then you must call up to them yourself. The implementations in -:class:`TestCase` are empty. - -If an exception is raised during a ``setUpClass`` then the tests in the class -are not run and the ``tearDownClass`` is not run. Skipped classes will not -have ``setUpClass`` or ``tearDownClass`` run. If the exception is a -:exc:`SkipTest` exception then the class will be reported as having been skipped -instead of as an error. - - -setUpModule and tearDownModule -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These should be implemented as functions:: - - def setUpModule(): - createConnection() - - def tearDownModule(): - closeConnection() - -If an exception is raised in a ``setUpModule`` then none of the tests in the -module will be run and the ``tearDownModule`` will not be run. If the exception is a -:exc:`SkipTest` exception then the module will be reported as having been skipped -instead of as an error. - - -Signal Handling ---------------- - -.. versionadded:: 3.2 - -The :option:`-c/--catch ` command-line option to unittest, -along with the ``catchbreak`` parameter to :func:`unittest.main()`, provide -more friendly handling of control-C during a test run. With catch break -behavior enabled control-C will allow the currently running test to complete, -and the test run will then end and report all the results so far. A second -control-c will raise a :exc:`KeyboardInterrupt` in the usual way. - -The control-c handling signal handler attempts to remain compatible with code or -tests that install their own :const:`signal.SIGINT` handler. If the ``unittest`` -handler is called but *isn't* the installed :const:`signal.SIGINT` handler, -i.e. it has been replaced by the system under test and delegated to, then it -calls the default handler. This will normally be the expected behavior by code -that replaces an installed handler and delegates to it. For individual tests -that need ``unittest`` control-c handling disabled the :func:`removeHandler` -decorator can be used. - -There are a few utility functions for framework authors to enable control-c -handling functionality within test frameworks. - -.. function:: installHandler() - - Install the control-c handler. When a :const:`signal.SIGINT` is received - (usually in response to the user pressing control-c) all registered results - have :meth:`~TestResult.stop` called. - - -.. function:: registerResult(result) - - Register a :class:`TestResult` object for control-c handling. Registering a - result stores a weak reference to it, so it doesn't prevent the result from - being garbage collected. - - Registering a :class:`TestResult` object has no side-effects if control-c - handling is not enabled, so test frameworks can unconditionally register - all results they create independently of whether or not handling is enabled. - - -.. function:: removeResult(result) - - Remove a registered result. Once a result has been removed then - :meth:`~TestResult.stop` will no longer be called on that result object in - response to a control-c. - - -.. function:: removeHandler(function=None) - - When called without arguments this function removes the control-c handler - if it has been installed. This function can also be used as a test decorator - to temporarily remove the handler while the test is being executed:: - - @unittest.removeHandler - def test_signal_handling(self): - ... diff --git a/third_party/python/Doc/library/unix.rst b/third_party/python/Doc/library/unix.rst deleted file mode 100644 index 04d4081f4..000000000 --- a/third_party/python/Doc/library/unix.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _unix: - -********************** -Unix Specific Services -********************** - -The modules described in this chapter provide interfaces to features that are -unique to the Unix operating system, or in some cases to some or many variants -of it. Here's an overview: - - -.. toctree:: - - posix.rst - pwd.rst - spwd.rst - grp.rst - crypt.rst - termios.rst - tty.rst - pty.rst - fcntl.rst - pipes.rst - resource.rst - nis.rst - syslog.rst diff --git a/third_party/python/Doc/library/urllib.error.rst b/third_party/python/Doc/library/urllib.error.rst deleted file mode 100644 index f7d47ed76..000000000 --- a/third_party/python/Doc/library/urllib.error.rst +++ /dev/null @@ -1,66 +0,0 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== - -.. module:: urllib.error - :synopsis: Exception classes raised by urllib.request. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Senthil Kumaran - -**Source code:** :source:`Lib/urllib/error.py` - --------------- - -The :mod:`urllib.error` module defines the exception classes for exceptions -raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`. - -The following exceptions are raised by :mod:`urllib.error` as appropriate: - -.. exception:: URLError - - The handlers raise this exception (or derived exceptions) when they run into - a problem. It is a subclass of :exc:`OSError`. - - .. attribute:: reason - - The reason for this error. It can be a message string or another - exception instance. - - .. versionchanged:: 3.3 - :exc:`URLError` has been made a subclass of :exc:`OSError` instead - of :exc:`IOError`. - - -.. exception:: HTTPError - - Though being an exception (a subclass of :exc:`URLError`), an - :exc:`HTTPError` can also function as a non-exceptional file-like return - value (the same thing that :func:`~urllib.request.urlopen` returns). This - is useful when handling exotic HTTP errors, such as requests for - authentication. - - .. attribute:: code - - An HTTP status code as defined in :rfc:`2616`. This numeric value corresponds - to a value found in the dictionary of codes as found in - :attr:`http.server.BaseHTTPRequestHandler.responses`. - - .. attribute:: reason - - This is usually a string explaining the reason for this error. - - .. attribute:: headers - - The HTTP response headers for the HTTP request that caused the - :exc:`HTTPError`. - - .. versionadded:: 3.4 - -.. exception:: ContentTooShortError(msg, content) - - This exception is raised when the :func:`~urllib.request.urlretrieve` - function detects that - the amount of the downloaded data is less than the expected amount (given by - the *Content-Length* header). The :attr:`content` attribute stores the - downloaded (and supposedly truncated) data. - diff --git a/third_party/python/Doc/library/urllib.parse.rst b/third_party/python/Doc/library/urllib.parse.rst deleted file mode 100644 index b717d7cc0..000000000 --- a/third_party/python/Doc/library/urllib.parse.rst +++ /dev/null @@ -1,671 +0,0 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== - -.. module:: urllib.parse - :synopsis: Parse URLs into or assemble them from components. - -**Source code:** :source:`Lib/urllib/parse.py` - -.. index:: - single: WWW - single: World Wide Web - single: URL - pair: URL; parsing - pair: relative; URL - --------------- - -This module defines a standard interface to break Uniform Resource Locator (URL) -strings up in components (addressing scheme, network location, path etc.), to -combine the components back into a URL string, and to convert a "relative URL" -to an absolute URL given a "base URL." - -The module has been designed to match the Internet RFC on Relative Uniform -Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, -``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, -``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``, ``ws``, ``wss``. - -The :mod:`urllib.parse` module defines functions that fall into two broad -categories: URL parsing and URL quoting. These are covered in detail in -the following sections. - -URL Parsing ------------ - -The URL parsing functions focus on splitting a URL string into its components, -or on combining URL components into a URL string. - -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) - - Parse a URL into six components, returning a 6-tuple. This corresponds to the - general structure of a URL: ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up in - smaller parts (for example, the network location is a single string), and % - escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: - - >>> from urllib.parse import urlparse - >>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html') - >>> o # doctest: +NORMALIZE_WHITESPACE - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> o.scheme - 'http' - >>> o.port - 80 - >>> o.geturl() - 'http://www.cwi.nl:80/%7Eguido/Python.html' - - Following the syntax specifications in :rfc:`1808`, urlparse recognizes - a netloc only if it is properly introduced by '//'. Otherwise the - input is presumed to be a relative URL and thus to start with - a path component. - - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', - query='', fragment='') - - The *scheme* argument gives the default addressing scheme, to be - used only if the URL does not specify one. It should be the same type - (text or bytes) as *urlstring*, except that the default value ``''`` is - always allowed, and is automatically converted to ``b''`` if appropriate. - - If the *allow_fragments* argument is false, fragment identifiers are not - recognized. Instead, they are parsed as part of the path, parameters - or query component, and :attr:`fragment` is set to the empty string in - the return value. - - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: - - +------------------+-------+--------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+==========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+--------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`params` | 3 | Parameters for last path | empty string | - | | | element | | - +------------------+-------+--------------------------+----------------------+ - | :attr:`query` | 4 | Query component | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`fragment` | 5 | Fragment identifier | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+--------------------------+----------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - .. versionchanged:: 3.2 - Added IPv6 URL parsing capabilities. - - .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is - false), in accordance with :rfc:`3986`. Previously, a whitelist of - schemes that support fragments existed. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - - .. versionchanged:: 3.6.9 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. - - -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') - - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a - dictionary. The dictionary keys are the unique query variable names and the - values are lists of values for each name. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in percent-encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - The optional argument *max_num_fields* is the maximum number of fields to - read. If set, then throws a :exc:`ValueError` if there are more than - *max_num_fields* fields read. - - The optional argument *separator* is the symbol to use for separating the - query arguments. It defaults to ``&``. - - Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` - parameter set to ``True``) to convert such dictionaries into query - strings. - - - .. versionchanged:: 3.2 - Add *encoding* and *errors* parameters. - - .. versionchanged:: 3.6.8 - Added *max_num_fields* parameter. - - .. versionchanged:: 3.6.13 - Added *separator* parameter with the default value of ``&``. Python - versions earlier than Python 3.6.13 allowed using both ``;`` and ``&`` as - query parameter separator. This has been changed to allow only a single - separator key, with ``&`` as the default separator. - - -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') - - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of - name, value pairs. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in percent-encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - The optional argument *max_num_fields* is the maximum number of fields to - read. If set, then throws a :exc:`ValueError` if there are more than - *max_num_fields* fields read. - - The optional argument *separator* is the symbol to use for separating the - query arguments. It defaults to ``&``. - - Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into - query strings. - - .. versionchanged:: 3.2 - Add *encoding* and *errors* parameters. - - .. versionchanged:: 3.6.8 - Added *max_num_fields* parameter. - - .. versionchanged:: 3.6.13 - Added *separator* parameter with the default value of ``&``. Python - versions earlier than Python 3.6.13 allowed using both ``;`` and ``&`` as - query parameter separator. This has been changed to allow only a single - separator key, with ``&`` as the default separator. - - -.. function:: urlunparse(parts) - - Construct a URL from a tuple as returned by ``urlparse()``. The *parts* - argument can be any six-item iterable. This may result in a slightly - different, but equivalent URL, if the URL that was parsed originally had - unnecessary delimiters (for example, a ``?`` with an empty query; the RFC - states that these are equivalent). - - -.. function:: urlsplit(urlstring, scheme='', allow_fragments=True) - - This is similar to :func:`urlparse`, but does not split the params from the URL. - This should generally be used instead of :func:`urlparse` if the more recent URL - syntax allowing parameters to be applied to each segment of the *path* portion - of the URL (see :rfc:`2396`) is wanted. A separate function is needed to - separate the path segments and parameters. This function returns a 5-tuple: - (addressing scheme, network location, path, query, fragment identifier). - - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`query` | 3 | Query component | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 4 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+----------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline - ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - - .. versionchanged:: 3.6.9 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. - - .. versionchanged:: 3.6.14 - ASCII newline and tab characters are stripped from the URL. - -.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser - -.. function:: urlunsplit(parts) - - Combine the elements of a tuple as returned by :func:`urlsplit` into a - complete URL as a string. The *parts* argument can be any five-item - iterable. This may result in a slightly different, but equivalent URL, if the - URL that was parsed originally had unnecessary delimiters (for example, a ? - with an empty query; the RFC states that these are equivalent). - - -.. function:: urljoin(base, url, allow_fragments=True) - - Construct a full ("absolute") URL by combining a "base URL" (*base*) with - another URL (*url*). Informally, this uses components of the base URL, in - particular the addressing scheme, the network location and (part of) the - path, to provide missing components in the relative URL. For example: - - >>> from urllib.parse import urljoin - >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html') - 'http://www.cwi.nl/%7Eguido/FAQ.html' - - The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. - - .. note:: - - If *url* is an absolute URL (that is, starting with ``//`` or ``scheme://``), - the *url*'s host name and/or scheme will be present in the result. For example: - - .. doctest:: - - >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', - ... '//www.python.org/%7Eguido') - 'http://www.python.org/%7Eguido' - - If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and - :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. - - - .. versionchanged:: 3.5 - - Behaviour updated to match the semantics defined in :rfc:`3986`. - - -.. function:: urldefrag(url) - - If *url* contains a fragment identifier, return a modified version of *url* - with no fragment identifier, and the fragment identifier as a separate - string. If there is no fragment identifier in *url*, return *url* unmodified - and an empty string. - - The return value is actually an instance of a subclass of :class:`tuple`. This - class has the following additional read-only convenience attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`url` | 0 | URL with no fragment | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 1 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - - See section :ref:`urlparse-result-object` for more information on the result - object. - - .. versionchanged:: 3.2 - Result is a structured object rather than a simple 2-tuple. - -.. _parsing-ascii-encoded-bytes: - -Parsing ASCII Encoded Bytes ---------------------------- - -The URL parsing functions were originally designed to operate on character -strings only. In practice, it is useful to be able to manipulate properly -quoted and encoded URLs as sequences of ASCII bytes. Accordingly, the -URL parsing functions in this module all operate on :class:`bytes` and -:class:`bytearray` objects in addition to :class:`str` objects. - -If :class:`str` data is passed in, the result will also contain only -:class:`str` data. If :class:`bytes` or :class:`bytearray` data is -passed in, the result will contain only :class:`bytes` data. - -Attempting to mix :class:`str` data with :class:`bytes` or -:class:`bytearray` in a single function call will result in a -:exc:`TypeError` being raised, while attempting to pass in non-ASCII -byte values will trigger :exc:`UnicodeDecodeError`. - -To support easier conversion of result objects between :class:`str` and -:class:`bytes`, all return values from URL parsing functions provide -either an :meth:`encode` method (when the result contains :class:`str` -data) or a :meth:`decode` method (when the result contains :class:`bytes` -data). The signatures of these methods match those of the corresponding -:class:`str` and :class:`bytes` methods (except that the default encoding -is ``'ascii'`` rather than ``'utf-8'``). Each produces a value of a -corresponding type that contains either :class:`bytes` data (for -:meth:`encode` methods) or :class:`str` data (for -:meth:`decode` methods). - -Applications that need to operate on potentially improperly quoted URLs -that may contain non-ASCII data will need to do their own decoding from -bytes to characters before invoking the URL parsing methods. - -The behaviour described in this section applies only to the URL parsing -functions. The URL quoting functions use their own rules when producing -or consuming byte sequences as detailed in the documentation of the -individual URL quoting functions. - -.. versionchanged:: 3.2 - URL parsing functions now accept ASCII encoded byte sequences - - -.. _urlparse-result-object: - -Structured Parse Results ------------------------- - -The result objects from the :func:`urlparse`, :func:`urlsplit` and -:func:`urldefrag` functions are subclasses of the :class:`tuple` type. -These subclasses add the attributes listed in the documentation for -those functions, the encoding and decoding support described in the -previous section, as well as an additional method: - -.. method:: urllib.parse.SplitResult.geturl() - - Return the re-combined version of the original URL as a string. This may - differ from the original URL in that the scheme may be normalized to lower - case and empty components may be dropped. Specifically, empty parameters, - queries, and fragment identifiers will be removed. - - For :func:`urldefrag` results, only empty fragment identifiers will be removed. - For :func:`urlsplit` and :func:`urlparse` results, all noted changes will be - made to the URL returned by this method. - - The result of this method remains unchanged if passed back through the original - parsing function: - - >>> from urllib.parse import urlsplit - >>> url = 'HTTP://www.Python.org/doc/#' - >>> r1 = urlsplit(url) - >>> r1.geturl() - 'http://www.Python.org/doc/' - >>> r2 = urlsplit(r1.geturl()) - >>> r2.geturl() - 'http://www.Python.org/doc/' - - -The following classes provide the implementations of the structured parse -results when operating on :class:`str` objects: - -.. class:: DefragResult(url, fragment) - - Concrete class for :func:`urldefrag` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`DefragResultBytes` - instance. - - .. versionadded:: 3.2 - -.. class:: ParseResult(scheme, netloc, path, params, query, fragment) - - Concrete class for :func:`urlparse` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`ParseResultBytes` - instance. - -.. class:: SplitResult(scheme, netloc, path, query, fragment) - - Concrete class for :func:`urlsplit` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`SplitResultBytes` - instance. - - -The following classes provide the implementations of the parse results when -operating on :class:`bytes` or :class:`bytearray` objects: - -.. class:: DefragResultBytes(url, fragment) - - Concrete class for :func:`urldefrag` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`DefragResult` - instance. - - .. versionadded:: 3.2 - -.. class:: ParseResultBytes(scheme, netloc, path, params, query, fragment) - - Concrete class for :func:`urlparse` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`ParseResult` - instance. - - .. versionadded:: 3.2 - -.. class:: SplitResultBytes(scheme, netloc, path, query, fragment) - - Concrete class for :func:`urlsplit` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`SplitResult` - instance. - - .. versionadded:: 3.2 - - -URL Quoting ------------ - -The URL quoting functions focus on taking program data and making it safe -for use as URL components by quoting special characters and appropriately -encoding non-ASCII text. They also support reversing these operations to -recreate the original data from the contents of a URL component if that -task isn't already covered by the URL parsing functions above. - -.. function:: quote(string, safe='/', encoding=None, errors=None) - - Replace special characters in *string* using the ``%xx`` escape. Letters, - digits, and the characters ``'_.-'`` are never quoted. By default, this - function is intended for quoting the path section of URL. The optional *safe* - parameter specifies additional ASCII characters that should not be quoted - --- its default value is ``'/'``. - - *string* may be either a :class:`str` or a :class:`bytes`. - - The optional *encoding* and *errors* parameters specify how to deal with - non-ASCII characters, as accepted by the :meth:`str.encode` method. - *encoding* defaults to ``'utf-8'``. - *errors* defaults to ``'strict'``, meaning unsupported characters raise a - :class:`UnicodeEncodeError`. - *encoding* and *errors* must not be supplied if *string* is a - :class:`bytes`, or a :class:`TypeError` is raised. - - Note that ``quote(string, safe, encoding, errors)`` is equivalent to - ``quote_from_bytes(string.encode(encoding, errors), safe)``. - - Example: ``quote('/El Niño/')`` yields ``'/El%20Ni%C3%B1o/'``. - - -.. function:: quote_plus(string, safe='', encoding=None, errors=None) - - Like :func:`quote`, but also replace spaces by plus signs, as required for - quoting HTML form values when building up a query string to go into a URL. - Plus signs in the original string are escaped unless they are included in - *safe*. It also does not have *safe* default to ``'/'``. - - Example: ``quote_plus('/El Niño/')`` yields ``'%2FEl+Ni%C3%B1o%2F'``. - - -.. function:: quote_from_bytes(bytes, safe='/') - - Like :func:`quote`, but accepts a :class:`bytes` object rather than a - :class:`str`, and does not perform string-to-bytes encoding. - - Example: ``quote_from_bytes(b'a&\xef')`` yields - ``'a%26%EF'``. - - -.. function:: unquote(string, encoding='utf-8', errors='replace') - - Replace ``%xx`` escapes by their single-character equivalent. - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - *string* must be a :class:`str`. - - *encoding* defaults to ``'utf-8'``. - *errors* defaults to ``'replace'``, meaning invalid sequences are replaced - by a placeholder character. - - Example: ``unquote('/El%20Ni%C3%B1o/')`` yields ``'/El Niño/'``. - - -.. function:: unquote_plus(string, encoding='utf-8', errors='replace') - - Like :func:`unquote`, but also replace plus signs by spaces, as required for - unquoting HTML form values. - - *string* must be a :class:`str`. - - Example: ``unquote_plus('/El+Ni%C3%B1o/')`` yields ``'/El Niño/'``. - - -.. function:: unquote_to_bytes(string) - - Replace ``%xx`` escapes by their single-octet equivalent, and return a - :class:`bytes` object. - - *string* may be either a :class:`str` or a :class:`bytes`. - - If it is a :class:`str`, unescaped non-ASCII characters in *string* - are encoded into UTF-8 bytes. - - Example: ``unquote_to_bytes('a%26%EF')`` yields ``b'a&\xef'``. - - -.. function:: urlencode(query, doseq=False, safe='', encoding=None, \ - errors=None, quote_via=quote_plus) - - Convert a mapping object or a sequence of two-element tuples, which may - contain :class:`str` or :class:`bytes` objects, to a percent-encoded ASCII - text string. If the resultant string is to be used as a *data* for POST - operation with the :func:`~urllib.request.urlopen` function, then - it should be encoded to bytes, otherwise it would result in a - :exc:`TypeError`. - - The resulting string is a series of ``key=value`` pairs separated by ``'&'`` - characters, where both *key* and *value* are quoted using the *quote_via* - function. By default, :func:`quote_plus` is used to quote the values, which - means spaces are quoted as a ``'+'`` character and '/' characters are - encoded as ``%2F``, which follows the standard for GET requests - (``application/x-www-form-urlencoded``). An alternate function that can be - passed as *quote_via* is :func:`quote`, which will encode spaces as ``%20`` - and not encode '/' characters. For maximum control of what is quoted, use - ``quote`` and specify a value for *safe*. - - When a sequence of two-element tuples is used as the *query* - argument, the first element of each tuple is a key and the second is a - value. The value element in itself can be a sequence and in that case, if - the optional parameter *doseq* is evaluates to ``True``, individual - ``key=value`` pairs separated by ``'&'`` are generated for each element of - the value sequence for the key. The order of parameters in the encoded - string will match the order of parameter tuples in the sequence. - - The *safe*, *encoding*, and *errors* parameters are passed down to - *quote_via* (the *encoding* and *errors* parameters are only passed - when a query element is a :class:`str`). - - To reverse this encoding process, :func:`parse_qs` and :func:`parse_qsl` are - provided in this module to parse query strings into Python data structures. - - Refer to :ref:`urllib examples ` to find out how urlencode - method can be used for generating query string for a URL or data for POST. - - .. versionchanged:: 3.2 - Query parameter supports bytes and string objects. - - .. versionadded:: 3.5 - *quote_via* parameter. - - -.. seealso:: - - `WHATWG`_ - URL Living standard - Working Group for the URL Standard that defines URLs, domains, IP addresses, the - application/x-www-form-urlencoded format, and their API. - - :rfc:`3986` - Uniform Resource Identifiers - This is the current standard (STD66). Any changes to urllib.parse module - should conform to this. Certain deviations could be observed, which are - mostly for backward compatibility purposes and for certain de-facto - parsing requirements as commonly observed in major browsers. - - :rfc:`2732` - Format for Literal IPv6 Addresses in URL's. - This specifies the parsing requirements of IPv6 URLs. - - :rfc:`2396` - Uniform Resource Identifiers (URI): Generic Syntax - Document describing the generic syntactic requirements for both Uniform Resource - Names (URNs) and Uniform Resource Locators (URLs). - - :rfc:`2368` - The mailto URL scheme. - Parsing requirements for mailto URL schemes. - - :rfc:`1808` - Relative Uniform Resource Locators - This Request For Comments includes the rules for joining an absolute and a - relative URL, including a fair number of "Abnormal Examples" which govern the - treatment of border cases. - - :rfc:`1738` - Uniform Resource Locators (URL) - This specifies the formal syntax and semantics of absolute URLs. - -.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/third_party/python/Doc/library/urllib.request.rst b/third_party/python/Doc/library/urllib.request.rst deleted file mode 100644 index 8af97f074..000000000 --- a/third_party/python/Doc/library/urllib.request.rst +++ /dev/null @@ -1,1565 +0,0 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= - -.. module:: urllib.request - :synopsis: Extensible library for opening URLs. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Senthil Kumaran - -**Source code:** :source:`Lib/urllib/request.py` - --------------- - -The :mod:`urllib.request` module defines functions and classes which help in -opening URLs (mostly HTTP) in a complex world --- basic and digest -authentication, redirections, cookies and more. - -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level HTTP client interface. - - -The :mod:`urllib.request` module defines the following functions: - - -.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False, context=None) - - Open the URL *url*, which can be either a string or a - :class:`Request` object. - - *data* must be an object specifying additional data to be sent to the - server, or ``None`` if no such data is needed. See :class:`Request` - for details. - - urllib.request module uses HTTP/1.1 and includes ``Connection:close`` header - in its HTTP requests. - - The optional *timeout* parameter specifies a timeout in seconds for - blocking operations like the connection attempt (if not specified, - the global default timeout setting will be used). This actually - only works for HTTP, HTTPS and FTP connections. - - If *context* is specified, it must be a :class:`ssl.SSLContext` instance - describing the various SSL options. See :class:`~http.client.HTTPSConnection` - for more details. - - The optional *cafile* and *capath* parameters specify a set of trusted - CA certificates for HTTPS requests. *cafile* should point to a single - file containing a bundle of CA certificates, whereas *capath* should - point to a directory of hashed certificate files. More information can - be found in :meth:`ssl.SSLContext.load_verify_locations`. - - The *cadefault* parameter is ignored. - - This function always returns an object which can work as a - :term:`context manager` and has methods such as - - * :meth:`~urllib.response.addinfourl.geturl` --- return the URL of the resource retrieved, - commonly used to determine if a redirect was followed - - * :meth:`~urllib.response.addinfourl.info` --- return the meta-information of the page, such as headers, - in the form of an :func:`email.message_from_string` instance (see - `Quick Reference to HTTP Headers `_) - - * :meth:`~urllib.response.addinfourl.getcode` -- return the HTTP status code of the response. - - For HTTP and HTTPS URLs, this function returns a - :class:`http.client.HTTPResponse` object slightly modified. In addition - to the three new methods above, the msg attribute contains the - same information as the :attr:`~http.client.HTTPResponse.reason` - attribute --- the reason phrase returned by server --- instead of - the response headers as it is specified in the documentation for - :class:`~http.client.HTTPResponse`. - - For FTP, file, and data URLs and requests explicitly handled by legacy - :class:`URLopener` and :class:`FancyURLopener` classes, this function - returns a :class:`urllib.response.addinfourl` object. - - Raises :exc:`~urllib.error.URLError` on protocol errors. - - Note that ``None`` may be returned if no handler handles the request (though - the default installed global :class:`OpenerDirector` uses - :class:`UnknownHandler` to ensure this never happens). - - In addition, if proxy settings are detected (for example, when a ``*_proxy`` - environment variable like :envvar:`http_proxy` is set), - :class:`ProxyHandler` is default installed and makes sure the requests are - handled through the proxy. - - The legacy ``urllib.urlopen`` function from Python 2.6 and earlier has been - discontinued; :func:`urllib.request.urlopen` corresponds to the old - ``urllib2.urlopen``. Proxy handling, which was done by passing a dictionary - parameter to ``urllib.urlopen``, can be obtained by using - :class:`ProxyHandler` objects. - - .. versionchanged:: 3.2 - *cafile* and *capath* were added. - - .. versionchanged:: 3.2 - HTTPS virtual hosts are now supported if possible (that is, if - :data:`ssl.HAS_SNI` is true). - - .. versionadded:: 3.2 - *data* can be an iterable object. - - .. versionchanged:: 3.3 - *cadefault* was added. - - .. versionchanged:: 3.4.3 - *context* was added. - - .. deprecated:: 3.6 - - *cafile*, *capath* and *cadefault* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - -.. function:: install_opener(opener) - - Install an :class:`OpenerDirector` instance as the default global opener. - Installing an opener is only necessary if you want urlopen to use that - opener; otherwise, simply call :meth:`OpenerDirector.open` instead of - :func:`~urllib.request.urlopen`. The code does not check for a real - :class:`OpenerDirector`, and any class with the appropriate interface will - work. - - -.. function:: build_opener([handler, ...]) - - Return an :class:`OpenerDirector` instance, which chains the handlers in the - order given. *handler*\s can be either instances of :class:`BaseHandler`, or - subclasses of :class:`BaseHandler` (in which case it must be possible to call - the constructor without any parameters). Instances of the following classes - will be in front of the *handler*\s, unless the *handler*\s contain them, - instances of them or subclasses of them: :class:`ProxyHandler` (if proxy - settings are detected), :class:`UnknownHandler`, :class:`HTTPHandler`, - :class:`HTTPDefaultErrorHandler`, :class:`HTTPRedirectHandler`, - :class:`FTPHandler`, :class:`FileHandler`, :class:`HTTPErrorProcessor`. - - If the Python installation has SSL support (i.e., if the :mod:`ssl` module - can be imported), :class:`HTTPSHandler` will also be added. - - A :class:`BaseHandler` subclass may also change its :attr:`handler_order` - attribute to modify its position in the handlers list. - - -.. function:: pathname2url(path) - - Convert the pathname *path* from the local syntax for a path to the form used in - the path component of a URL. This does not produce a complete URL. The return - value will already be quoted using the :func:`~urllib.parse.quote` function. - - -.. function:: url2pathname(path) - - Convert the path component *path* from a percent-encoded URL to the local syntax for a - path. This does not accept a complete URL. This function uses - :func:`~urllib.parse.unquote` to decode *path*. - -.. function:: getproxies() - - This helper function returns a dictionary of scheme to proxy server URL - mappings. It scans the environment for variables named ``_proxy``, - in a case insensitive approach, for all operating systems first, and when it - cannot find it, looks for proxy information from Mac OSX System - Configuration for Mac OS X and Windows Systems Registry for Windows. - If both lowercase and uppercase environment variables exist (and disagree), - lowercase is preferred. - - .. note:: - - If the environment variable ``REQUEST_METHOD`` is set, which usually - indicates your script is running in a CGI environment, the environment - variable ``HTTP_PROXY`` (uppercase ``_PROXY``) will be ignored. This is - because that variable can be injected by a client using the "Proxy:" HTTP - header. If you need to use an HTTP proxy in a CGI environment, either use - ``ProxyHandler`` explicitly, or make sure the variable name is in - lowercase (or at least the ``_proxy`` suffix). - - -The following classes are provided: - -.. class:: Request(url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None) - - This class is an abstraction of a URL request. - - *url* should be a string containing a valid URL. - - *data* must be an object specifying additional data to send to the - server, or ``None`` if no such data is needed. Currently HTTP - requests are the only ones that use *data*. The supported object - types include bytes, file-like objects, and iterables. If no - ``Content-Length`` nor ``Transfer-Encoding`` header field - has been provided, :class:`HTTPHandler` will set these headers according - to the type of *data*. ``Content-Length`` will be used to send - bytes objects, while ``Transfer-Encoding: chunked`` as specified in - :rfc:`7230`, Section 3.3.1 will be used to send files and other iterables. - - For an HTTP POST request method, *data* should be a buffer in the - standard :mimetype:`application/x-www-form-urlencoded` format. The - :func:`urllib.parse.urlencode` function takes a mapping or sequence - of 2-tuples and returns an ASCII string in this format. It should - be encoded to bytes before being used as the *data* parameter. - - *headers* should be a dictionary, and will be treated as if - :meth:`add_header` was called with each key and value as arguments. - This is often used to "spoof" the ``User-Agent`` header value, which is - used by a browser to identify itself -- some HTTP servers only - allow requests coming from common browsers as opposed to scripts. - For example, Mozilla Firefox may identify itself as ``"Mozilla/5.0 - (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"``, while - :mod:`urllib`'s default user agent string is - ``"Python-urllib/2.6"`` (on Python 2.6). - - An appropriate ``Content-Type`` header should be included if the *data* - argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will - be added as a default. - - The final two arguments are only of interest for correct handling - of third-party HTTP cookies: - - *origin_req_host* should be the request-host of the origin - transaction, as defined by :rfc:`2965`. It defaults to - ``http.cookiejar.request_host(self)``. This is the host name or IP - address of the original request that was initiated by the user. - For example, if the request is for an image in an HTML document, - this should be the request-host of the request for the page - containing the image. - - *unverifiable* should indicate whether the request is unverifiable, - as defined by :rfc:`2965`. It defaults to ``False``. An unverifiable - request is one whose URL the user did not have the option to - approve. For example, if the request is for an image in an HTML - document, and the user had no option to approve the automatic - fetching of the image, this should be true. - - *method* should be a string that indicates the HTTP request method that - will be used (e.g. ``'HEAD'``). If provided, its value is stored in the - :attr:`~Request.method` attribute and is used by :meth:`get_method()`. - The default is ``'GET'`` if *data* is ``None`` or ``'POST'`` otherwise. - Subclasses may indicate a different default method by setting the - :attr:`~Request.method` attribute in the class itself. - - .. note:: - The request will not work as expected if the data object is unable - to deliver its content more than once (e.g. a file or an iterable - that can produce the content only once) and the request is retried - for HTTP redirects or authentication. The *data* is sent to the - HTTP server right away after the headers. There is no support for - a 100-continue expectation in the library. - - .. versionchanged:: 3.3 - :attr:`Request.method` argument is added to the Request class. - - .. versionchanged:: 3.4 - Default :attr:`Request.method` may be indicated at the class level. - - .. versionchanged:: 3.6 - Do not raise an error if the ``Content-Length`` has not been - provided and *data* is neither ``None`` nor a bytes object. - Fall back to use chunked transfer encoding instead. - -.. class:: OpenerDirector() - - The :class:`OpenerDirector` class opens URLs via :class:`BaseHandler`\ s chained - together. It manages the chaining of handlers, and recovery from errors. - - -.. class:: BaseHandler() - - This is the base class for all registered handlers --- and handles only the - simple mechanics of registration. - - -.. class:: HTTPDefaultErrorHandler() - - A class which defines a default handler for HTTP error responses; all responses - are turned into :exc:`~urllib.error.HTTPError` exceptions. - - -.. class:: HTTPRedirectHandler() - - A class to handle redirections. - - -.. class:: HTTPCookieProcessor(cookiejar=None) - - A class to handle HTTP Cookies. - - -.. class:: ProxyHandler(proxies=None) - - Cause requests to go through a proxy. If *proxies* is given, it must be a - dictionary mapping protocol names to URLs of proxies. The default is to read - the list of proxies from the environment variables - ``_proxy``. If no proxy environment variables are set, then - in a Windows environment proxy settings are obtained from the registry's - Internet Settings section, and in a Mac OS X environment proxy information - is retrieved from the OS X System Configuration Framework. - - To disable autodetected proxy pass an empty dictionary. - - The :envvar:`no_proxy` environment variable can be used to specify hosts - which shouldn't be reached via proxy; if set, it should be a comma-separated - list of hostname suffixes, optionally with ``:port`` appended, for example - ``cern.ch,ncsa.uiuc.edu,some.host:8080``. - - .. note:: - - ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; - see the documentation on :func:`~urllib.request.getproxies`. - - -.. class:: HTTPPasswordMgr() - - Keep a database of ``(realm, uri) -> (user, password)`` mappings. - - -.. class:: HTTPPasswordMgrWithDefaultRealm() - - Keep a database of ``(realm, uri) -> (user, password)`` mappings. A realm of - ``None`` is considered a catch-all realm, which is searched if no other realm - fits. - - -.. class:: HTTPPasswordMgrWithPriorAuth() - - A variant of :class:`HTTPPasswordMgrWithDefaultRealm` that also has a - database of ``uri -> is_authenticated`` mappings. Can be used by a - BasicAuth handler to determine when to send authentication credentials - immediately instead of waiting for a ``401`` response first. - - .. versionadded:: 3.5 - - -.. class:: AbstractBasicAuthHandler(password_mgr=None) - - This is a mixin class that helps with HTTP authentication, both to the remote - host and to a proxy. *password_mgr*, if given, should be something that is - compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. If *passwd_mgr* also provides ``is_authenticated`` and - ``update_authenticated`` methods (see - :ref:`http-password-mgr-with-prior-auth`), then the handler will use the - ``is_authenticated`` result for a given URI to determine whether or not to - send authentication credentials with the request. If ``is_authenticated`` - returns ``True`` for the URI, credentials are sent. If ``is_authenticated`` - is ``False``, credentials are not sent, and then if a ``401`` response is - received the request is re-sent with the authentication credentials. If - authentication succeeds, ``update_authenticated`` is called to set - ``is_authenticated`` ``True`` for the URI, so that subsequent requests to - the URI or any of its super-URIs will automatically include the - authentication credentials. - - .. versionadded:: 3.5 - Added ``is_authenticated`` support. - - -.. class:: HTTPBasicAuthHandler(password_mgr=None) - - Handle authentication with the remote host. *password_mgr*, if given, should - be something that is compatible with :class:`HTTPPasswordMgr`; refer to - section :ref:`http-password-mgr` for information on the interface that must - be supported. HTTPBasicAuthHandler will raise a :exc:`ValueError` when - presented with a wrong Authentication scheme. - - -.. class:: ProxyBasicAuthHandler(password_mgr=None) - - Handle authentication with the proxy. *password_mgr*, if given, should be - something that is compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: AbstractDigestAuthHandler(password_mgr=None) - - This is a mixin class that helps with HTTP authentication, both to the remote - host and to a proxy. *password_mgr*, if given, should be something that is - compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: HTTPDigestAuthHandler(password_mgr=None) - - Handle authentication with the remote host. *password_mgr*, if given, should - be something that is compatible with :class:`HTTPPasswordMgr`; refer to - section :ref:`http-password-mgr` for information on the interface that must - be supported. When both Digest Authentication Handler and Basic - Authentication Handler are both added, Digest Authentication is always tried - first. If the Digest Authentication returns a 40x response again, it is sent - to Basic Authentication handler to Handle. This Handler method will raise a - :exc:`ValueError` when presented with an authentication scheme other than - Digest or Basic. - - .. versionchanged:: 3.3 - Raise :exc:`ValueError` on unsupported Authentication Scheme. - - - -.. class:: ProxyDigestAuthHandler(password_mgr=None) - - Handle authentication with the proxy. *password_mgr*, if given, should be - something that is compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: HTTPHandler() - - A class to handle opening of HTTP URLs. - - -.. class:: HTTPSHandler(debuglevel=0, context=None, check_hostname=None) - - A class to handle opening of HTTPS URLs. *context* and *check_hostname* - have the same meaning as in :class:`http.client.HTTPSConnection`. - - .. versionchanged:: 3.2 - *context* and *check_hostname* were added. - - -.. class:: FileHandler() - - Open local files. - -.. class:: DataHandler() - - Open data URLs. - - .. versionadded:: 3.4 - -.. class:: FTPHandler() - - Open FTP URLs. - - -.. class:: CacheFTPHandler() - - Open FTP URLs, keeping a cache of open FTP connections to minimize delays. - - -.. class:: UnknownHandler() - - A catch-all class to handle unknown URLs. - - -.. class:: HTTPErrorProcessor() - - Process HTTP error responses. - - -.. _request-objects: - -Request Objects ---------------- - -The following methods describe :class:`Request`'s public interface, -and so all may be overridden in subclasses. It also defines several -public attributes that can be used by clients to inspect the parsed -request. - -.. attribute:: Request.full_url - - The original URL passed to the constructor. - - .. versionchanged:: 3.4 - - Request.full_url is a property with setter, getter and a deleter. Getting - :attr:`~Request.full_url` returns the original request URL with the - fragment, if it was present. - -.. attribute:: Request.type - - The URI scheme. - -.. attribute:: Request.host - - The URI authority, typically a host, but may also contain a port - separated by a colon. - -.. attribute:: Request.origin_req_host - - The original host for the request, without port. - -.. attribute:: Request.selector - - The URI path. If the :class:`Request` uses a proxy, then selector - will be the full URL that is passed to the proxy. - -.. attribute:: Request.data - - The entity body for the request, or ``None`` if not specified. - - .. versionchanged:: 3.4 - Changing value of :attr:`Request.data` now deletes "Content-Length" - header if it was previously set or calculated. - -.. attribute:: Request.unverifiable - - boolean, indicates whether the request is unverifiable as defined - by :rfc:`2965`. - -.. attribute:: Request.method - - The HTTP request method to use. By default its value is :const:`None`, - which means that :meth:`~Request.get_method` will do its normal computation - of the method to be used. Its value can be set (thus overriding the default - computation in :meth:`~Request.get_method`) either by providing a default - value by setting it at the class level in a :class:`Request` subclass, or by - passing a value in to the :class:`Request` constructor via the *method* - argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - A default value can now be set in subclasses; previously it could only - be set via the constructor argument. - - -.. method:: Request.get_method() - - Return a string indicating the HTTP request method. If - :attr:`Request.method` is not ``None``, return its value, otherwise return - ``'GET'`` if :attr:`Request.data` is ``None``, or ``'POST'`` if it's not. - This is only meaningful for HTTP requests. - - .. versionchanged:: 3.3 - get_method now looks at the value of :attr:`Request.method`. - - -.. method:: Request.add_header(key, val) - - Add another header to the request. Headers are currently ignored by all - handlers except HTTP handlers, where they are added to the list of headers sent - to the server. Note that there cannot be more than one header with the same - name, and later calls will overwrite previous calls in case the *key* collides. - Currently, this is no loss of HTTP functionality, since all headers which have - meaning when used more than once have a (header-specific) way of gaining the - same functionality using only one header. - - -.. method:: Request.add_unredirected_header(key, header) - - Add a header that will not be added to a redirected request. - - -.. method:: Request.has_header(header) - - Return whether the instance has the named header (checks both regular and - unredirected). - - -.. method:: Request.remove_header(header) - - Remove named header from the request instance (both from regular and - unredirected headers). - - .. versionadded:: 3.4 - - -.. method:: Request.get_full_url() - - Return the URL given in the constructor. - - .. versionchanged:: 3.4 - - Returns :attr:`Request.full_url` - - -.. method:: Request.set_proxy(host, type) - - Prepare the request by connecting to a proxy server. The *host* and *type* will - replace those of the instance, and the instance's selector will be the original - URL given in the constructor. - - -.. method:: Request.get_header(header_name, default=None) - - Return the value of the given header. If the header is not present, return - the default value. - - -.. method:: Request.header_items() - - Return a list of tuples (header_name, header_value) of the Request headers. - -.. versionchanged:: 3.4 - The request methods add_data, has_data, get_data, get_type, get_host, - get_selector, get_origin_req_host and is_unverifiable that were deprecated - since 3.3 have been removed. - - -.. _opener-director-objects: - -OpenerDirector Objects ----------------------- - -:class:`OpenerDirector` instances have the following methods: - - -.. method:: OpenerDirector.add_handler(handler) - - *handler* should be an instance of :class:`BaseHandler`. The following methods - are searched, and added to the possible chains (note that HTTP errors are a - special case). - - * :meth:`protocol_open` --- signal that the handler knows how to open *protocol* - URLs. - - * :meth:`http_error_type` --- signal that the handler knows how to handle HTTP - errors with HTTP error code *type*. - - * :meth:`protocol_error` --- signal that the handler knows how to handle errors - from (non-\ ``http``) *protocol*. - - * :meth:`protocol_request` --- signal that the handler knows how to pre-process - *protocol* requests. - - * :meth:`protocol_response` --- signal that the handler knows how to - post-process *protocol* responses. - - -.. method:: OpenerDirector.open(url, data=None[, timeout]) - - Open the given *url* (which can be a request object or a string), optionally - passing the given *data*. Arguments, return values and exceptions raised are - the same as those of :func:`urlopen` (which simply calls the :meth:`open` - method on the currently installed global :class:`OpenerDirector`). The - optional *timeout* parameter specifies a timeout in seconds for blocking - operations like the connection attempt (if not specified, the global default - timeout setting will be used). The timeout feature actually works only for - HTTP, HTTPS and FTP connections). - - -.. method:: OpenerDirector.error(proto, *args) - - Handle an error of the given protocol. This will call the registered error - handlers for the given protocol with the given arguments (which are protocol - specific). The HTTP protocol is a special case which uses the HTTP response - code to determine the specific error handler; refer to the :meth:`http_error_\*` - methods of the handler classes. - - Return values and exceptions raised are the same as those of :func:`urlopen`. - -OpenerDirector objects open URLs in three stages: - -The order in which these methods are called within each stage is determined by -sorting the handler instances. - -#. Every handler with a method named like :meth:`protocol_request` has that - method called to pre-process the request. - -#. Handlers with a method named like :meth:`protocol_open` are called to handle - the request. This stage ends when a handler either returns a non-\ :const:`None` - value (ie. a response), or raises an exception (usually - :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. - - In fact, the above algorithm is first tried for methods named - :meth:`default_open`. If all such methods return :const:`None`, the algorithm - is repeated for methods named like :meth:`protocol_open`. If all such methods - return :const:`None`, the algorithm is repeated for methods named - :meth:`unknown_open`. - - Note that the implementation of these methods may involve calls of the parent - :class:`OpenerDirector` instance's :meth:`~OpenerDirector.open` and - :meth:`~OpenerDirector.error` methods. - -#. Every handler with a method named like :meth:`protocol_response` has that - method called to post-process the response. - - -.. _base-handler-objects: - -BaseHandler Objects -------------------- - -:class:`BaseHandler` objects provide a couple of methods that are directly -useful, and others that are meant to be used by derived classes. These are -intended for direct use: - - -.. method:: BaseHandler.add_parent(director) - - Add a director as parent. - - -.. method:: BaseHandler.close() - - Remove any parents. - -The following attribute and methods should only be used by classes derived from -:class:`BaseHandler`. - -.. note:: - - The convention has been adopted that subclasses defining - :meth:`protocol_request` or :meth:`protocol_response` methods are named - :class:`\*Processor`; all others are named :class:`\*Handler`. - - -.. attribute:: BaseHandler.parent - - A valid :class:`OpenerDirector`, which can be used to open using a different - protocol, or handle errors. - - -.. method:: BaseHandler.default_open(req) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to catch all URLs. - - This method, if implemented, will be called by the parent - :class:`OpenerDirector`. It should return a file-like object as described in - the return value of the :meth:`open` of :class:`OpenerDirector`, or ``None``. - It should raise :exc:`~urllib.error.URLError`, unless a truly exceptional - thing happens (for example, :exc:`MemoryError` should not be mapped to - :exc:`URLError`). - - This method will be called before any protocol-specific open method. - - -.. method:: BaseHandler.protocol_open(req) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to handle URLs with the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - Return values should be the same as for :meth:`default_open`. - - -.. method:: BaseHandler.unknown_open(req) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to catch all URLs with no specific registered handler to - open it. - - This method, if implemented, will be called by the :attr:`parent` - :class:`OpenerDirector`. Return values should be the same as for - :meth:`default_open`. - - -.. method:: BaseHandler.http_error_default(req, fp, code, msg, hdrs) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - override it if they intend to provide a catch-all for otherwise unhandled HTTP - errors. It will be called automatically by the :class:`OpenerDirector` getting - the error, and should not normally be called in other circumstances. - - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. - - Return values and exceptions raised should be the same as those of - :func:`urlopen`. - - -.. method:: BaseHandler.http_error_nnn(req, fp, code, msg, hdrs) - - *nnn* should be a three-digit HTTP error code. This method is also not defined - in :class:`BaseHandler`, but will be called, if it exists, on an instance of a - subclass, when an HTTP error with code *nnn* occurs. - - Subclasses should override this method to handle specific HTTP errors. - - Arguments, return values and exceptions raised should be the same as for - :meth:`http_error_default`. - - -.. method:: BaseHandler.protocol_request(req) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to pre-process requests of the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - *req* will be a :class:`Request` object. The return value should be a - :class:`Request` object. - - -.. method:: BaseHandler.protocol_response(req, response) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to post-process responses of the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - *req* will be a :class:`Request` object. *response* will be an object - implementing the same interface as the return value of :func:`urlopen`. The - return value should implement the same interface as the return value of - :func:`urlopen`. - - -.. _http-redirect-handler: - -HTTPRedirectHandler Objects ---------------------------- - -.. note:: - - Some HTTP redirections require action from this module's client code. If this - is the case, :exc:`~urllib.error.HTTPError` is raised. See :rfc:`2616` for - details of the precise meanings of the various redirection codes. - - An :class:`HTTPError` exception raised as a security consideration if the - HTTPRedirectHandler is presented with a redirected URL which is not an HTTP, - HTTPS or FTP URL. - - -.. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) - - Return a :class:`Request` or ``None`` in response to a redirect. This is called - by the default implementations of the :meth:`http_error_30\*` methods when a - redirection is received from the server. If a redirection should take place, - return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the - redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if - no other handler should try to handle this URL, or return ``None`` if you - can't but another handler might. - - .. note:: - - The default implementation of this method does not strictly follow :rfc:`2616`, - which says that 301 and 302 responses to ``POST`` requests must not be - automatically redirected without confirmation by the user. In reality, browsers - do allow automatic redirection of these responses, changing the POST to a - ``GET``, and the default implementation reproduces this behavior. - - -.. method:: HTTPRedirectHandler.http_error_301(req, fp, code, msg, hdrs) - - Redirect to the ``Location:`` or ``URI:`` URL. This method is called by the - parent :class:`OpenerDirector` when getting an HTTP 'moved permanently' response. - - -.. method:: HTTPRedirectHandler.http_error_302(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'found' response. - - -.. method:: HTTPRedirectHandler.http_error_303(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'see other' response. - - -.. method:: HTTPRedirectHandler.http_error_307(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'temporary redirect' - response. - - -.. _http-cookie-processor: - -HTTPCookieProcessor Objects ---------------------------- - -:class:`HTTPCookieProcessor` instances have one attribute: - -.. attribute:: HTTPCookieProcessor.cookiejar - - The :class:`http.cookiejar.CookieJar` in which cookies are stored. - - -.. _proxy-handler: - -ProxyHandler Objects --------------------- - - -.. method:: ProxyHandler.protocol_open(request) - :noindex: - - The :class:`ProxyHandler` will have a method :meth:`protocol_open` for every - *protocol* which has a proxy in the *proxies* dictionary given in the - constructor. The method will modify requests to go through the proxy, by - calling ``request.set_proxy()``, and call the next handler in the chain to - actually execute the protocol. - - -.. _http-password-mgr: - -HTTPPasswordMgr Objects ------------------------ - -These methods are available on :class:`HTTPPasswordMgr` and -:class:`HTTPPasswordMgrWithDefaultRealm` objects. - - -.. method:: HTTPPasswordMgr.add_password(realm, uri, user, passwd) - - *uri* can be either a single URI, or a sequence of URIs. *realm*, *user* and - *passwd* must be strings. This causes ``(user, passwd)`` to be used as - authentication tokens when authentication for *realm* and a super-URI of any of - the given URIs is given. - - -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) - - Get user/password for given realm and URI, if any. This method will return - ``(None, None)`` if there is no matching user/password. - - For :class:`HTTPPasswordMgrWithDefaultRealm` objects, the realm ``None`` will be - searched if the given *realm* has no matching user/password. - - -.. _http-password-mgr-with-prior-auth: - -HTTPPasswordMgrWithPriorAuth Objects ------------------------------------- - -This password manager extends :class:`HTTPPasswordMgrWithDefaultRealm` to support -tracking URIs for which authentication credentials should always be sent. - - -.. method:: HTTPPasswordMgrWithPriorAuth.add_password(realm, uri, user, \ - passwd, is_authenticated=False) - - *realm*, *uri*, *user*, *passwd* are as for - :meth:`HTTPPasswordMgr.add_password`. *is_authenticated* sets the initial - value of the ``is_authenticated`` flag for the given URI or list of URIs. - If *is_authenticated* is specified as ``True``, *realm* is ignored. - - -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) - - Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects - - -.. method:: HTTPPasswordMgrWithPriorAuth.update_authenticated(self, uri, \ - is_authenticated=False) - - Update the ``is_authenticated`` flag for the given *uri* or list - of URIs. - - -.. method:: HTTPPasswordMgrWithPriorAuth.is_authenticated(self, authuri) - - Returns the current state of the ``is_authenticated`` flag for - the given URI. - - -.. _abstract-basic-auth-handler: - -AbstractBasicAuthHandler Objects --------------------------------- - - -.. method:: AbstractBasicAuthHandler.http_error_auth_reqed(authreq, host, req, headers) - - Handle an authentication request by getting a user/password pair, and re-trying - the request. *authreq* should be the name of the header where the information - about the realm is included in the request, *host* specifies the URL and path to - authenticate for, *req* should be the (failed) :class:`Request` object, and - *headers* should be the error headers. - - *host* is either an authority (e.g. ``"python.org"``) or a URL containing an - authority component (e.g. ``"http://python.org/"``). In either case, the - authority must not contain a userinfo component (so, ``"python.org"`` and - ``"python.org:80"`` are fine, ``"joe:password@python.org"`` is not). - - -.. _http-basic-auth-handler: - -HTTPBasicAuthHandler Objects ----------------------------- - - -.. method:: HTTPBasicAuthHandler.http_error_401(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _proxy-basic-auth-handler: - -ProxyBasicAuthHandler Objects ------------------------------ - - -.. method:: ProxyBasicAuthHandler.http_error_407(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _abstract-digest-auth-handler: - -AbstractDigestAuthHandler Objects ---------------------------------- - - -.. method:: AbstractDigestAuthHandler.http_error_auth_reqed(authreq, host, req, headers) - - *authreq* should be the name of the header where the information about the realm - is included in the request, *host* should be the host to authenticate to, *req* - should be the (failed) :class:`Request` object, and *headers* should be the - error headers. - - -.. _http-digest-auth-handler: - -HTTPDigestAuthHandler Objects ------------------------------ - - -.. method:: HTTPDigestAuthHandler.http_error_401(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _proxy-digest-auth-handler: - -ProxyDigestAuthHandler Objects ------------------------------- - - -.. method:: ProxyDigestAuthHandler.http_error_407(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _http-handler-objects: - -HTTPHandler Objects -------------------- - - -.. method:: HTTPHandler.http_open(req) - - Send an HTTP request, which can be either GET or POST, depending on - ``req.has_data()``. - - -.. _https-handler-objects: - -HTTPSHandler Objects --------------------- - - -.. method:: HTTPSHandler.https_open(req) - - Send an HTTPS request, which can be either GET or POST, depending on - ``req.has_data()``. - - -.. _file-handler-objects: - -FileHandler Objects -------------------- - - -.. method:: FileHandler.file_open(req) - - Open the file locally, if there is no host name, or the host name is - ``'localhost'``. - - .. versionchanged:: 3.2 - This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. - - -.. _data-handler-objects: - -DataHandler Objects -------------------- - -.. method:: DataHandler.data_open(req) - - Read a data URL. This kind of URL contains the content encoded in the URL - itself. The data URL syntax is specified in :rfc:`2397`. This implementation - ignores white spaces in base64 encoded data URLs so the URL may be wrapped - in whatever source file it comes from. But even though some browsers don't - mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. - - -.. _ftp-handler-objects: - -FTPHandler Objects ------------------- - - -.. method:: FTPHandler.ftp_open(req) - - Open the FTP file indicated by *req*. The login is always done with empty - username and password. - - -.. _cacheftp-handler-objects: - -CacheFTPHandler Objects ------------------------ - -:class:`CacheFTPHandler` objects are :class:`FTPHandler` objects with the -following additional methods: - - -.. method:: CacheFTPHandler.setTimeout(t) - - Set timeout of connections to *t* seconds. - - -.. method:: CacheFTPHandler.setMaxConns(m) - - Set maximum number of cached connections to *m*. - - -.. _unknown-handler-objects: - -UnknownHandler Objects ----------------------- - - -.. method:: UnknownHandler.unknown_open() - - Raise a :exc:`~urllib.error.URLError` exception. - - -.. _http-error-processor-objects: - -HTTPErrorProcessor Objects --------------------------- - -.. method:: HTTPErrorProcessor.http_response(request, response) - - Process HTTP error responses. - - For 200 error codes, the response object is returned immediately. - - For non-200 error codes, this simply passes the job on to the - :meth:`protocol_error_code` handler methods, via :meth:`OpenerDirector.error`. - Eventually, :class:`HTTPDefaultErrorHandler` will raise an - :exc:`~urllib.error.HTTPError` if no other handler handles the error. - - -.. method:: HTTPErrorProcessor.https_response(request, response) - - Process HTTPS error responses. - - The behavior is same as :meth:`http_response`. - - -.. _urllib-request-examples: - -Examples --------- - -In addition to the examples below, more examples are given in -:ref:`urllib-howto`. - -This example gets the python.org main page and displays the first 300 bytes of -it. :: - - >>> import urllib.request - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(300)) - ... - b'\n\n\n\n\n\n - \n - Python Programming ' - -Note that urlopen returns a bytes object. This is because there is no way -for urlopen to automatically determine the encoding of the byte stream -it receives from the HTTP server. In general, a program will decode -the returned bytes object to string once it determines or guesses -the appropriate encoding. - -The following W3C document, https://www.w3.org/International/O-charset\ , lists -the various ways in which an (X)HTML or an XML document could have specified its -encoding information. - -As the python.org website uses *utf-8* encoding as specified in its meta tag, we -will use the same for decoding the bytes object. :: - - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(100).decode('utf-8')) - ... - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtm - -It is also possible to achieve the same result without using the -:term:`context manager` approach. :: - - >>> import urllib.request - >>> f = urllib.request.urlopen('http://www.python.org/') - >>> print(f.read(100).decode('utf-8')) - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtm - -In the following example, we are sending a data-stream to the stdin of a CGI -and reading the data it returns to us. Note that this example will only work -when the Python installation supports SSL. :: - - >>> import urllib.request - >>> req = urllib.request.Request(url='https://localhost/cgi-bin/test.cgi', - ... data=b'This data is passed to stdin of the CGI') - >>> with urllib.request.urlopen(req) as f: - ... print(f.read().decode('utf-8')) - ... - Got Data: "This data is passed to stdin of the CGI" - -The code for the sample CGI used in the above example is:: - - #!/usr/bin/env python - import sys - data = sys.stdin.read() - print('Content-type: text/plain\n\nGot Data: "%s"' % data) - -Here is an example of doing a ``PUT`` request using :class:`Request`:: - - import urllib.request - DATA = b'some data' - req = urllib.request.Request(url='http://localhost:8080', data=DATA,method='PUT') - with urllib.request.urlopen(req) as f: - pass - print(f.status) - print(f.reason) - -Use of Basic HTTP Authentication:: - - import urllib.request - # Create an OpenerDirector with support for Basic HTTP Authentication... - auth_handler = urllib.request.HTTPBasicAuthHandler() - auth_handler.add_password(realm='PDQ Application', - uri='https://mahler:8092/site-updates.py', - user='klem', - passwd='kadidd!ehopper') - opener = urllib.request.build_opener(auth_handler) - # ...and install it globally so it can be used with urlopen. - urllib.request.install_opener(opener) - urllib.request.urlopen('http://www.example.com/login.html') - -:func:`build_opener` provides many handlers by default, including a -:class:`ProxyHandler`. By default, :class:`ProxyHandler` uses the environment -variables named ``<scheme>_proxy``, where ``<scheme>`` is the URL scheme -involved. For example, the :envvar:`http_proxy` environment variable is read to -obtain the HTTP proxy's URL. - -This example replaces the default :class:`ProxyHandler` with one that uses -programmatically-supplied proxy URLs, and adds proxy authorization support with -:class:`ProxyBasicAuthHandler`. :: - - proxy_handler = urllib.request.ProxyHandler({'http': 'http://www.example.com:3128/'}) - proxy_auth_handler = urllib.request.ProxyBasicAuthHandler() - proxy_auth_handler.add_password('realm', 'host', 'username', 'password') - - opener = urllib.request.build_opener(proxy_handler, proxy_auth_handler) - # This time, rather than install the OpenerDirector, we use it directly: - opener.open('http://www.example.com/login.html') - -Adding HTTP headers: - -Use the *headers* argument to the :class:`Request` constructor, or:: - - import urllib.request - req = urllib.request.Request('http://www.example.com/') - req.add_header('Referer', 'http://www.python.org/') - # Customize the default User-Agent header value: - req.add_header('User-Agent', 'urllib-example/0.1 (Contact: . . .)') - r = urllib.request.urlopen(req) - -:class:`OpenerDirector` automatically adds a :mailheader:`User-Agent` header to -every :class:`Request`. To change this:: - - import urllib.request - opener = urllib.request.build_opener() - opener.addheaders = [('User-agent', 'Mozilla/5.0')] - opener.open('http://www.example.com/') - -Also, remember that a few standard headers (:mailheader:`Content-Length`, -:mailheader:`Content-Type` and :mailheader:`Host`) -are added when the :class:`Request` is passed to :func:`urlopen` (or -:meth:`OpenerDirector.open`). - -.. _urllib-examples: - -Here is an example session that uses the ``GET`` method to retrieve a URL -containing parameters:: - - >>> import urllib.request - >>> import urllib.parse - >>> params = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> url = "http://www.musi-cal.com/cgi-bin/query?%s" % params - >>> with urllib.request.urlopen(url) as f: - ... print(f.read().decode('utf-8')) - ... - -The following example uses the ``POST`` method instead. Note that params output -from urlencode is encoded to bytes before it is sent to urlopen as data:: - - >>> import urllib.request - >>> import urllib.parse - >>> data = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> data = data.encode('ascii') - >>> with urllib.request.urlopen("http://requestb.in/xrbl82xr", data) as f: - ... print(f.read().decode('utf-8')) - ... - -The following example uses an explicitly specified HTTP proxy, overriding -environment settings:: - - >>> import urllib.request - >>> proxies = {'http': 'http://proxy.example.com:8080/'} - >>> opener = urllib.request.FancyURLopener(proxies) - >>> with opener.open("http://www.python.org") as f: - ... f.read().decode('utf-8') - ... - -The following example uses no proxies at all, overriding environment settings:: - - >>> import urllib.request - >>> opener = urllib.request.FancyURLopener({}) - >>> with opener.open("http://www.python.org/") as f: - ... f.read().decode('utf-8') - ... - - -Legacy interface ----------------- - -The following functions and classes are ported from the Python 2 module -``urllib`` (as opposed to ``urllib2``). They might become deprecated at -some point in the future. - -.. function:: urlretrieve(url, filename=None, reporthook=None, data=None) - - Copy a network object denoted by a URL to a local file. If the URL - points to a local file, the object will not be copied unless filename is supplied. - Return a tuple ``(filename, headers)`` where *filename* is the - local file name under which the object can be found, and *headers* is whatever - the :meth:`info` method of the object returned by :func:`urlopen` returned (for - a remote object). Exceptions are the same as for :func:`urlopen`. - - The second argument, if present, specifies the file location to copy to (if - absent, the location will be a tempfile with a generated name). The third - argument, if present, is a callable that will be called once on - establishment of the network connection and once after each block read - thereafter. The callable will be passed three arguments; a count of blocks - transferred so far, a block size in bytes, and the total size of the file. The - third argument may be ``-1`` on older FTP servers which do not return a file - size in response to a retrieval request. - - The following example illustrates the most common usage scenario:: - - >>> import urllib.request - >>> local_filename, headers = urllib.request.urlretrieve('http://python.org/') - >>> html = open(local_filename) - >>> html.close() - - If the *url* uses the :file:`http:` scheme identifier, the optional *data* - argument may be given to specify a ``POST`` request (normally the request - type is ``GET``). The *data* argument must be a bytes object in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. - - :func:`urlretrieve` will raise :exc:`ContentTooShortError` when it detects that - the amount of data available was less than the expected amount (which is the - size reported by a *Content-Length* header). This can occur, for example, when - the download is interrupted. - - The *Content-Length* is treated as a lower bound: if there's more data to read, - urlretrieve reads more data, but if less data is available, it raises the - exception. - - You can still retrieve the downloaded data in this case, it is stored in the - :attr:`content` attribute of the exception instance. - - If no *Content-Length* header was supplied, urlretrieve can not check the size - of the data it has downloaded, and just returns it. In this case you just have - to assume that the download was successful. - -.. function:: urlcleanup() - - Cleans up temporary files that may have been left behind by previous - calls to :func:`urlretrieve`. - -.. class:: URLopener(proxies=None, **x509) - - .. deprecated:: 3.3 - - Base class for opening and reading URLs. Unless you need to support opening - objects using schemes other than :file:`http:`, :file:`ftp:`, or :file:`file:`, - you probably want to use :class:`FancyURLopener`. - - By default, the :class:`URLopener` class sends a :mailheader:`User-Agent` header - of ``urllib/VVV``, where *VVV* is the :mod:`urllib` version number. - Applications can define their own :mailheader:`User-Agent` header by subclassing - :class:`URLopener` or :class:`FancyURLopener` and setting the class attribute - :attr:`version` to an appropriate string value in the subclass definition. - - The optional *proxies* parameter should be a dictionary mapping scheme names to - proxy URLs, where an empty dictionary turns proxies off completely. Its default - value is ``None``, in which case environmental proxy settings will be used if - present, as discussed in the definition of :func:`urlopen`, above. - - Additional keyword parameters, collected in *x509*, may be used for - authentication of the client when using the :file:`https:` scheme. The keywords - *key_file* and *cert_file* are supported to provide an SSL key and certificate; - both are needed to support client authentication. - - :class:`URLopener` objects will raise an :exc:`OSError` exception if the server - returns an error code. - - .. method:: open(fullurl, data=None) - - Open *fullurl* using the appropriate protocol. This method sets up cache and - proxy information, then calls the appropriate open method with its input - arguments. If the scheme is not recognized, :meth:`open_unknown` is called. - The *data* argument has the same meaning as the *data* argument of - :func:`urlopen`. - - - .. method:: open_unknown(fullurl, data=None) - - Overridable interface to open unknown URL types. - - - .. method:: retrieve(url, filename=None, reporthook=None, data=None) - - Retrieves the contents of *url* and places it in *filename*. The return value - is a tuple consisting of a local filename and either an - :class:`email.message.Message` object containing the response headers (for remote - URLs) or ``None`` (for local URLs). The caller must then open and read the - contents of *filename*. If *filename* is not given and the URL refers to a - local file, the input filename is returned. If the URL is non-local and - *filename* is not given, the filename is the output of :func:`tempfile.mktemp` - with a suffix that matches the suffix of the last path component of the input - URL. If *reporthook* is given, it must be a function accepting three numeric - parameters: A chunk number, the maximum size chunks are read in and the total size of the download - (-1 if unknown). It will be called once at the start and after each chunk of data is read from the - network. *reporthook* is ignored for local URLs. - - If the *url* uses the :file:`http:` scheme identifier, the optional *data* - argument may be given to specify a ``POST`` request (normally the request type - is ``GET``). The *data* argument must in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. - - - .. attribute:: version - - Variable that specifies the user agent of the opener object. To get - :mod:`urllib` to tell servers that it is a particular user agent, set this in a - subclass as a class variable or in the constructor before calling the base - constructor. - - -.. class:: FancyURLopener(...) - - .. deprecated:: 3.3 - - :class:`FancyURLopener` subclasses :class:`URLopener` providing default handling - for the following HTTP response codes: 301, 302, 303, 307 and 401. For the 30x - response codes listed above, the :mailheader:`Location` header is used to fetch - the actual URL. For 401 response codes (authentication required), basic HTTP - authentication is performed. For the 30x response codes, recursion is bounded - by the value of the *maxtries* attribute, which defaults to 10. - - For all other response codes, the method :meth:`http_error_default` is called - which you can override in subclasses to handle the error appropriately. - - .. note:: - - According to the letter of :rfc:`2616`, 301 and 302 responses to POST requests - must not be automatically redirected without confirmation by the user. In - reality, browsers do allow automatic redirection of these responses, changing - the POST to a GET, and :mod:`urllib` reproduces this behaviour. - - The parameters to the constructor are the same as those for :class:`URLopener`. - - .. note:: - - When performing basic authentication, a :class:`FancyURLopener` instance calls - its :meth:`prompt_user_passwd` method. The default implementation asks the - users for the required information on the controlling terminal. A subclass may - override this method to support more appropriate behavior if needed. - - The :class:`FancyURLopener` class offers one additional method that should be - overloaded to provide the appropriate behavior: - - .. method:: prompt_user_passwd(host, realm) - - Return information needed to authenticate the user at the given host in the - specified security realm. The return value should be a tuple, ``(user, - password)``, which can be used for basic authentication. - - The implementation prompts for this information on the terminal; an application - should override this method to use an appropriate interaction model in the local - environment. - - -:mod:`urllib.request` Restrictions ----------------------------------- - - .. index:: - pair: HTTP; protocol - pair: FTP; protocol - -* Currently, only the following protocols are supported: HTTP (versions 0.9 and - 1.0), FTP, local files, and data URLs. - - .. versionchanged:: 3.4 Added support for data URLs. - -* The caching feature of :func:`urlretrieve` has been disabled until someone - finds the time to hack proper processing of Expiration time headers. - -* There should be a function to query whether a particular URL is in the cache. - -* For backward compatibility, if a URL appears to point to a local file but the - file can't be opened, the URL is re-interpreted using the FTP protocol. This - can sometimes cause confusing error messages. - -* The :func:`urlopen` and :func:`urlretrieve` functions can cause arbitrarily - long delays while waiting for a network connection to be set up. This means - that it is difficult to build an interactive Web client using these functions - without using threads. - - .. index:: - single: HTML - pair: HTTP; protocol - -* The data returned by :func:`urlopen` or :func:`urlretrieve` is the raw data - returned by the server. This may be binary data (such as an image), plain text - or (for example) HTML. The HTTP protocol provides type information in the reply - header, which can be inspected by looking at the :mailheader:`Content-Type` - header. If the returned data is HTML, you can use the module - :mod:`html.parser` to parse it. - - .. index:: single: FTP - -* The code handling the FTP protocol cannot differentiate between a file and a - directory. This can lead to unexpected behavior when attempting to read a URL - that points to a file that is not accessible. If the URL ends in a ``/``, it is - assumed to refer to a directory and will be handled accordingly. But if an - attempt to read a file leads to a 550 error (meaning the URL cannot be found or - is not accessible, often for permission reasons), then the path is treated as a - directory in order to handle the case when a directory is specified by a URL but - the trailing ``/`` has been left off. This can cause misleading results when - you try to fetch a file whose read permissions make it inaccessible; the FTP - code will try to read it, fail with a 550 error, and then perform a directory - listing for the unreadable file. If fine-grained control is needed, consider - using the :mod:`ftplib` module, subclassing :class:`FancyURLopener`, or changing - *_urlopener* to meet your needs. - - - -:mod:`urllib.response` --- Response classes used by urllib -========================================================== - -.. module:: urllib.response - :synopsis: Response classes used by urllib. - -The :mod:`urllib.response` module defines functions and classes which define a -minimal file like interface, including ``read()`` and ``readline()``. The -typical response object is an addinfourl instance, which defines an ``info()`` -method and that returns headers and a ``geturl()`` method that returns the url. -Functions defined by this module are used internally by the -:mod:`urllib.request` module. - diff --git a/third_party/python/Doc/library/urllib.robotparser.rst b/third_party/python/Doc/library/urllib.robotparser.rst deleted file mode 100644 index e3b90e673..000000000 --- a/third_party/python/Doc/library/urllib.robotparser.rst +++ /dev/null @@ -1,97 +0,0 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== - -.. module:: urllib.robotparser - :synopsis: Load a robots.txt file and answer questions about - fetchability of other URLs. - -.. sectionauthor:: Skip Montanaro <skip@pobox.com> - -**Source code:** :source:`Lib/urllib/robotparser.py` - -.. index:: - single: WWW - single: World Wide Web - single: URL - single: robots.txt - --------------- - -This module provides a single class, :class:`RobotFileParser`, which answers -questions about whether or not a particular user agent can fetch a URL on the -Web site that published the :file:`robots.txt` file. For more details on the -structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. - - -.. class:: RobotFileParser(url='') - - This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. - - .. method:: set_url(url) - - Sets the URL referring to a :file:`robots.txt` file. - - .. method:: read() - - Reads the :file:`robots.txt` URL and feeds it to the parser. - - .. method:: parse(lines) - - Parses the lines argument. - - .. method:: can_fetch(useragent, url) - - Returns ``True`` if the *useragent* is allowed to fetch the *url* - according to the rules contained in the parsed :file:`robots.txt` - file. - - .. method:: mtime() - - Returns the time the ``robots.txt`` file was last fetched. This is - useful for long-running web spiders that need to check for new - ``robots.txt`` files periodically. - - .. method:: modified() - - Sets the time the ``robots.txt`` file was last fetched to the current - time. - - .. method:: crawl_delay(useragent) - - Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` - for the *useragent* in question. If there is no such parameter or it - doesn't apply to the *useragent* specified or the ``robots.txt`` entry - for this parameter has invalid syntax, return ``None``. - - .. versionadded:: 3.6 - - .. method:: request_rate(useragent) - - Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. - If there is no such parameter or it doesn't apply to the *useragent* - specified or the ``robots.txt`` entry for this parameter has invalid - syntax, return ``None``. - - .. versionadded:: 3.6 - - -The following example demonstrates basic use of the :class:`RobotFileParser` -class:: - - >>> import urllib.robotparser - >>> rp = urllib.robotparser.RobotFileParser() - >>> rp.set_url("http://www.musi-cal.com/robots.txt") - >>> rp.read() - >>> rrate = rp.request_rate("*") - >>> rrate.requests - 3 - >>> rrate.seconds - 20 - >>> rp.crawl_delay("*") - 6 - >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") - False - >>> rp.can_fetch("*", "http://www.musi-cal.com/") - True diff --git a/third_party/python/Doc/library/urllib.rst b/third_party/python/Doc/library/urllib.rst deleted file mode 100644 index 624e16462..000000000 --- a/third_party/python/Doc/library/urllib.rst +++ /dev/null @@ -1,15 +0,0 @@ -:mod:`urllib` --- URL handling modules -====================================== - -.. module:: urllib - -**Source code:** :source:`Lib/urllib/` - --------------- - -``urllib`` is a package that collects several modules for working with URLs: - -* :mod:`urllib.request` for opening and reading URLs -* :mod:`urllib.error` containing the exceptions raised by :mod:`urllib.request` -* :mod:`urllib.parse` for parsing URLs -* :mod:`urllib.robotparser` for parsing ``robots.txt`` files diff --git a/third_party/python/Doc/library/uu.rst b/third_party/python/Doc/library/uu.rst deleted file mode 100644 index 33fb36d0b..000000000 --- a/third_party/python/Doc/library/uu.rst +++ /dev/null @@ -1,62 +0,0 @@ -:mod:`uu` --- Encode and decode uuencode files -============================================== - -.. module:: uu - :synopsis: Encode and decode files in uuencode format. - -.. moduleauthor:: Lance Ellinghouse - -**Source code:** :source:`Lib/uu.py` - --------------- - -This module encodes and decodes files in uuencode format, allowing arbitrary -binary data to be transferred over ASCII-only connections. Wherever a file -argument is expected, the methods accept a file-like object. For backwards -compatibility, a string containing a pathname is also accepted, and the -corresponding file will be opened for reading and writing; the pathname ``'-'`` -is understood to mean the standard input or output. However, this interface is -deprecated; it's better for the caller to open the file itself, and be sure -that, when required, the mode is ``'rb'`` or ``'wb'`` on Windows. - -.. index:: - single: Jansen, Jack - single: Ellinghouse, Lance - -This code was contributed by Lance Ellinghouse, and modified by Jack Jansen. - -The :mod:`uu` module defines the following functions: - - -.. function:: encode(in_file, out_file, name=None, mode=None) - - Uuencode file *in_file* into file *out_file*. The uuencoded file will have - the header specifying *name* and *mode* as the defaults for the results of - decoding the file. The default defaults are taken from *in_file*, or ``'-'`` - and ``0o666`` respectively. - - -.. function:: decode(in_file, out_file=None, mode=None, quiet=False) - - This call decodes uuencoded file *in_file* placing the result on file - *out_file*. If *out_file* is a pathname, *mode* is used to set the permission - bits if the file must be created. Defaults for *out_file* and *mode* are taken - from the uuencode header. However, if the file specified in the header already - exists, a :exc:`uu.Error` is raised. - - :func:`decode` may print a warning to standard error if the input was produced - by an incorrect uuencoder and Python could recover from that error. Setting - *quiet* to a true value silences this warning. - - -.. exception:: Error() - - Subclass of :exc:`Exception`, this can be raised by :func:`uu.decode` under - various situations, such as described above, but also including a badly - formatted header, or truncated input file. - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. diff --git a/third_party/python/Doc/library/uuid.rst b/third_party/python/Doc/library/uuid.rst deleted file mode 100644 index b3193d744..000000000 --- a/third_party/python/Doc/library/uuid.rst +++ /dev/null @@ -1,266 +0,0 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== - -.. module:: uuid - :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 -.. moduleauthor:: Ka-Ping Yee <ping@zesty.ca> -.. sectionauthor:: George Yoshida <quiver@users.sourceforge.net> - -**Source code:** :source:`Lib/uuid.py` - --------------- - -This module provides immutable :class:`UUID` objects (the :class:`UUID` class) -and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5` for -generating version 1, 3, 4, and 5 UUIDs as specified in :rfc:`4122`. - -If all you want is a unique ID, you should probably call :func:`uuid1` or -:func:`uuid4`. Note that :func:`uuid1` may compromise privacy since it creates -a UUID containing the computer's network address. :func:`uuid4` creates a -random UUID. - - -.. class:: UUID(hex=None, bytes=None, bytes_le=None, fields=None, int=None, version=None) - - Create a UUID from either a string of 32 hexadecimal digits, a string of 16 - bytes in big-endian order as the *bytes* argument, a string of 16 bytes in - little-endian order as the *bytes_le* argument, a tuple of six integers - (32-bit *time_low*, 16-bit *time_mid*, 16-bit *time_hi_version*, - 8-bit *clock_seq_hi_variant*, 8-bit *clock_seq_low*, 48-bit *node*) as the - *fields* argument, or a single 128-bit integer as the *int* argument. - When a string of hex digits is given, curly braces, hyphens, - and a URN prefix are all optional. For example, these - expressions all yield the same UUID:: - - UUID('{12345678-1234-5678-1234-567812345678}') - UUID('12345678123456781234567812345678') - UUID('urn:uuid:12345678-1234-5678-1234-567812345678') - UUID(bytes=b'\x12\x34\x56\x78'*4) - UUID(bytes_le=b'\x78\x56\x34\x12\x34\x12\x78\x56' + - b'\x12\x34\x56\x78\x12\x34\x56\x78') - UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)) - UUID(int=0x12345678123456781234567812345678) - - Exactly one of *hex*, *bytes*, *bytes_le*, *fields*, or *int* must be given. - The *version* argument is optional; if given, the resulting UUID will have its - variant and version number set according to :rfc:`4122`, overriding bits in the - given *hex*, *bytes*, *bytes_le*, *fields*, or *int*. - - Comparison of UUID objects are made by way of comparing their - :attr:`UUID.int` attributes. Comparison with a non-UUID object - raises a :exc:`TypeError`. - - ``str(uuid)`` returns a string in the form - ``12345678-1234-5678-1234-567812345678`` where the 32 hexadecimal digits - represent the UUID. - -:class:`UUID` instances have these read-only attributes: - -.. attribute:: UUID.bytes - - The UUID as a 16-byte string (containing the six integer fields in big-endian - byte order). - - -.. attribute:: UUID.bytes_le - - The UUID as a 16-byte string (with *time_low*, *time_mid*, and *time_hi_version* - in little-endian byte order). - - -.. attribute:: UUID.fields - - A tuple of the six integer fields of the UUID, which are also available as six - individual attributes and two derived attributes: - - +------------------------------+-------------------------------+ - | Field | Meaning | - +==============================+===============================+ - | :attr:`time_low` | the first 32 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_mid` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time_hi_version` | the next 16 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_hi_variant` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`clock_seq_low` | the next 8 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`node` | the last 48 bits of the UUID | - +------------------------------+-------------------------------+ - | :attr:`time` | the 60-bit timestamp | - +------------------------------+-------------------------------+ - | :attr:`clock_seq` | the 14-bit sequence number | - +------------------------------+-------------------------------+ - - -.. attribute:: UUID.hex - - The UUID as a 32-character hexadecimal string. - - -.. attribute:: UUID.int - - The UUID as a 128-bit integer. - - -.. attribute:: UUID.urn - - The UUID as a URN as specified in :rfc:`4122`. - - -.. attribute:: UUID.variant - - The UUID variant, which determines the internal layout of the UUID. This will be - one of the constants :const:`RESERVED_NCS`, :const:`RFC_4122`, - :const:`RESERVED_MICROSOFT`, or :const:`RESERVED_FUTURE`. - - -.. attribute:: UUID.version - - The UUID version number (1 through 5, meaningful only when the variant is - :const:`RFC_4122`). - -The :mod:`uuid` module defines the following functions: - - -.. function:: getnode() - - Get the hardware address as a 48-bit positive integer. The first time this - runs, it may launch a separate program, which could be quite slow. If all - attempts to obtain the hardware address fail, we choose a random 48-bit number - with its eighth bit set to 1 as recommended in :rfc:`4122`. "Hardware address" - means the MAC address of a network interface, and on a machine with multiple - network interfaces the MAC address of any one of them may be returned. - -.. index:: single: getnode - - -.. function:: uuid1(node=None, clock_seq=None) - - Generate a UUID from a host ID, sequence number, and the current time. If *node* - is not given, :func:`getnode` is used to obtain the hardware address. If - *clock_seq* is given, it is used as the sequence number; otherwise a random - 14-bit sequence number is chosen. - -.. index:: single: uuid1 - - -.. function:: uuid3(namespace, name) - - Generate a UUID based on the MD5 hash of a namespace identifier (which is a - UUID) and a name (which is a string). - -.. index:: single: uuid3 - - -.. function:: uuid4() - - Generate a random UUID. - -.. index:: single: uuid4 - - -.. function:: uuid5(namespace, name) - - Generate a UUID based on the SHA-1 hash of a namespace identifier (which is a - UUID) and a name (which is a string). - -.. index:: single: uuid5 - -The :mod:`uuid` module defines the following namespace identifiers for use with -:func:`uuid3` or :func:`uuid5`. - - -.. data:: NAMESPACE_DNS - - When this namespace is specified, the *name* string is a fully-qualified domain - name. - - -.. data:: NAMESPACE_URL - - When this namespace is specified, the *name* string is a URL. - - -.. data:: NAMESPACE_OID - - When this namespace is specified, the *name* string is an ISO OID. - - -.. data:: NAMESPACE_X500 - - When this namespace is specified, the *name* string is an X.500 DN in DER or a - text output format. - -The :mod:`uuid` module defines the following constants for the possible values -of the :attr:`variant` attribute: - - -.. data:: RESERVED_NCS - - Reserved for NCS compatibility. - - -.. data:: RFC_4122 - - Specifies the UUID layout given in :rfc:`4122`. - - -.. data:: RESERVED_MICROSOFT - - Reserved for Microsoft compatibility. - - -.. data:: RESERVED_FUTURE - - Reserved for future definition. - - -.. seealso:: - - :rfc:`4122` - A Universally Unique IDentifier (UUID) URN Namespace - This specification defines a Uniform Resource Name namespace for UUIDs, the - internal format of UUIDs, and methods of generating UUIDs. - - -.. _uuid-example: - -Example -------- - -Here are some examples of typical usage of the :mod:`uuid` module:: - - >>> import uuid - - >>> # make a UUID based on the host ID and current time - >>> uuid.uuid1() - UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') - - >>> # make a UUID using an MD5 hash of a namespace UUID and a name - >>> uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') - UUID('6fa459ea-ee8a-3ca4-894e-db77e160355e') - - >>> # make a random UUID - >>> uuid.uuid4() - UUID('16fd2706-8baf-433b-82eb-8c7fada847da') - - >>> # make a UUID using a SHA-1 hash of a namespace UUID and a name - >>> uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') - UUID('886313e1-3b8a-5372-9b90-0c9aee199e5d') - - >>> # make a UUID from a string of hex digits (braces and hyphens ignored) - >>> x = uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}') - - >>> # convert a UUID to a string of hex digits in standard form - >>> str(x) - '00010203-0405-0607-0809-0a0b0c0d0e0f' - - >>> # get the raw 16 bytes of the UUID - >>> x.bytes - b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f' - - >>> # make a UUID from a 16-byte string - >>> uuid.UUID(bytes=x.bytes) - UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') - diff --git a/third_party/python/Doc/library/venv.rst b/third_party/python/Doc/library/venv.rst deleted file mode 100644 index c50231e53..000000000 --- a/third_party/python/Doc/library/venv.rst +++ /dev/null @@ -1,460 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. - -.. moduleauthor:: Vinay Sajip <vinay_sajip@yahoo.co.uk> -.. sectionauthor:: Vinay Sajip <vinay_sajip@yahoo.co.uk> - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv/` - -.. index:: pair: Environments; virtual - --------------- - -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (allowing -creation of environments with various Python versions) and can have its own -independent set of installed Python packages in its site directories. - -See :pep:`405` for more information about Python virtual environments. - -.. note:: - The ``pyvenv`` script has been deprecated as of Python 3.6 in favor of using - ``python3 -m venv`` to help prevent any potential confusion as to which - Python interpreter a virtual environment will be based on. - - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - - -.. _venv-def: - -.. note:: A virtual environment is a Python environment such that the Python - interpreter, libraries and scripts installed into it are isolated from those - installed in other virtual environments, and (by default) any libraries - installed in a "system" Python, i.e., one which is installed as part of your - operating system. - - A virtual environment is a directory tree which contains Python executable - files and other files which indicate that it is a virtual environment. - - Common installation tools such as ``Setuptools`` and ``pip`` work as - expected with virtual environments. In other words, when a virtual - environment is active, they install Python packages into the virtual - environment without needing to be told to do so explicitly. - - When a virtual environment is active (i.e., the virtual environment's Python - interpreter is running), the attributes :attr:`sys.prefix` and - :attr:`sys.exec_prefix` point to the base directory of the virtual - environment, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-virtual environment Python - installation which was used to create the virtual environment. If a virtual - environment is not active, then :attr:`sys.prefix` is the same as - :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as - :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment - Python installation). - - When a virtual environment is active, any options that change the - installation path will be ignored from all distutils configuration files to - prevent projects being inadvertently installed outside of the virtual - environment. - - When working in a command shell, users can make a virtual environment active - by running an ``activate`` script in the virtual environment's executables - directory (the precise filename is shell-dependent), which prepends the - virtual environment's directory for executables to the ``PATH`` environment - variable for the running shell. There should be no need in other - circumstances to activate a virtual environment—scripts installed into - virtual environments have a "shebang" line which points to the virtual - environment's Python interpreter. This means that the script will run with - that interpreter regardless of the value of ``PATH``. On Windows, "shebang" - line processing is supported if you have the Python Launcher for Windows - installed (this was added to Python in 3.3 - see :pep:`397` for more - details). Thus, double-clicking an installed script in a Windows Explorer - window should run the script with the correct interpreter without there - needing to be any reference to its virtual environment in ``PATH``. - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False, \ - prompt=None) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary (and any necessary DLLs or other binaries, - e.g. ``pythonw.exe``), rather than copying. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - * ``prompt`` -- a String to be used after virtual environment is activated - (defaults to ``None`` which means directory name of the environment would - be used). - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionadded:: 3.6 - Added the ``prompt`` parameter - - - Creators of third-party virtual environment tools will be free to use the - provided ``EnvBuilder`` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - This method takes as required argument the path (absolute or relative to - the current directory) of the target directory which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the ``EnvBuilder`` class illustrates the hooks - available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary directories, and - returns a context object. This is just a holder for attributes (such as - paths), for use by the other methods. The directories are allowed to - exist already, as long as either ``clear`` or ``upgrade`` were - specified to allow operating on an existing environment directory. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy of the Python executable (and, under Windows, DLLs) in - the environment. On a POSIX system, if a specific executable - ``python3.x`` was used, symlinks to ``python`` and ``python3`` will be - created pointing to that executable, unless files with those names - already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created virtual environment:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created virtual environment. - - :param nodist: If True, setuptools and pip are not installed into the - created virtual environment. - :param nopip: If True, pip is not installed into the created - virtual environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - virtual environment being created. - - :param context: The information for the virtual environment - creation request being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the virtual environment's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the virtual environment - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -<https://gist.github.com/4673395>`_. diff --git a/third_party/python/Doc/library/warnings.rst b/third_party/python/Doc/library/warnings.rst deleted file mode 100644 index f67f4bc24..000000000 --- a/third_party/python/Doc/library/warnings.rst +++ /dev/null @@ -1,425 +0,0 @@ -:mod:`warnings` --- Warning control -=================================== - -.. module:: warnings - :synopsis: Issue warning messages and control their disposition. - -**Source code:** :source:`Lib/warnings.py` - -.. index:: single: warnings - --------------- - -Warning messages are typically issued in situations where it is useful to alert -the user of some condition in a program, where that condition (normally) doesn't -warrant raising an exception and terminating the program. For example, one -might want to issue a warning when a program uses an obsolete module. - -Python programmers issue warnings by calling the :func:`warn` function defined -in this module. (C programmers use :c:func:`PyErr_WarnEx`; see -:ref:`exceptionhandling` for details). - -Warning messages are normally written to ``sys.stderr``, but their disposition -can be changed flexibly, from ignoring all warnings to turning them into -exceptions. The disposition of warnings can vary based on the warning category -(see below), the text of the warning message, and the source location where it -is issued. Repetitions of a particular warning for the same source location are -typically suppressed. - -There are two stages in warning control: first, each time a warning is issued, a -determination is made whether a message should be issued or not; next, if a -message is to be issued, it is formatted and printed using a user-settable hook. - -The determination whether to issue a warning message is controlled by the -warning filter, which is a sequence of matching rules and actions. Rules can be -added to the filter by calling :func:`filterwarnings` and reset to its default -state by calling :func:`resetwarnings`. - -The printing of warning messages is done by calling :func:`showwarning`, which -may be overridden; the default implementation of this function formats the -message by calling :func:`formatwarning`, which is also available for use by -custom implementations. - -.. seealso:: - :func:`logging.captureWarnings` allows you to handle all warnings with - the standard logging infrastructure. - - -.. _warning-categories: - -Warning Categories ------------------- - -There are a number of built-in exceptions that represent warning categories. -This categorization is useful to be able to filter out groups of warnings. The -following warnings category classes are currently defined: - -.. tabularcolumns:: |l|p{0.6\linewidth}| - -+----------------------------------+-----------------------------------------------+ -| Class | Description | -+==================================+===============================================+ -| :exc:`Warning` | This is the base class of all warning | -| | category classes. It is a subclass of | -| | :exc:`Exception`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`UserWarning` | The default category for :func:`warn`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`DeprecationWarning` | Base category for warnings about deprecated | -| | features (ignored by default). | -+----------------------------------+-----------------------------------------------+ -| :exc:`SyntaxWarning` | Base category for warnings about dubious | -| | syntactic features. | -+----------------------------------+-----------------------------------------------+ -| :exc:`RuntimeWarning` | Base category for warnings about dubious | -| | runtime features. | -+----------------------------------+-----------------------------------------------+ -| :exc:`FutureWarning` | Base category for warnings about constructs | -| | that will change semantically in the future. | -+----------------------------------+-----------------------------------------------+ -| :exc:`PendingDeprecationWarning` | Base category for warnings about features | -| | that will be deprecated in the future | -| | (ignored by default). | -+----------------------------------+-----------------------------------------------+ -| :exc:`ImportWarning` | Base category for warnings triggered during | -| | the process of importing a module (ignored by | -| | default). | -+----------------------------------+-----------------------------------------------+ -| :exc:`UnicodeWarning` | Base category for warnings related to | -| | Unicode. | -+----------------------------------+-----------------------------------------------+ -| :exc:`BytesWarning` | Base category for warnings related to | -| | :class:`bytes` and :class:`bytearray`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`ResourceWarning` | Base category for warnings related to | -| | resource usage. | -+----------------------------------+-----------------------------------------------+ - - -While these are technically built-in exceptions, they are documented here, -because conceptually they belong to the warnings mechanism. - -User code can define additional warning categories by subclassing one of the -standard warning categories. A warning category must always be a subclass of -the :exc:`Warning` class. - - -.. _warning-filter: - -The Warnings Filter -------------------- - -The warnings filter controls whether warnings are ignored, displayed, or turned -into errors (raising an exception). - -Conceptually, the warnings filter maintains an ordered list of filter -specifications; any specific warning is matched against each filter -specification in the list in turn until a match is found; the match determines -the disposition of the match. Each entry is a tuple of the form (*action*, -*message*, *category*, *module*, *lineno*), where: - -* *action* is one of the following strings: - - +---------------+----------------------------------------------+ - | Value | Disposition | - +===============+==============================================+ - | ``"error"`` | turn matching warnings into exceptions | - +---------------+----------------------------------------------+ - | ``"ignore"`` | never print matching warnings | - +---------------+----------------------------------------------+ - | ``"always"`` | always print matching warnings | - +---------------+----------------------------------------------+ - | ``"default"`` | print the first occurrence of matching | - | | warnings for each location where the warning | - | | is issued | - +---------------+----------------------------------------------+ - | ``"module"`` | print the first occurrence of matching | - | | warnings for each module where the warning | - | | is issued | - +---------------+----------------------------------------------+ - | ``"once"`` | print only the first occurrence of matching | - | | warnings, regardless of location | - +---------------+----------------------------------------------+ - -* *message* is a string containing a regular expression that the start of - the warning message must match. The expression is compiled to always be - case-insensitive. - -* *category* is a class (a subclass of :exc:`Warning`) of which the warning - category must be a subclass in order to match. - -* *module* is a string containing a regular expression that the module name must - match. The expression is compiled to be case-sensitive. - -* *lineno* is an integer that the line number where the warning occurred must - match, or ``0`` to match all line numbers. - -Since the :exc:`Warning` class is derived from the built-in :exc:`Exception` -class, to turn a warning into an error we simply raise ``category(message)``. - -The warnings filter is initialized by :option:`-W` options passed to the Python -interpreter command line. The interpreter saves the arguments for all -:option:`-W` options without interpretation in ``sys.warnoptions``; the -:mod:`warnings` module parses these when it is first imported (invalid options -are ignored, after printing a message to ``sys.stderr``). - - -Default Warning Filters -~~~~~~~~~~~~~~~~~~~~~~~ - -By default, Python installs several warning filters, which can be overridden by -the command-line options passed to :option:`-W` and calls to -:func:`filterwarnings`. - -* :exc:`DeprecationWarning` and :exc:`PendingDeprecationWarning`, and - :exc:`ImportWarning` are ignored. - -* :exc:`BytesWarning` is ignored unless the :option:`-b` option is given once or - twice; in this case this warning is either printed (``-b``) or turned into an - exception (``-bb``). - -* :exc:`ResourceWarning` is ignored unless Python was built in debug mode. - -.. versionchanged:: 3.2 - :exc:`DeprecationWarning` is now ignored by default in addition to - :exc:`PendingDeprecationWarning`. - - -.. _warning-suppress: - -Temporarily Suppressing Warnings --------------------------------- - -If you are using code that you know will raise a warning, such as a deprecated -function, but do not want to see the warning, then it is possible to suppress -the warning using the :class:`catch_warnings` context manager:: - - import warnings - - def fxn(): - warnings.warn("deprecated", DeprecationWarning) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fxn() - -While within the context manager all warnings will simply be ignored. This -allows you to use known-deprecated code without having to see the warning while -not suppressing the warning for other code that might not be aware of its use -of deprecated code. Note: this can only be guaranteed in a single-threaded -application. If two or more threads use the :class:`catch_warnings` context -manager at the same time, the behavior is undefined. - - - -.. _warning-testing: - -Testing Warnings ----------------- - -To test warnings raised by code, use the :class:`catch_warnings` context -manager. With it you can temporarily mutate the warnings filter to facilitate -your testing. For instance, do the following to capture all raised warnings to -check:: - - import warnings - - def fxn(): - warnings.warn("deprecated", DeprecationWarning) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - # Trigger a warning. - fxn() - # Verify some things - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated" in str(w[-1].message) - -One can also cause all warnings to be exceptions by using ``error`` instead of -``always``. One thing to be aware of is that if a warning has already been -raised because of a ``once``/``default`` rule, then no matter what filters are -set the warning will not be seen again unless the warnings registry related to -the warning has been cleared. - -Once the context manager exits, the warnings filter is restored to its state -when the context was entered. This prevents tests from changing the warnings -filter in unexpected ways between tests and leading to indeterminate test -results. The :func:`showwarning` function in the module is also restored to -its original value. Note: this can only be guaranteed in a single-threaded -application. If two or more threads use the :class:`catch_warnings` context -manager at the same time, the behavior is undefined. - -When testing multiple operations that raise the same kind of warning, it -is important to test them in a manner that confirms each operation is raising -a new warning (e.g. set warnings to be raised as exceptions and check the -operations raise exceptions, check that the length of the warning list -continues to increase after each operation, or else delete the previous -entries from the warnings list before each new operation). - - -.. _warning-ignored: - -Updating Code For New Versions of Python ----------------------------------------- - -Warnings that are only of interest to the developer are ignored by default. As -such you should make sure to test your code with typically ignored warnings -made visible. You can do this from the command-line by passing :option:`-Wd <-W>` -to the interpreter (this is shorthand for :option:`!-W default`). This enables -default handling for all warnings, including those that are ignored by default. -To change what action is taken for encountered warnings you simply change what -argument is passed to :option:`-W`, e.g. :option:`!-W error`. See the -:option:`-W` flag for more details on what is possible. - -To programmatically do the same as :option:`!-Wd`, use:: - - warnings.simplefilter('default') - -Make sure to execute this code as soon as possible. This prevents the -registering of what warnings have been raised from unexpectedly influencing how -future warnings are treated. - -Having certain warnings ignored by default is done to prevent a user from -seeing warnings that are only of interest to the developer. As you do not -necessarily have control over what interpreter a user uses to run their code, -it is possible that a new version of Python will be released between your -release cycles. The new interpreter release could trigger new warnings in your -code that were not there in an older interpreter, e.g. -:exc:`DeprecationWarning` for a module that you are using. While you as a -developer want to be notified that your code is using a deprecated module, to a -user this information is essentially noise and provides no benefit to them. - -The :mod:`unittest` module has been also updated to use the ``'default'`` -filter while running tests. - - -.. _warning-functions: - -Available Functions -------------------- - - -.. function:: warn(message, category=None, stacklevel=1, source=None) - - Issue a warning, or maybe ignore it or raise an exception. The *category* - argument, if given, must be a warning category class (see above); it defaults to - :exc:`UserWarning`. Alternatively *message* can be a :exc:`Warning` instance, - in which case *category* will be ignored and ``message.__class__`` will be used. - In this case the message text will be ``str(message)``. This function raises an - exception if the particular warning issued is changed into an error by the - warnings filter see above. The *stacklevel* argument can be used by wrapper - functions written in Python, like this:: - - def deprecation(message): - warnings.warn(message, DeprecationWarning, stacklevel=2) - - This makes the warning refer to :func:`deprecation`'s caller, rather than to the - source of :func:`deprecation` itself (since the latter would defeat the purpose - of the warning message). - - *source*, if supplied, is the destroyed object which emitted a - :exc:`ResourceWarning`. - - .. versionchanged:: 3.6 - Added *source* parameter. - - -.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) - - This is a low-level interface to the functionality of :func:`warn`, passing in - explicitly the message, category, filename and line number, and optionally the - module name and the registry (which should be the ``__warningregistry__`` - dictionary of the module). The module name defaults to the filename with - ``.py`` stripped; if no registry is passed, the warning is never suppressed. - *message* must be a string and *category* a subclass of :exc:`Warning` or - *message* may be a :exc:`Warning` instance, in which case *category* will be - ignored. - - *module_globals*, if supplied, should be the global namespace in use by the code - for which the warning is issued. (This argument is used to support displaying - source for modules found in zipfiles or other non-filesystem import - sources). - - *source*, if supplied, is the destroyed object which emitted a - :exc:`ResourceWarning`. - - .. versionchanged:: 3.6 - Add the *source* parameter. - - -.. function:: showwarning(message, category, filename, lineno, file=None, line=None) - - Write a warning to a file. The default implementation calls - ``formatwarning(message, category, filename, lineno, line)`` and writes the - resulting string to *file*, which defaults to ``sys.stderr``. You may replace - this function with any callable by assigning to ``warnings.showwarning``. - *line* is a line of source code to be included in the warning - message; if *line* is not supplied, :func:`showwarning` will - try to read the line specified by *filename* and *lineno*. - - -.. function:: formatwarning(message, category, filename, lineno, line=None) - - Format a warning the standard way. This returns a string which may contain - embedded newlines and ends in a newline. *line* is a line of source code to - be included in the warning message; if *line* is not supplied, - :func:`formatwarning` will try to read the line specified by *filename* and - *lineno*. - - -.. function:: filterwarnings(action, message='', category=Warning, module='', lineno=0, append=False) - - Insert an entry into the list of :ref:`warnings filter specifications - <warning-filter>`. The entry is inserted at the front by default; if - *append* is true, it is inserted at the end. This checks the types of the - arguments, compiles the *message* and *module* regular expressions, and - inserts them as a tuple in the list of warnings filters. Entries closer to - the front of the list override entries later in the list, if both match a - particular warning. Omitted arguments default to a value that matches - everything. - - -.. function:: simplefilter(action, category=Warning, lineno=0, append=False) - - Insert a simple entry into the list of :ref:`warnings filter specifications - <warning-filter>`. The meaning of the function parameters is as for - :func:`filterwarnings`, but regular expressions are not needed as the filter - inserted always matches any message in any module as long as the category and - line number match. - - -.. function:: resetwarnings() - - Reset the warnings filter. This discards the effect of all previous calls to - :func:`filterwarnings`, including that of the :option:`-W` command line options - and calls to :func:`simplefilter`. - - -Available Context Managers --------------------------- - -.. class:: catch_warnings(\*, record=False, module=None) - - A context manager that copies and, upon exit, restores the warnings filter - and the :func:`showwarning` function. - If the *record* argument is :const:`False` (the default) the context manager - returns :class:`None` on entry. If *record* is :const:`True`, a list is - returned that is progressively populated with objects as seen by a custom - :func:`showwarning` function (which also suppresses output to ``sys.stdout``). - Each object in the list has attributes with the same names as the arguments to - :func:`showwarning`. - - The *module* argument takes a module that will be used instead of the - module returned when you import :mod:`warnings` whose filter will be - protected. This argument exists primarily for testing the :mod:`warnings` - module itself. - - .. note:: - - The :class:`catch_warnings` manager works by replacing and - then later restoring the module's - :func:`showwarning` function and internal list of filter - specifications. This means the context manager is modifying - global state and therefore is not thread-safe. diff --git a/third_party/python/Doc/library/wave.rst b/third_party/python/Doc/library/wave.rst deleted file mode 100644 index a9b320532..000000000 --- a/third_party/python/Doc/library/wave.rst +++ /dev/null @@ -1,247 +0,0 @@ -:mod:`wave` --- Read and write WAV files -======================================== - -.. module:: wave - :synopsis: Provide an interface to the WAV sound format. - -.. sectionauthor:: Moshe Zadka <moshez@zadka.site.co.il> -.. Documentations stolen from comments in file. - -**Source code:** :source:`Lib/wave.py` - --------------- - -The :mod:`wave` module provides a convenient interface to the WAV sound format. -It does not support compression/decompression, but it does support mono/stereo. - -The :mod:`wave` module defines the following function and exception: - - -.. function:: open(file, mode=None) - - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: - - ``'rb'`` - Read only mode. - - ``'wb'`` - Write only mode. - - Note that it does not allow read/write WAV files. - - A *mode* of ``'rb'`` returns a :class:`Wave_read` object, while a *mode* of - ``'wb'`` returns a :class:`Wave_write` object. If *mode* is omitted and a - file-like object is passed as *file*, ``file.mode`` is used as the default - value for *mode*. - - If you pass in a file-like object, the wave object will not close it when its - :meth:`close` method is called; it is the caller's responsibility to close - the file object. - - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`with` block completes, the :meth:`Wave_read.close() - <wave.Wave_read.close>` or :meth:`Wave_write.close() - <wave.Wave_write.close()>` method is called. - - .. versionchanged:: 3.4 - Added support for unseekable files. - -.. function:: openfp(file, mode) - - A synonym for :func:`.open`, maintained for backwards compatibility. - - -.. exception:: Error - - An error raised when something is impossible because it violates the WAV - specification or hits an implementation deficiency. - - -.. _wave-read-objects: - -Wave_read Objects ------------------ - -Wave_read objects, as returned by :func:`.open`, have the following methods: - - -.. method:: Wave_read.close() - - Close the stream if it was opened by :mod:`wave`, and make the instance - unusable. This is called automatically on object collection. - - -.. method:: Wave_read.getnchannels() - - Returns number of audio channels (``1`` for mono, ``2`` for stereo). - - -.. method:: Wave_read.getsampwidth() - - Returns sample width in bytes. - - -.. method:: Wave_read.getframerate() - - Returns sampling frequency. - - -.. method:: Wave_read.getnframes() - - Returns number of audio frames. - - -.. method:: Wave_read.getcomptype() - - Returns compression type (``'NONE'`` is the only supported type). - - -.. method:: Wave_read.getcompname() - - Human-readable version of :meth:`getcomptype`. Usually ``'not compressed'`` - parallels ``'NONE'``. - - -.. method:: Wave_read.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: Wave_read.readframes(n) - - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. - - -.. method:: Wave_read.rewind() - - Rewind the file pointer to the beginning of the audio stream. - -The following two methods are defined for compatibility with the :mod:`aifc` -module, and don't do anything interesting. - - -.. method:: Wave_read.getmarkers() - - Returns ``None``. - - -.. method:: Wave_read.getmark(id) - - Raise an error. - -The following two methods define a term "position" which is compatible between -them, and is otherwise implementation dependent. - - -.. method:: Wave_read.setpos(pos) - - Set the file pointer to the specified position. - - -.. method:: Wave_read.tell() - - Return current file pointer position. - - -.. _wave-write-objects: - -Wave_write Objects ------------------- - -For seekable output streams, the ``wave`` header will automatically be updated -to reflect the number of frames actually written. For unseekable streams, the -*nframes* value must be accurate when the first frame data is written. An -accurate *nframes* value can be achieved either by calling -:meth:`~Wave_write.setnframes` or :meth:`~Wave_write.setparams` with the number -of frames that will be written before :meth:`~Wave_write.close` is called and -then using :meth:`~Wave_write.writeframesraw` to write the frame data, or by -calling :meth:`~Wave_write.writeframes` with all of the frame data to be -written. In the latter case :meth:`~Wave_write.writeframes` will calculate -the number of frames in the data and set *nframes* accordingly before writing -the frame data. - -Wave_write objects, as returned by :func:`.open`, have the following methods: - -.. versionchanged:: 3.4 - Added support for unseekable files. - - -.. method:: Wave_write.close() - - Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise - an exception if the output stream is not seekable and *nframes* does not - match the number of frames actually written. - - -.. method:: Wave_write.setnchannels(n) - - Set the number of channels. - - -.. method:: Wave_write.setsampwidth(n) - - Set the sample width to *n* bytes. - - -.. method:: Wave_write.setframerate(n) - - Set the frame rate to *n*. - - .. versionchanged:: 3.2 - A non-integral input to this method is rounded to the nearest - integer. - - -.. method:: Wave_write.setnframes(n) - - Set the number of frames to *n*. This will be changed later if the number - of frames actually written is different (this update attempt will - raise an error if the output stream is not seekable). - - -.. method:: Wave_write.setcomptype(type, name) - - Set the compression type and description. At the moment, only compression type - ``NONE`` is supported, meaning no compression. - - -.. method:: Wave_write.setparams(tuple) - - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the :meth:`set\*` methods. Sets all - parameters. - - -.. method:: Wave_write.tell() - - Return current position in the file, with the same disclaimer for the - :meth:`Wave_read.tell` and :meth:`Wave_read.setpos` methods. - - -.. method:: Wave_write.writeframesraw(data) - - Write audio frames, without correcting *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: Wave_write.writeframes(data) - - Write audio frames and make sure *nframes* is correct. It will raise an - error if the output stream is not seekable and the total number of frames - that have been written after *data* has been written does not match the - previously set value for *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -Note that it is invalid to set any parameters after calling :meth:`writeframes` -or :meth:`writeframesraw`, and any attempt to do so will raise -:exc:`wave.Error`. - diff --git a/third_party/python/Doc/library/weakref.rst b/third_party/python/Doc/library/weakref.rst deleted file mode 100644 index b02a006d7..000000000 --- a/third_party/python/Doc/library/weakref.rst +++ /dev/null @@ -1,574 +0,0 @@ -:mod:`weakref` --- Weak references -================================== - -.. module:: weakref - :synopsis: Support for weak references and weak dictionaries. - -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. moduleauthor:: Neil Schemenauer <nas@arctrix.com> -.. moduleauthor:: Martin von Löwis <martin@loewis.home.cs.tu-berlin.de> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - -**Source code:** :source:`Lib/weakref.py` - --------------- - -The :mod:`weakref` module allows the Python programmer to create :dfn:`weak -references` to objects. - -.. When making changes to the examples in this file, be sure to update - Lib/test/test_weakref.py::libreftest too! - -In the following, the term :dfn:`referent` means the object which is referred to -by a weak reference. - -A weak reference to an object is not enough to keep the object alive: when the -only remaining references to a referent are weak references, -:term:`garbage collection` is free to destroy the referent and reuse its memory -for something else. However, until the object is actually destroyed the weak -reference may return the object even if there are no strong references to it. - -A primary use for weak references is to implement caches or -mappings holding large objects, where it's desired that a large object not be -kept alive solely because it appears in a cache or mapping. - -For example, if you have a number of large binary image objects, you may wish to -associate a name with each. If you used a Python dictionary to map names to -images, or images to names, the image objects would remain alive just because -they appeared as values or keys in the dictionaries. The -:class:`WeakKeyDictionary` and :class:`WeakValueDictionary` classes supplied by -the :mod:`weakref` module are an alternative, using weak references to construct -mappings that don't keep objects alive solely because they appear in the mapping -objects. If, for example, an image object is a value in a -:class:`WeakValueDictionary`, then when the last remaining references to that -image object are the weak references held by weak mappings, garbage collection -can reclaim the object, and its corresponding entries in weak mappings are -simply deleted. - -:class:`WeakKeyDictionary` and :class:`WeakValueDictionary` use weak references -in their implementation, setting up callback functions on the weak references -that notify the weak dictionaries when a key or value has been reclaimed by -garbage collection. :class:`WeakSet` implements the :class:`set` interface, -but keeps weak references to its elements, just like a -:class:`WeakKeyDictionary` does. - -:class:`finalize` provides a straight forward way to register a -cleanup function to be called when an object is garbage collected. -This is simpler to use than setting up a callback function on a raw -weak reference, since the module automatically ensures that the finalizer -remains alive until the object is collected. - -Most programs should find that using one of these weak container types -or :class:`finalize` is all they need -- it's not usually necessary to -create your own weak references directly. The low-level machinery is -exposed by the :mod:`weakref` module for the benefit of advanced uses. - -Not all objects can be weakly referenced; those objects which can include class -instances, functions written in Python (but not in C), instance methods, sets, -frozensets, some :term:`file objects <file object>`, :term:`generator`\s, type -objects, sockets, arrays, deques, regular expression pattern objects, and code -objects. - -.. versionchanged:: 3.2 - Added support for thread.lock, threading.Lock, and code objects. - -Several built-in types such as :class:`list` and :class:`dict` do not directly -support weak references but can add support through subclassing:: - - class Dict(dict): - pass - - obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable - -Other built-in types such as :class:`tuple` and :class:`int` do not support weak -references even when subclassed (This is an implementation detail and may be -different across various Python implementations.). - -Extension types can easily be made to support weak references; see -:ref:`weakref-support`. - - -.. class:: ref(object[, callback]) - - Return a weak reference to *object*. The original object can be retrieved by - calling the reference object if the referent is still alive; if the referent is - no longer alive, calling the reference object will cause :const:`None` to be - returned. If *callback* is provided and not :const:`None`, and the returned - weakref object is still alive, the callback will be called when the object is - about to be finalized; the weak reference object will be passed as the only - parameter to the callback; the referent will no longer be available. - - It is allowable for many weak references to be constructed for the same object. - Callbacks registered for each weak reference will be called from the most - recently registered callback to the oldest registered callback. - - Exceptions raised by the callback will be noted on the standard error output, - but cannot be propagated; they are handled in exactly the same way as exceptions - raised from an object's :meth:`__del__` method. - - Weak references are :term:`hashable` if the *object* is hashable. They will - maintain their hash value even after the *object* was deleted. If - :func:`hash` is called the first time only after the *object* was deleted, - the call will raise :exc:`TypeError`. - - Weak references support tests for equality, but not ordering. If the referents - are still alive, two references have the same equality relationship as their - referents (regardless of the *callback*). If either referent has been deleted, - the references are equal only if the reference objects are the same object. - - This is a subclassable type rather than a factory function. - - .. attribute:: __callback__ - - This read-only attribute returns the callback currently associated to the - weakref. If there is no callback or if the referent of the weakref is - no longer alive then this attribute will have value ``None``. - - .. versionchanged:: 3.4 - Added the :attr:`__callback__` attribute. - - -.. function:: proxy(object[, callback]) - - Return a proxy to *object* which uses a weak reference. This supports use of - the proxy in most contexts instead of requiring the explicit dereferencing used - with weak reference objects. The returned object will have a type of either - ``ProxyType`` or ``CallableProxyType``, depending on whether *object* is - callable. Proxy objects are not :term:`hashable` regardless of the referent; this - avoids a number of problems related to their fundamentally mutable nature, and - prevent their use as dictionary keys. *callback* is the same as the parameter - of the same name to the :func:`ref` function. - - -.. function:: getweakrefcount(object) - - Return the number of weak references and proxies which refer to *object*. - - -.. function:: getweakrefs(object) - - Return a list of all weak reference and proxy objects which refer to *object*. - - -.. class:: WeakKeyDictionary([dict]) - - Mapping class that references keys weakly. Entries in the dictionary will be - discarded when there is no longer a strong reference to the key. This can be - used to associate additional data with an object owned by other parts of an - application without adding attributes to those objects. This can be especially - useful with objects that override attribute accesses. - - .. note:: - - Caution: Because a :class:`WeakKeyDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakKeyDictionary` because actions - performed by the program during iteration may cause items in the - dictionary to vanish "by magic" (as a side effect of garbage collection). - -:class:`WeakKeyDictionary` objects have an additional method that -exposes the internal references directly. The references are not guaranteed to -be "live" at the time they are used, so the result of calling the references -needs to be checked before being used. This can be used to avoid creating -references that will cause the garbage collector to keep the keys around longer -than needed. - - -.. method:: WeakKeyDictionary.keyrefs() - - Return an iterable of the weak references to the keys. - - -.. class:: WeakValueDictionary([dict]) - - Mapping class that references values weakly. Entries in the dictionary will be - discarded when no strong reference to the value exists any more. - - .. note:: - - Caution: Because a :class:`WeakValueDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakValueDictionary` because actions performed - by the program during iteration may cause items in the dictionary to vanish "by - magic" (as a side effect of garbage collection). - -:class:`WeakValueDictionary` objects have an additional method that has the -same issues as the :meth:`keyrefs` method of :class:`WeakKeyDictionary` -objects. - - -.. method:: WeakValueDictionary.valuerefs() - - Return an iterable of the weak references to the values. - - -.. class:: WeakSet([elements]) - - Set class that keeps weak references to its elements. An element will be - discarded when no strong reference to it exists any more. - - -.. class:: WeakMethod(method) - - A custom :class:`ref` subclass which simulates a weak reference to a bound - method (i.e., a method defined on a class and looked up on an instance). - Since a bound method is ephemeral, a standard weak reference cannot keep - hold of it. :class:`WeakMethod` has special code to recreate the bound - method until either the object or the original function dies:: - - >>> class C: - ... def method(self): - ... print("method called!") - ... - >>> c = C() - >>> r = weakref.ref(c.method) - >>> r() - >>> r = weakref.WeakMethod(c.method) - >>> r() - <bound method C.method of <__main__.C object at 0x7fc859830220>> - >>> r()() - method called! - >>> del c - >>> gc.collect() - 0 - >>> r() - >>> - - .. versionadded:: 3.4 - -.. class:: finalize(obj, func, *args, **kwargs) - - Return a callable finalizer object which will be called when *obj* - is garbage collected. Unlike an ordinary weak reference, a finalizer - will always survive until the reference object is collected, greatly - simplifying lifecycle management. - - A finalizer is considered *alive* until it is called (either explicitly - or at garbage collection), and after that it is *dead*. Calling a live - finalizer returns the result of evaluating ``func(*arg, **kwargs)``, - whereas calling a dead finalizer returns :const:`None`. - - Exceptions raised by finalizer callbacks during garbage collection - will be shown on the standard error output, but cannot be - propagated. They are handled in the same way as exceptions raised - from an object's :meth:`__del__` method or a weak reference's - callback. - - When the program exits, each remaining live finalizer is called - unless its :attr:`atexit` attribute has been set to false. They - are called in reverse order of creation. - - A finalizer will never invoke its callback during the later part of - the :term:`interpreter shutdown` when module globals are liable to have - been replaced by :const:`None`. - - .. method:: __call__() - - If *self* is alive then mark it as dead and return the result of - calling ``func(*args, **kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: detach() - - If *self* is alive then mark it as dead and return the tuple - ``(obj, func, args, kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: peek() - - If *self* is alive then return the tuple ``(obj, func, args, - kwargs)``. If *self* is dead then return :const:`None`. - - .. attribute:: alive - - Property which is true if the finalizer is alive, false otherwise. - - .. attribute:: atexit - - A writable boolean property which by default is true. When the - program exits, it calls all remaining live finalizers for which - :attr:`.atexit` is true. They are called in reverse order of - creation. - - .. note:: - - It is important to ensure that *func*, *args* and *kwargs* do - not own any references to *obj*, either directly or indirectly, - since otherwise *obj* will never be garbage collected. In - particular, *func* should not be a bound method of *obj*. - - .. versionadded:: 3.4 - - -.. data:: ReferenceType - - The type object for weak references objects. - - -.. data:: ProxyType - - The type object for proxies of objects which are not callable. - - -.. data:: CallableProxyType - - The type object for proxies of callable objects. - - -.. data:: ProxyTypes - - Sequence containing all the type objects for proxies. This can make it simpler - to test if an object is a proxy without being dependent on naming both proxy - types. - - -.. exception:: ReferenceError - - Exception raised when a proxy object is used but the underlying object has been - collected. This is the same as the standard :exc:`ReferenceError` exception. - - -.. seealso:: - - :pep:`205` - Weak References - The proposal and rationale for this feature, including links to earlier - implementations and information about similar features in other languages. - - -.. _weakref-objects: - -Weak Reference Objects ----------------------- - -Weak reference objects have no methods and no attributes besides -:attr:`ref.__callback__`. A weak reference object allows the referent to be -obtained, if it still exists, by calling it: - - >>> import weakref - >>> class Object: - ... pass - ... - >>> o = Object() - >>> r = weakref.ref(o) - >>> o2 = r() - >>> o is o2 - True - -If the referent no longer exists, calling the reference object returns -:const:`None`: - - >>> del o, o2 - >>> print(r()) - None - -Testing that a weak reference object is still live should be done using the -expression ``ref() is not None``. Normally, application code that needs to use -a reference object should follow this pattern:: - - # r is a weak reference object - o = r() - if o is None: - # referent has been garbage collected - print("Object has been deallocated; can't frobnicate.") - else: - print("Object is still live!") - o.do_something_useful() - -Using a separate test for "liveness" creates race conditions in threaded -applications; another thread can cause a weak reference to become invalidated -before the weak reference is called; the idiom shown above is safe in threaded -applications as well as single-threaded applications. - -Specialized versions of :class:`ref` objects can be created through subclassing. -This is used in the implementation of the :class:`WeakValueDictionary` to reduce -the memory overhead for each entry in the mapping. This may be most useful to -associate additional information with a reference, but could also be used to -insert additional processing on calls to retrieve the referent. - -This example shows how a subclass of :class:`ref` can be used to store -additional information about an object and affect the value that's returned when -the referent is accessed:: - - import weakref - - class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, **annotations): - super(ExtendedRef, self).__init__(ob, callback) - self.__counter = 0 - for k, v in annotations.items(): - setattr(self, k, v) - - def __call__(self): - """Return a pair containing the referent and the number of - times the reference has been called. - """ - ob = super(ExtendedRef, self).__call__() - if ob is not None: - self.__counter += 1 - ob = (ob, self.__counter) - return ob - - -.. _weakref-example: - -Example -------- - -This simple example shows how an application can use object IDs to retrieve -objects that it has seen before. The IDs of the objects can then be used in -other data structures without forcing the objects to remain alive, but the -objects can still be retrieved by ID if they do. - -.. Example contributed by Tim Peters. - -:: - - import weakref - - _id2obj_dict = weakref.WeakValueDictionary() - - def remember(obj): - oid = id(obj) - _id2obj_dict[oid] = obj - return oid - - def id2obj(oid): - return _id2obj_dict[oid] - - -.. _finalize-examples: - -Finalizer Objects ------------------ - -The main benefit of using :class:`finalize` is that it makes it simple -to register a callback without needing to preserve the returned finalizer -object. For instance - - >>> import weakref - >>> class Object: - ... pass - ... - >>> kenny = Object() - >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS - <finalize object at ...; for 'Object' at ...> - >>> del kenny - You killed Kenny! - -The finalizer can be called directly as well. However the finalizer -will invoke the callback at most once. - - >>> def callback(x, y, z): - ... print("CALLBACK") - ... return x + y + z - ... - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> assert f.alive - >>> assert f() == 6 - CALLBACK - >>> assert not f.alive - >>> f() # callback not called because finalizer dead - >>> del obj # callback not called because finalizer dead - -You can unregister a finalizer using its :meth:`~finalize.detach` -method. This kills the finalizer and returns the arguments passed to -the constructor when it was created. - - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> f.detach() #doctest:+ELLIPSIS - (<__main__.Object object ...>, <function callback ...>, (1, 2), {'z': 3}) - >>> newobj, func, args, kwargs = _ - >>> assert not f.alive - >>> assert newobj is obj - >>> assert func(*args, **kwargs) == 6 - CALLBACK - -Unless you set the :attr:`~finalize.atexit` attribute to -:const:`False`, a finalizer will be called when the program exits if it -is still alive. For instance - - >>> obj = Object() - >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS - <finalize object at ...; for 'Object' at ...> - >>> exit() #doctest:+SKIP - obj dead or exiting - - -Comparing finalizers with :meth:`__del__` methods -------------------------------------------------- - -Suppose we want to create a class whose instances represent temporary -directories. The directories should be deleted with their contents -when the first of the following events occurs: - -* the object is garbage collected, -* the object's :meth:`remove` method is called, or -* the program exits. - -We might try to implement the class using a :meth:`__del__` method as -follows:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - - def remove(self): - if self.name is not None: - shutil.rmtree(self.name) - self.name = None - - @property - def removed(self): - return self.name is None - - def __del__(self): - self.remove() - -Starting with Python 3.4, :meth:`__del__` methods no longer prevent -reference cycles from being garbage collected, and module globals are -no longer forced to :const:`None` during :term:`interpreter shutdown`. -So this code should work without any issues on CPython. - -However, handling of :meth:`__del__` methods is notoriously implementation -specific, since it depends on internal details of the interpreter's garbage -collector implementation. - -A more robust alternative can be to define a finalizer which only references -the specific functions and objects that it needs, rather than having access -to the full state of the object:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - self._finalizer = weakref.finalize(self, shutil.rmtree, self.name) - - def remove(self): - self._finalizer() - - @property - def removed(self): - return not self._finalizer.alive - -Defined like this, our finalizer only receives a reference to the details -it needs to clean up the directory appropriately. If the object never gets -garbage collected the finalizer will still be called at exit. - -The other advantage of weakref based finalizers is that they can be used to -register finalizers for classes where the definition is controlled by a -third party, such as running code when a module is unloaded:: - - import weakref, sys - def unloading_module(): - # implicit reference to the module globals from the function body - weakref.finalize(sys.modules[__name__], unloading_module) - - -.. note:: - - If you create a finalizer object in a daemonic thread just as the program - exits then there is the possibility that the finalizer - does not get called at exit. However, in a daemonic thread - :func:`atexit.register`, ``try: ... finally: ...`` and ``with: ...`` - do not guarantee that cleanup occurs either. diff --git a/third_party/python/Doc/library/webbrowser.rst b/third_party/python/Doc/library/webbrowser.rst deleted file mode 100644 index 85d363672..000000000 --- a/third_party/python/Doc/library/webbrowser.rst +++ /dev/null @@ -1,213 +0,0 @@ -:mod:`webbrowser` --- Convenient Web-browser controller -======================================================= - -.. module:: webbrowser - :synopsis: Easy-to-use controller for Web browsers. - -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - -**Source code:** :source:`Lib/webbrowser.py` - --------------- - -The :mod:`webbrowser` module provides a high-level interface to allow displaying -Web-based documents to users. Under most circumstances, simply calling the -:func:`.open` function from this module will do the right thing. - -Under Unix, graphical browsers are preferred under X11, but text-mode browsers -will be used if graphical browsers are not available or an X11 display isn't -available. If text-mode browsers are used, the calling process will block until -the user exits the browser. - -If the environment variable :envvar:`BROWSER` exists, it is interpreted as the -:data:`os.pathsep`-separated list of browsers to try ahead of the platform -defaults. When the value of a list part contains the string ``%s``, then it is -interpreted as a literal browser command line to be used with the argument URL -substituted for ``%s``; if the part does not contain ``%s``, it is simply -interpreted as the name of the browser to launch. [1]_ - -For non-Unix platforms, or when a remote browser is available on Unix, the -controlling process will not wait for the user to finish with the browser, but -allow the remote browser to maintain its own windows on the display. If remote -browsers are not available on Unix, the controlling process will launch a new -browser and wait. - -The script :program:`webbrowser` can be used as a command-line interface for the -module. It accepts a URL as the argument. It accepts the following optional -parameters: ``-n`` opens the URL in a new browser window, if possible; -``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: - - python -m webbrowser -t "http://www.python.org" - -The following exception is defined: - - -.. exception:: Error - - Exception raised when a browser control error occurs. - -The following functions are defined: - - -.. function:: open(url, new=0, autoraise=True) - - Display *url* using the default browser. If *new* is 0, the *url* is opened - in the same browser window if possible. If *new* is 1, a new browser window - is opened if possible. If *new* is 2, a new browser page ("tab") is opened - if possible. If *autoraise* is ``True``, the window is raised if possible - (note that under many window managers this will occur regardless of the - setting of this variable). - - Note that on some platforms, trying to open a filename using this function, - may work and start the operating system's associated program. However, this - is neither supported nor portable. - - -.. function:: open_new(url) - - Open *url* in a new window of the default browser, if possible, otherwise, open - *url* in the only browser window. - -.. function:: open_new_tab(url) - - Open *url* in a new page ("tab") of the default browser, if possible, otherwise - equivalent to :func:`open_new`. - - -.. function:: get(using=None) - - Return a controller object for the browser type *using*. If *using* is - ``None``, return a controller for a default browser appropriate to the - caller's environment. - - -.. function:: register(name, constructor, instance=None) - - Register the browser type *name*. Once a browser type is registered, the - :func:`get` function can return a controller for that browser type. If - *instance* is not provided, or is ``None``, *constructor* will be called without - parameters to create an instance when needed. If *instance* is provided, - *constructor* will never be called, and may be ``None``. - - This entry point is only useful if you plan to either set the :envvar:`BROWSER` - variable or call :func:`get` with a nonempty argument matching the name of a - handler you declare. - -A number of browser types are predefined. This table gives the type names that -may be passed to the :func:`get` function and the corresponding instantiations -for the controller classes, all defined in this module. - -+------------------------+-----------------------------------------+-------+ -| Type Name | Class Name | Notes | -+========================+=========================================+=======+ -| ``'mozilla'`` | :class:`Mozilla('mozilla')` | | -+------------------------+-----------------------------------------+-------+ -| ``'firefox'`` | :class:`Mozilla('mozilla')` | | -+------------------------+-----------------------------------------+-------+ -| ``'netscape'`` | :class:`Mozilla('netscape')` | | -+------------------------+-----------------------------------------+-------+ -| ``'galeon'`` | :class:`Galeon('galeon')` | | -+------------------------+-----------------------------------------+-------+ -| ``'epiphany'`` | :class:`Galeon('epiphany')` | | -+------------------------+-----------------------------------------+-------+ -| ``'skipstone'`` | :class:`BackgroundBrowser('skipstone')` | | -+------------------------+-----------------------------------------+-------+ -| ``'kfmclient'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'konqueror'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'kfm'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'mosaic'`` | :class:`BackgroundBrowser('mosaic')` | | -+------------------------+-----------------------------------------+-------+ -| ``'opera'`` | :class:`Opera()` | | -+------------------------+-----------------------------------------+-------+ -| ``'grail'`` | :class:`Grail()` | | -+------------------------+-----------------------------------------+-------+ -| ``'links'`` | :class:`GenericBrowser('links')` | | -+------------------------+-----------------------------------------+-------+ -| ``'elinks'`` | :class:`Elinks('elinks')` | | -+------------------------+-----------------------------------------+-------+ -| ``'lynx'`` | :class:`GenericBrowser('lynx')` | | -+------------------------+-----------------------------------------+-------+ -| ``'w3m'`` | :class:`GenericBrowser('w3m')` | | -+------------------------+-----------------------------------------+-------+ -| ``'windows-default'`` | :class:`WindowsDefault` | \(2) | -+------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | :class:`MacOSX('default')` | \(3) | -+------------------------+-----------------------------------------+-------+ -| ``'safari'`` | :class:`MacOSX('safari')` | \(3) | -+------------------------+-----------------------------------------+-------+ -| ``'google-chrome'`` | :class:`Chrome('google-chrome')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chrome'`` | :class:`Chrome('chrome')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chromium'`` | :class:`Chromium('chromium')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chromium-browser'`` | :class:`Chromium('chromium-browser')` | | -+------------------------+-----------------------------------------+-------+ - -Notes: - -(1) - "Konqueror" is the file manager for the KDE desktop environment for Unix, and - only makes sense to use if KDE is running. Some way of reliably detecting KDE - would be nice; the :envvar:`KDEDIR` variable is not sufficient. Note also that - the name "kfm" is used even when using the :program:`konqueror` command with KDE - 2 --- the implementation selects the best strategy for running Konqueror. - -(2) - Only on Windows platforms. - -(3) - Only on Mac OS X platform. - -.. versionadded:: 3.3 - Support for Chrome/Chromium has been added. - -Here are some simple examples:: - - url = 'http://docs.python.org/' - - # Open URL in a new tab, if a browser window is already open. - webbrowser.open_new_tab(url) - - # Open URL in new window, raising the window if possible. - webbrowser.open_new(url) - - -.. _browser-controllers: - -Browser Controller Objects --------------------------- - -Browser controllers provide these methods which parallel three of the -module-level convenience functions: - - -.. method:: controller.open(url, new=0, autoraise=True) - - Display *url* using the browser handled by this controller. If *new* is 1, a new - browser window is opened if possible. If *new* is 2, a new browser page ("tab") - is opened if possible. - - -.. method:: controller.open_new(url) - - Open *url* in a new window of the browser handled by this controller, if - possible, otherwise, open *url* in the only browser window. Alias - :func:`open_new`. - - -.. method:: controller.open_new_tab(url) - - Open *url* in a new page ("tab") of the browser handled by this controller, if - possible, otherwise equivalent to :func:`open_new`. - - -.. rubric:: Footnotes - -.. [1] Executables named here without a full path will be searched in the - directories given in the :envvar:`PATH` environment variable. diff --git a/third_party/python/Doc/library/windows.rst b/third_party/python/Doc/library/windows.rst deleted file mode 100644 index b60d4e4cc..000000000 --- a/third_party/python/Doc/library/windows.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _mswin-specific-services: - -**************************** -MS Windows Specific Services -**************************** - -This chapter describes modules that are only available on MS Windows platforms. - - -.. toctree:: - - msilib.rst - msvcrt.rst - winreg.rst - winsound.rst diff --git a/third_party/python/Doc/library/winreg.rst b/third_party/python/Doc/library/winreg.rst deleted file mode 100644 index e9c026102..000000000 --- a/third_party/python/Doc/library/winreg.rst +++ /dev/null @@ -1,756 +0,0 @@ -:mod:`winreg` --- Windows registry access -========================================= - -.. module:: winreg - :platform: Windows - :synopsis: Routines and objects for manipulating the Windows registry. - -.. sectionauthor:: Mark Hammond <MarkH@ActiveState.com> - --------------- - -These functions expose the Windows registry API to Python. Instead of using an -integer as the registry handle, a :ref:`handle object <handle-object>` is used -to ensure that the handles are closed correctly, even if the programmer neglects -to explicitly close them. - -.. _exception-changed: - -.. versionchanged:: 3.3 - Several functions in this module used to raise a - :exc:`WindowsError`, which is now an alias of :exc:`OSError`. - -.. _functions: - -Functions ------------------- - -This module offers the following functions: - - -.. function:: CloseKey(hkey) - - Closes a previously opened registry key. The *hkey* argument specifies a - previously opened key. - - .. note:: - - If *hkey* is not closed using this method (or via :meth:`hkey.Close() - <PyHKEY.Close>`), it is closed when the *hkey* object is destroyed by - Python. - - -.. function:: ConnectRegistry(computer_name, key) - - Establishes a connection to a predefined registry handle on another computer, - and returns a :ref:`handle object <handle-object>`. - - *computer_name* is the name of the remote computer, of the form - ``r"\\computername"``. If ``None``, the local computer is used. - - *key* is the predefined handle to connect to. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: CreateKey(key, sub_key) - - Creates or opens the specified key, returning a - :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the key this method opens or creates. - - If *key* is one of the predefined keys, *sub_key* may be ``None``. In that - case, the handle returned is the same key handle passed in to the function. - - If the key already exists, this function opens the existing key. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: CreateKeyEx(key, sub_key, reserved=0, access=KEY_WRITE) - - Creates or opens the specified key, returning a - :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the key this method opens or creates. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_WRITE`. See - :ref:`Access Rights <access-rights>` for other allowed values. - - If *key* is one of the predefined keys, *sub_key* may be ``None``. In that - case, the handle returned is the same key handle passed in to the function. - - If the key already exists, this function opens the existing key. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteKey(key, sub_key) - - Deletes the specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that must be a subkey of the key identified by the *key* - parameter. This value must not be ``None``, and the key may not have subkeys. - - *This method can not delete keys with subkeys.* - - If the method succeeds, the entire key, including all of its values, is removed. - If the method fails, an :exc:`OSError` exception is raised. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteKeyEx(key, sub_key, access=KEY_WOW64_64KEY, reserved=0) - - Deletes the specified key. - - .. note:: - The :func:`DeleteKeyEx` function is implemented with the RegDeleteKeyEx - Windows API function, which is specific to 64-bit versions of Windows. - See the `RegDeleteKeyEx documentation - <https://msdn.microsoft.com/en-us/library/ms724847%28VS.85%29.aspx>`__. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that must be a subkey of the key identified by the - *key* parameter. This value must not be ``None``, and the key may not have - subkeys. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_WOW64_64KEY`. See - :ref:`Access Rights <access-rights>` for other allowed values. - - *This method can not delete keys with subkeys.* - - If the method succeeds, the entire key, including all of its values, is - removed. If the method fails, an :exc:`OSError` exception is raised. - - On unsupported Windows versions, :exc:`NotImplementedError` is raised. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteValue(key, value) - - Removes a named value from a registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value* is a string that identifies the value to remove. - - -.. function:: EnumKey(key, index) - - Enumerates subkeys of an open registry key, returning a string. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *index* is an integer that identifies the index of the key to retrieve. - - The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly until an :exc:`OSError` exception is - raised, indicating, no more values are available. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: EnumValue(key, index) - - Enumerates values of an open registry key, returning a tuple. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *index* is an integer that identifies the index of the value to retrieve. - - The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly, until an :exc:`OSError` exception is - raised, indicating no more values. - - The result is a tuple of 3 items: - - +-------+--------------------------------------------+ - | Index | Meaning | - +=======+============================================+ - | ``0`` | A string that identifies the value name | - +-------+--------------------------------------------+ - | ``1`` | An object that holds the value data, and | - | | whose type depends on the underlying | - | | registry type | - +-------+--------------------------------------------+ - | ``2`` | An integer that identifies the type of the | - | | value data (see table in docs for | - | | :meth:`SetValueEx`) | - +-------+--------------------------------------------+ - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. index:: - single: % (percent); environment variables expansion (Windows) - -.. function:: ExpandEnvironmentStrings(str) - - Expands environment variable placeholders ``%NAME%`` in strings like - :const:`REG_EXPAND_SZ`:: - - >>> ExpandEnvironmentStrings('%windir%') - 'C:\\Windows' - - -.. function:: FlushKey(key) - - Writes all the attributes of a key to the registry. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - It is not necessary to call :func:`FlushKey` to change a key. Registry changes are - flushed to disk by the registry using its lazy flusher. Registry changes are - also flushed to disk at system shutdown. Unlike :func:`CloseKey`, the - :func:`FlushKey` method returns only when all the data has been written to the - registry. An application should only call :func:`FlushKey` if it requires - absolute certainty that registry changes are on disk. - - .. note:: - - If you don't know whether a :func:`FlushKey` call is required, it probably - isn't. - - -.. function:: LoadKey(key, sub_key, file_name) - - Creates a subkey under the specified key and stores registration information - from a specified file into that subkey. - - *key* is a handle returned by :func:`ConnectRegistry` or one of the constants - :const:`HKEY_USERS` or :const:`HKEY_LOCAL_MACHINE`. - - *sub_key* is a string that identifies the subkey to load. - - *file_name* is the name of the file to load registry data from. This file must - have been created with the :func:`SaveKey` function. Under the file allocation - table (FAT) file system, the filename may not have an extension. - - A call to :func:`LoadKey` fails if the calling process does not have the - :const:`SE_RESTORE_PRIVILEGE` privilege. Note that privileges are different - from permissions -- see the `RegLoadKey documentation - <https://msdn.microsoft.com/en-us/library/ms724889%28v=VS.85%29.aspx>`__ for - more details. - - If *key* is a handle returned by :func:`ConnectRegistry`, then the path - specified in *file_name* is relative to the remote computer. - - -.. function:: OpenKey(key, sub_key, reserved=0, access=KEY_READ) - OpenKeyEx(key, sub_key, reserved=0, access=KEY_READ) - - Opens the specified key, returning a :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that identifies the sub_key to open. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_READ`. See :ref:`Access - Rights <access-rights>` for other allowed values. - - The result is a new handle to the specified key. - - If the function fails, :exc:`OSError` is raised. - - .. versionchanged:: 3.2 - Allow the use of named arguments. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: QueryInfoKey(key) - - Returns information about a key, as a tuple. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - The result is a tuple of 3 items: - - +-------+---------------------------------------------+ - | Index | Meaning | - +=======+=============================================+ - | ``0`` | An integer giving the number of sub keys | - | | this key has. | - +-------+---------------------------------------------+ - | ``1`` | An integer giving the number of values this | - | | key has. | - +-------+---------------------------------------------+ - | ``2`` | An integer giving when the key was last | - | | modified (if available) as 100's of | - | | nanoseconds since Jan 1, 1601. | - +-------+---------------------------------------------+ - - -.. function:: QueryValue(key, sub_key) - - Retrieves the unnamed value for a key, as a string. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that holds the name of the subkey with which the value is - associated. If this parameter is ``None`` or empty, the function retrieves the - value set by the :func:`SetValue` method for the key identified by *key*. - - Values in the registry have name, type, and data components. This method - retrieves the data for a key's first value that has a NULL name. But the - underlying API call doesn't return the type, so always use - :func:`QueryValueEx` if possible. - - -.. function:: QueryValueEx(key, value_name) - - Retrieves the type and data for a specified value name associated with - an open registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value_name* is a string indicating the value to query. - - The result is a tuple of 2 items: - - +-------+-----------------------------------------+ - | Index | Meaning | - +=======+=========================================+ - | ``0`` | The value of the registry item. | - +-------+-----------------------------------------+ - | ``1`` | An integer giving the registry type for | - | | this value (see table in docs for | - | | :meth:`SetValueEx`) | - +-------+-----------------------------------------+ - - -.. function:: SaveKey(key, file_name) - - Saves the specified key, and all its subkeys to the specified file. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *file_name* is the name of the file to save registry data to. This file - cannot already exist. If this filename includes an extension, it cannot be - used on file allocation table (FAT) file systems by the :meth:`LoadKey` - method. - - If *key* represents a key on a remote computer, the path described by - *file_name* is relative to the remote computer. The caller of this method must - possess the :const:`SeBackupPrivilege` security privilege. Note that - privileges are different than permissions -- see the - `Conflicts Between User Rights and Permissions documentation - <https://msdn.microsoft.com/en-us/library/ms724878%28v=VS.85%29.aspx>`__ - for more details. - - This function passes NULL for *security_attributes* to the API. - - -.. function:: SetValue(key, sub_key, type, value) - - Associates a value with a specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the subkey with which the value is associated. - - *type* is an integer that specifies the type of the data. Currently this must be - :const:`REG_SZ`, meaning only strings are supported. Use the :func:`SetValueEx` - function for support for other data types. - - *value* is a string that specifies the new value. - - If the key specified by the *sub_key* parameter does not exist, the SetValue - function creates it. - - Value lengths are limited by available memory. Long values (more than 2048 - bytes) should be stored as files with the filenames stored in the configuration - registry. This helps the registry perform efficiently. - - The key identified by the *key* parameter must have been opened with - :const:`KEY_SET_VALUE` access. - - -.. function:: SetValueEx(key, value_name, reserved, type, value) - - Stores data in the value field of an open registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value_name* is a string that names the subkey with which the value is - associated. - - *reserved* can be anything -- zero is always passed to the API. - - *type* is an integer that specifies the type of the data. See - :ref:`Value Types <value-types>` for the available types. - - *value* is a string that specifies the new value. - - This method can also set additional value and type information for the specified - key. The key identified by the key parameter must have been opened with - :const:`KEY_SET_VALUE` access. - - To open the key, use the :func:`CreateKey` or :func:`OpenKey` methods. - - Value lengths are limited by available memory. Long values (more than 2048 - bytes) should be stored as files with the filenames stored in the configuration - registry. This helps the registry perform efficiently. - - -.. function:: DisableReflectionKey(key) - - Disables registry reflection for 32-bit processes running on a 64-bit - operating system. - - *key* is an already open key, or one of the predefined :ref:`HKEY_* constants - <hkey-constants>`. - - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating - system. - - If the key is not on the reflection list, the function succeeds but has no - effect. Disabling reflection for a key does not affect reflection of any - subkeys. - - -.. function:: EnableReflectionKey(key) - - Restores registry reflection for the specified disabled key. - - *key* is an already open key, or one of the predefined :ref:`HKEY_* constants - <hkey-constants>`. - - Will generally raise :exc:`NotImplemented` if executed on a 32-bit operating - system. - - Restoring reflection for a key does not affect reflection of any subkeys. - - -.. function:: QueryReflectionKey(key) - - Determines the reflection state for the specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - Returns ``True`` if reflection is disabled. - - Will generally raise :exc:`NotImplemented` if executed on a 32-bit - operating system. - - -.. _constants: - -Constants ------------------- - -The following constants are defined for use in many :mod:`_winreg` functions. - -.. _hkey-constants: - -HKEY_* Constants -++++++++++++++++ - -.. data:: HKEY_CLASSES_ROOT - - Registry entries subordinate to this key define types (or classes) of - documents and the properties associated with those types. Shell and - COM applications use the information stored under this key. - - -.. data:: HKEY_CURRENT_USER - - Registry entries subordinate to this key define the preferences of - the current user. These preferences include the settings of - environment variables, data about program groups, colors, printers, - network connections, and application preferences. - -.. data:: HKEY_LOCAL_MACHINE - - Registry entries subordinate to this key define the physical state - of the computer, including data about the bus type, system memory, - and installed hardware and software. - -.. data:: HKEY_USERS - - Registry entries subordinate to this key define the default user - configuration for new users on the local computer and the user - configuration for the current user. - -.. data:: HKEY_PERFORMANCE_DATA - - Registry entries subordinate to this key allow you to access - performance data. The data is not actually stored in the registry; - the registry functions cause the system to collect the data from - its source. - - -.. data:: HKEY_CURRENT_CONFIG - - Contains information about the current hardware profile of the - local computer system. - -.. data:: HKEY_DYN_DATA - - This key is not used in versions of Windows after 98. - - -.. _access-rights: - -Access Rights -+++++++++++++ - -For more information, see `Registry Key Security and Access -<https://msdn.microsoft.com/en-us/library/ms724878%28v=VS.85%29.aspx>`__. - -.. data:: KEY_ALL_ACCESS - - Combines the STANDARD_RIGHTS_REQUIRED, :const:`KEY_QUERY_VALUE`, - :const:`KEY_SET_VALUE`, :const:`KEY_CREATE_SUB_KEY`, - :const:`KEY_ENUMERATE_SUB_KEYS`, :const:`KEY_NOTIFY`, - and :const:`KEY_CREATE_LINK` access rights. - -.. data:: KEY_WRITE - - Combines the STANDARD_RIGHTS_WRITE, :const:`KEY_SET_VALUE`, and - :const:`KEY_CREATE_SUB_KEY` access rights. - -.. data:: KEY_READ - - Combines the STANDARD_RIGHTS_READ, :const:`KEY_QUERY_VALUE`, - :const:`KEY_ENUMERATE_SUB_KEYS`, and :const:`KEY_NOTIFY` values. - -.. data:: KEY_EXECUTE - - Equivalent to :const:`KEY_READ`. - -.. data:: KEY_QUERY_VALUE - - Required to query the values of a registry key. - -.. data:: KEY_SET_VALUE - - Required to create, delete, or set a registry value. - -.. data:: KEY_CREATE_SUB_KEY - - Required to create a subkey of a registry key. - -.. data:: KEY_ENUMERATE_SUB_KEYS - - Required to enumerate the subkeys of a registry key. - -.. data:: KEY_NOTIFY - - Required to request change notifications for a registry key or for - subkeys of a registry key. - -.. data:: KEY_CREATE_LINK - - Reserved for system use. - - -.. _64-bit-access-rights: - -64-bit Specific -*************** - -For more information, see `Accessing an Alternate Registry View -<https://msdn.microsoft.com/en-us/library/aa384129(v=VS.85).aspx>`__. - -.. data:: KEY_WOW64_64KEY - - Indicates that an application on 64-bit Windows should operate on - the 64-bit registry view. - -.. data:: KEY_WOW64_32KEY - - Indicates that an application on 64-bit Windows should operate on - the 32-bit registry view. - - -.. _value-types: - -Value Types -+++++++++++ - -For more information, see `Registry Value Types -<https://msdn.microsoft.com/en-us/library/ms724884%28v=VS.85%29.aspx>`__. - -.. data:: REG_BINARY - - Binary data in any form. - -.. data:: REG_DWORD - - 32-bit number. - -.. data:: REG_DWORD_LITTLE_ENDIAN - - A 32-bit number in little-endian format. Equivalent to :const:`REG_DWORD`. - -.. data:: REG_DWORD_BIG_ENDIAN - - A 32-bit number in big-endian format. - -.. data:: REG_EXPAND_SZ - - Null-terminated string containing references to environment - variables (``%PATH%``). - -.. data:: REG_LINK - - A Unicode symbolic link. - -.. data:: REG_MULTI_SZ - - A sequence of null-terminated strings, terminated by two null characters. - (Python handles this termination automatically.) - -.. data:: REG_NONE - - No defined value type. - -.. data:: REG_QWORD - - A 64-bit number. - - .. versionadded:: 3.6 - -.. data:: REG_QWORD_LITTLE_ENDIAN - - A 64-bit number in little-endian format. Equivalent to :const:`REG_QWORD`. - - .. versionadded:: 3.6 - -.. data:: REG_RESOURCE_LIST - - A device-driver resource list. - -.. data:: REG_FULL_RESOURCE_DESCRIPTOR - - A hardware setting. - -.. data:: REG_RESOURCE_REQUIREMENTS_LIST - - A hardware resource list. - -.. data:: REG_SZ - - A null-terminated string. - - -.. _handle-object: - -Registry Handle Objects ------------------------ - -This object wraps a Windows HKEY object, automatically closing it when the -object is destroyed. To guarantee cleanup, you can call either the -:meth:`~PyHKEY.Close` method on the object, or the :func:`CloseKey` function. - -All registry functions in this module return one of these objects. - -All registry functions in this module which accept a handle object also accept -an integer, however, use of the handle object is encouraged. - -Handle objects provide semantics for :meth:`__bool__` -- thus :: - - if handle: - print("Yes") - -will print ``Yes`` if the handle is currently valid (has not been closed or -detached). - -The object also support comparison semantics, so handle objects will compare -true if they both reference the same underlying Windows handle value. - -Handle objects can be converted to an integer (e.g., using the built-in -:func:`int` function), in which case the underlying Windows handle value is -returned. You can also use the :meth:`~PyHKEY.Detach` method to return the -integer handle, and also disconnect the Windows handle from the handle object. - - -.. method:: PyHKEY.Close() - - Closes the underlying Windows handle. - - If the handle is already closed, no error is raised. - - -.. method:: PyHKEY.Detach() - - Detaches the Windows handle from the handle object. - - The result is an integer that holds the value of the handle before it is - detached. If the handle is already detached or closed, this will return - zero. - - After calling this function, the handle is effectively invalidated, but the - handle is not closed. You would call this function when you need the - underlying Win32 handle to exist beyond the lifetime of the handle object. - -.. method:: PyHKEY.__enter__() - PyHKEY.__exit__(\*exc_info) - - The HKEY object implements :meth:`~object.__enter__` and - :meth:`~object.__exit__` and thus supports the context protocol for the - :keyword:`with` statement:: - - with OpenKey(HKEY_LOCAL_MACHINE, "foo") as key: - ... # work with key - - will automatically close *key* when control leaves the :keyword:`with` block. - - diff --git a/third_party/python/Doc/library/winsound.rst b/third_party/python/Doc/library/winsound.rst deleted file mode 100644 index 372f792a0..000000000 --- a/third_party/python/Doc/library/winsound.rst +++ /dev/null @@ -1,161 +0,0 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= - -.. module:: winsound - :platform: Windows - :synopsis: Access to the sound-playing machinery for Windows. - -.. moduleauthor:: Toby Dickenson <htrd90@zepler.org> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - --------------- - -The :mod:`winsound` module provides access to the basic sound-playing machinery -provided by Windows platforms. It includes functions and several constants. - - -.. function:: Beep(frequency, duration) - - Beep the PC's speaker. The *frequency* parameter specifies frequency, in hertz, - of the sound, and must be in the range 37 through 32,767. The *duration* - parameter specifies the number of milliseconds the sound should last. If the - system is not able to beep the speaker, :exc:`RuntimeError` is raised. - - -.. function:: PlaySound(sound, flags) - - Call the underlying :c:func:`PlaySound` function from the Platform API. The - *sound* parameter may be a filename, a system sound alias, audio data as a - :term:`bytes-like object`, or ``None``. Its - interpretation depends on the value of *flags*, which can be a bitwise ORed - combination of the constants described below. If the *sound* parameter is - ``None``, any currently playing waveform sound is stopped. If the system - indicates an error, :exc:`RuntimeError` is raised. - - -.. function:: MessageBeep(type=MB_OK) - - Call the underlying :c:func:`MessageBeep` function from the Platform API. This - plays a sound as specified in the registry. The *type* argument specifies which - sound to play; possible values are ``-1``, ``MB_ICONASTERISK``, - ``MB_ICONEXCLAMATION``, ``MB_ICONHAND``, ``MB_ICONQUESTION``, and ``MB_OK``, all - described below. The value ``-1`` produces a "simple beep"; this is the final - fallback if a sound cannot be played otherwise. If the system indicates an - error, :exc:`RuntimeError` is raised. - - -.. data:: SND_FILENAME - - The *sound* parameter is the name of a WAV file. Do not use with - :const:`SND_ALIAS`. - - -.. data:: SND_ALIAS - - The *sound* parameter is a sound association name from the registry. If the - registry contains no such name, play the system default sound unless - :const:`SND_NODEFAULT` is also specified. If no default sound is registered, - raise :exc:`RuntimeError`. Do not use with :const:`SND_FILENAME`. - - All Win32 systems support at least the following; most systems support many - more: - - +--------------------------+----------------------------------------+ - | :func:`PlaySound` *name* | Corresponding Control Panel Sound name | - +==========================+========================================+ - | ``'SystemAsterisk'`` | Asterisk | - +--------------------------+----------------------------------------+ - | ``'SystemExclamation'`` | Exclamation | - +--------------------------+----------------------------------------+ - | ``'SystemExit'`` | Exit Windows | - +--------------------------+----------------------------------------+ - | ``'SystemHand'`` | Critical Stop | - +--------------------------+----------------------------------------+ - | ``'SystemQuestion'`` | Question | - +--------------------------+----------------------------------------+ - - For example:: - - import winsound - # Play Windows exit sound. - winsound.PlaySound("SystemExit", winsound.SND_ALIAS) - - # Probably play Windows default sound, if any is registered (because - # "*" probably isn't the registered name of any sound). - winsound.PlaySound("*", winsound.SND_ALIAS) - - -.. data:: SND_LOOP - - Play the sound repeatedly. The :const:`SND_ASYNC` flag must also be used to - avoid blocking. Cannot be used with :const:`SND_MEMORY`. - - -.. data:: SND_MEMORY - - The *sound* parameter to :func:`PlaySound` is a memory image of a WAV file, as a - :term:`bytes-like object`. - - .. note:: - - This module does not support playing from a memory image asynchronously, so a - combination of this flag and :const:`SND_ASYNC` will raise :exc:`RuntimeError`. - - -.. data:: SND_PURGE - - Stop playing all instances of the specified sound. - - .. note:: - - This flag is not supported on modern Windows platforms. - - -.. data:: SND_ASYNC - - Return immediately, allowing sounds to play asynchronously. - - -.. data:: SND_NODEFAULT - - If the specified sound cannot be found, do not play the system default sound. - - -.. data:: SND_NOSTOP - - Do not interrupt sounds currently playing. - - -.. data:: SND_NOWAIT - - Return immediately if the sound driver is busy. - - .. note:: - - This flag is not supported on modern Windows platforms. - - -.. data:: MB_ICONASTERISK - - Play the ``SystemDefault`` sound. - - -.. data:: MB_ICONEXCLAMATION - - Play the ``SystemExclamation`` sound. - - -.. data:: MB_ICONHAND - - Play the ``SystemHand`` sound. - - -.. data:: MB_ICONQUESTION - - Play the ``SystemQuestion`` sound. - - -.. data:: MB_OK - - Play the ``SystemDefault`` sound. - diff --git a/third_party/python/Doc/library/wsgiref.rst b/third_party/python/Doc/library/wsgiref.rst deleted file mode 100644 index ddfc2e547..000000000 --- a/third_party/python/Doc/library/wsgiref.rst +++ /dev/null @@ -1,781 +0,0 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== - -.. module:: wsgiref - :synopsis: WSGI Utilities and Reference Implementation. - -.. moduleauthor:: Phillip J. Eby <pje@telecommunity.com> -.. sectionauthor:: Phillip J. Eby <pje@telecommunity.com> - --------------- - -The Web Server Gateway Interface (WSGI) is a standard interface between web -server software and web applications written in Python. Having a standard -interface makes it easy to use an application that supports WSGI with a number -of different web servers. - -Only authors of web servers and programming frameworks need to know every detail -and corner case of the WSGI design. You don't need to understand every detail -of WSGI just to install a WSGI application or to write a web application using -an existing framework. - -:mod:`wsgiref` is a reference implementation of the WSGI specification that can -be used to add WSGI support to a web server or framework. It provides utilities -for manipulating WSGI environment variables and response headers, base classes -for implementing WSGI servers, a demo HTTP server that serves WSGI applications, -and a validation tool that checks WSGI servers and applications for conformance -to the WSGI specification (:pep:`3333`). - -See https://wsgi.readthedocs.org/ for more information about WSGI, and links to -tutorials and other resources. - -.. XXX If you're just trying to write a web application... - - -:mod:`wsgiref.util` -- WSGI environment utilities -------------------------------------------------- - -.. module:: wsgiref.util - :synopsis: WSGI environment utilities. - - -This module provides a variety of utility functions for working with WSGI -environments. A WSGI environment is a dictionary containing HTTP request -variables as described in :pep:`3333`. All of the functions taking an *environ* -parameter expect a WSGI-compliant dictionary to be supplied; please see -:pep:`3333` for a detailed specification. - - -.. function:: guess_scheme(environ) - - Return a guess for whether ``wsgi.url_scheme`` should be "http" or "https", by - checking for a ``HTTPS`` environment variable in the *environ* dictionary. The - return value is a string. - - This function is useful when creating a gateway that wraps CGI or a CGI-like - protocol such as FastCGI. Typically, servers providing such protocols will - include a ``HTTPS`` variable with a value of "1" "yes", or "on" when a request - is received via SSL. So, this function returns "https" if such a value is - found, and "http" otherwise. - - -.. function:: request_uri(environ, include_query=True) - - Return the full request URI, optionally including the query string, using the - algorithm found in the "URL Reconstruction" section of :pep:`3333`. If - *include_query* is false, the query string is not included in the resulting URI. - - -.. function:: application_uri(environ) - - Similar to :func:`request_uri`, except that the ``PATH_INFO`` and - ``QUERY_STRING`` variables are ignored. The result is the base URI of the - application object addressed by the request. - - -.. function:: shift_path_info(environ) - - Shift a single name from ``PATH_INFO`` to ``SCRIPT_NAME`` and return the name. - The *environ* dictionary is *modified* in-place; use a copy if you need to keep - the original ``PATH_INFO`` or ``SCRIPT_NAME`` intact. - - If there are no remaining path segments in ``PATH_INFO``, ``None`` is returned. - - Typically, this routine is used to process each portion of a request URI path, - for example to treat the path as a series of dictionary keys. This routine - modifies the passed-in environment to make it suitable for invoking another WSGI - application that is located at the target URI. For example, if there is a WSGI - application at ``/foo``, and the request URI path is ``/foo/bar/baz``, and the - WSGI application at ``/foo`` calls :func:`shift_path_info`, it will receive the - string "bar", and the environment will be updated to be suitable for passing to - a WSGI application at ``/foo/bar``. That is, ``SCRIPT_NAME`` will change from - ``/foo`` to ``/foo/bar``, and ``PATH_INFO`` will change from ``/bar/baz`` to - ``/baz``. - - When ``PATH_INFO`` is just a "/", this routine returns an empty string and - appends a trailing slash to ``SCRIPT_NAME``, even though empty path segments are - normally ignored, and ``SCRIPT_NAME`` doesn't normally end in a slash. This is - intentional behavior, to ensure that an application can tell the difference - between URIs ending in ``/x`` from ones ending in ``/x/`` when using this - routine to do object traversal. - - -.. function:: setup_testing_defaults(environ) - - Update *environ* with trivial defaults for testing purposes. - - This routine adds various parameters required for WSGI, including ``HTTP_HOST``, - ``SERVER_NAME``, ``SERVER_PORT``, ``REQUEST_METHOD``, ``SCRIPT_NAME``, - ``PATH_INFO``, and all of the :pep:`3333`\ -defined ``wsgi.*`` variables. It - only supplies default values, and does not replace any existing settings for - these variables. - - This routine is intended to make it easier for unit tests of WSGI servers and - applications to set up dummy environments. It should NOT be used by actual WSGI - servers or applications, since the data is fake! - - Example usage:: - - from wsgiref.util import setup_testing_defaults - from wsgiref.simple_server import make_server - - # A relatively simple WSGI application. It's going to print out the - # environment dictionary after being updated by setup_testing_defaults - def simple_app(environ, start_response): - setup_testing_defaults(environ) - - status = '200 OK' - headers = [('Content-type', 'text/plain; charset=utf-8')] - - start_response(status, headers) - - ret = [("%s: %s\n" % (key, value)).encode("utf-8") - for key, value in environ.items()] - return ret - - with make_server('', 8000, simple_app) as httpd: - print("Serving on port 8000...") - httpd.serve_forever() - - -In addition to the environment functions above, the :mod:`wsgiref.util` module -also provides these miscellaneous utilities: - - -.. function:: is_hop_by_hop(header_name) - - Return true if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by - :rfc:`2616`. - - -.. class:: FileWrapper(filelike, blksize=8192) - - A wrapper to convert a file-like object to an :term:`iterator`. The resulting objects - support both :meth:`__getitem__` and :meth:`__iter__` iteration styles, for - compatibility with Python 2.1 and Jython. As the object is iterated over, the - optional *blksize* parameter will be repeatedly passed to the *filelike* - object's :meth:`read` method to obtain bytestrings to yield. When :meth:`read` - returns an empty bytestring, iteration is ended and is not resumable. - - If *filelike* has a :meth:`close` method, the returned object will also have a - :meth:`close` method, and it will invoke the *filelike* object's :meth:`close` - method when called. - - Example usage:: - - from io import StringIO - from wsgiref.util import FileWrapper - - # We're using a StringIO-buffer for as the file-like object - filelike = StringIO("This is an example file-like object"*10) - wrapper = FileWrapper(filelike, blksize=5) - - for chunk in wrapper: - print(chunk) - - - -:mod:`wsgiref.headers` -- WSGI response header tools ----------------------------------------------------- - -.. module:: wsgiref.headers - :synopsis: WSGI response header tools. - - -This module provides a single class, :class:`Headers`, for convenient -manipulation of WSGI response headers using a mapping-like interface. - - -.. class:: Headers([headers]) - - Create a mapping-like object wrapping *headers*, which must be a list of header - name/value tuples as described in :pep:`3333`. The default value of *headers* is - an empty list. - - :class:`Headers` objects support typical mapping operations including - :meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, - :meth:`__delitem__` and :meth:`__contains__`. For each of - these methods, the key is the header name (treated case-insensitively), and the - value is the first value associated with that header name. Setting a header - deletes any existing values for that header, then adds a new value at the end of - the wrapped header list. Headers' existing order is generally maintained, with - new headers added to the end of the wrapped list. - - Unlike a dictionary, :class:`Headers` objects do not raise an error when you try - to get or delete a key that isn't in the wrapped header list. Getting a - nonexistent header just returns ``None``, and deleting a nonexistent header does - nothing. - - :class:`Headers` objects also support :meth:`keys`, :meth:`values`, and - :meth:`items` methods. The lists returned by :meth:`keys` and :meth:`items` can - include the same key more than once if there is a multi-valued header. The - ``len()`` of a :class:`Headers` object is the same as the length of its - :meth:`items`, which is the same as the length of the wrapped header list. In - fact, the :meth:`items` method just returns a copy of the wrapped header list. - - Calling ``bytes()`` on a :class:`Headers` object returns a formatted bytestring - suitable for transmission as HTTP response headers. Each header is placed on a - line with its value, separated by a colon and a space. Each line is terminated - by a carriage return and line feed, and the bytestring is terminated with a - blank line. - - In addition to their mapping interface and formatting features, :class:`Headers` - objects also have the following methods for querying and adding multi-valued - headers, and for adding headers with MIME parameters: - - - .. method:: Headers.get_all(name) - - Return a list of all the values for the named header. - - The returned list will be sorted in the order they appeared in the original - header list or were added to this instance, and may contain duplicates. Any - fields deleted and re-inserted are always appended to the header list. If no - fields exist with the given name, returns an empty list. - - - .. method:: Headers.add_header(name, value, **_params) - - Add a (possibly multi-valued) header, with optional MIME parameters specified - via keyword arguments. - - *name* is the header field to add. Keyword arguments can be used to set MIME - parameters for the header field. Each parameter must be a string or ``None``. - Underscores in parameter names are converted to dashes, since dashes are illegal - in Python identifiers, but many MIME parameter names include dashes. If the - parameter value is a string, it is added to the header value parameters in the - form ``name="value"``. If it is ``None``, only the parameter name is added. - (This is used for MIME parameters without a value.) Example usage:: - - h.add_header('content-disposition', 'attachment', filename='bud.gif') - - The above will add a header that looks like this:: - - Content-Disposition: attachment; filename="bud.gif" - - - .. versionchanged:: 3.5 - *headers* parameter is optional. - - -:mod:`wsgiref.simple_server` -- a simple WSGI HTTP server ---------------------------------------------------------- - -.. module:: wsgiref.simple_server - :synopsis: A simple WSGI HTTP server. - - -This module implements a simple HTTP server (based on :mod:`http.server`) -that serves WSGI applications. Each server instance serves a single WSGI -application on a given host and port. If you want to serve multiple -applications on a single host and port, you should create a WSGI application -that parses ``PATH_INFO`` to select which application to invoke for each -request. (E.g., using the :func:`shift_path_info` function from -:mod:`wsgiref.util`.) - - -.. function:: make_server(host, port, app, server_class=WSGIServer, handler_class=WSGIRequestHandler) - - Create a new WSGI server listening on *host* and *port*, accepting connections - for *app*. The return value is an instance of the supplied *server_class*, and - will process requests using the specified *handler_class*. *app* must be a WSGI - application object, as defined by :pep:`3333`. - - Example usage:: - - from wsgiref.simple_server import make_server, demo_app - - with make_server('', 8000, demo_app) as httpd: - print("Serving HTTP on port 8000...") - - # Respond to requests until process is killed - httpd.serve_forever() - - # Alternative: serve one request, then exit - httpd.handle_request() - - -.. function:: demo_app(environ, start_response) - - This function is a small but complete WSGI application that returns a text page - containing the message "Hello world!" and a list of the key/value pairs provided - in the *environ* parameter. It's useful for verifying that a WSGI server (such - as :mod:`wsgiref.simple_server`) is able to run a simple WSGI application - correctly. - - -.. class:: WSGIServer(server_address, RequestHandlerClass) - - Create a :class:`WSGIServer` instance. *server_address* should be a - ``(host,port)`` tuple, and *RequestHandlerClass* should be the subclass of - :class:`http.server.BaseHTTPRequestHandler` that will be used to process - requests. - - You do not normally need to call this constructor, as the :func:`make_server` - function can handle all the details for you. - - :class:`WSGIServer` is a subclass of :class:`http.server.HTTPServer`, so all - of its methods (such as :meth:`serve_forever` and :meth:`handle_request`) are - available. :class:`WSGIServer` also provides these WSGI-specific methods: - - - .. method:: WSGIServer.set_app(application) - - Sets the callable *application* as the WSGI application that will receive - requests. - - - .. method:: WSGIServer.get_app() - - Returns the currently-set application callable. - - Normally, however, you do not need to use these additional methods, as - :meth:`set_app` is normally called by :func:`make_server`, and the - :meth:`get_app` exists mainly for the benefit of request handler instances. - - -.. class:: WSGIRequestHandler(request, client_address, server) - - Create an HTTP handler for the given *request* (i.e. a socket), *client_address* - (a ``(host,port)`` tuple), and *server* (:class:`WSGIServer` instance). - - You do not need to create instances of this class directly; they are - automatically created as needed by :class:`WSGIServer` objects. You can, - however, subclass this class and supply it as a *handler_class* to the - :func:`make_server` function. Some possibly relevant methods for overriding in - subclasses: - - - .. method:: WSGIRequestHandler.get_environ() - - Returns a dictionary containing the WSGI environment for a request. The default - implementation copies the contents of the :class:`WSGIServer` object's - :attr:`base_environ` dictionary attribute and then adds various headers derived - from the HTTP request. Each call to this method should return a new dictionary - containing all of the relevant CGI environment variables as specified in - :pep:`3333`. - - - .. method:: WSGIRequestHandler.get_stderr() - - Return the object that should be used as the ``wsgi.errors`` stream. The default - implementation just returns ``sys.stderr``. - - - .. method:: WSGIRequestHandler.handle() - - Process the HTTP request. The default implementation creates a handler instance - using a :mod:`wsgiref.handlers` class to implement the actual WSGI application - interface. - - -:mod:`wsgiref.validate` --- WSGI conformance checker ----------------------------------------------------- - -.. module:: wsgiref.validate - :synopsis: WSGI conformance checker. - - -When creating new WSGI application objects, frameworks, servers, or middleware, -it can be useful to validate the new code's conformance using -:mod:`wsgiref.validate`. This module provides a function that creates WSGI -application objects that validate communications between a WSGI server or -gateway and a WSGI application object, to check both sides for protocol -conformance. - -Note that this utility does not guarantee complete :pep:`3333` compliance; an -absence of errors from this module does not necessarily mean that errors do not -exist. However, if this module does produce an error, then it is virtually -certain that either the server or application is not 100% compliant. - -This module is based on the :mod:`paste.lint` module from Ian Bicking's "Python -Paste" library. - - -.. function:: validator(application) - - Wrap *application* and return a new WSGI application object. The returned - application will forward all requests to the original *application*, and will - check that both the *application* and the server invoking it are conforming to - the WSGI specification and to :rfc:`2616`. - - Any detected nonconformance results in an :exc:`AssertionError` being raised; - note, however, that how these errors are handled is server-dependent. For - example, :mod:`wsgiref.simple_server` and other servers based on - :mod:`wsgiref.handlers` (that don't override the error handling methods to do - something else) will simply output a message that an error has occurred, and - dump the traceback to ``sys.stderr`` or some other error stream. - - This wrapper may also generate output using the :mod:`warnings` module to - indicate behaviors that are questionable but which may not actually be - prohibited by :pep:`3333`. Unless they are suppressed using Python command-line - options or the :mod:`warnings` API, any such warnings will be written to - ``sys.stderr`` (*not* ``wsgi.errors``, unless they happen to be the same - object). - - Example usage:: - - from wsgiref.validate import validator - from wsgiref.simple_server import make_server - - # Our callable object which is intentionally not compliant to the - # standard, so the validator is going to break - def simple_app(environ, start_response): - status = '200 OK' # HTTP Status - headers = [('Content-type', 'text/plain')] # HTTP Headers - start_response(status, headers) - - # This is going to break because we need to return a list, and - # the validator is going to inform us - return b"Hello World" - - # This is the application wrapped in a validator - validator_app = validator(simple_app) - - with make_server('', 8000, validator_app) as httpd: - print("Listening on port 8000....") - httpd.serve_forever() - - -:mod:`wsgiref.handlers` -- server/gateway base classes ------------------------------------------------------- - -.. module:: wsgiref.handlers - :synopsis: WSGI server/gateway base classes. - - -This module provides base handler classes for implementing WSGI servers and -gateways. These base classes handle most of the work of communicating with a -WSGI application, as long as they are given a CGI-like environment, along with -input, output, and error streams. - - -.. class:: CGIHandler() - - CGI-based invocation via ``sys.stdin``, ``sys.stdout``, ``sys.stderr`` and - ``os.environ``. This is useful when you have a WSGI application and want to run - it as a CGI script. Simply invoke ``CGIHandler().run(app)``, where ``app`` is - the WSGI application object you wish to invoke. - - This class is a subclass of :class:`BaseCGIHandler` that sets ``wsgi.run_once`` - to true, ``wsgi.multithread`` to false, and ``wsgi.multiprocess`` to true, and - always uses :mod:`sys` and :mod:`os` to obtain the necessary CGI streams and - environment. - - -.. class:: IISCGIHandler() - - A specialized alternative to :class:`CGIHandler`, for use when deploying on - Microsoft's IIS web server, without having set the config allowPathInfo - option (IIS>=7) or metabase allowPathInfoForScriptMappings (IIS<7). - - By default, IIS gives a ``PATH_INFO`` that duplicates the ``SCRIPT_NAME`` at - the front, causing problems for WSGI applications that wish to implement - routing. This handler strips any such duplicated path. - - IIS can be configured to pass the correct ``PATH_INFO``, but this causes - another bug where ``PATH_TRANSLATED`` is wrong. Luckily this variable is - rarely used and is not guaranteed by WSGI. On IIS<7, though, the - setting can only be made on a vhost level, affecting all other script - mappings, many of which break when exposed to the ``PATH_TRANSLATED`` bug. - For this reason IIS<7 is almost never deployed with the fix. (Even IIS7 - rarely uses it because there is still no UI for it.) - - There is no way for CGI code to tell whether the option was set, so a - separate handler class is provided. It is used in the same way as - :class:`CGIHandler`, i.e., by calling ``IISCGIHandler().run(app)``, where - ``app`` is the WSGI application object you wish to invoke. - - .. versionadded:: 3.2 - - -.. class:: BaseCGIHandler(stdin, stdout, stderr, environ, multithread=True, multiprocess=False) - - Similar to :class:`CGIHandler`, but instead of using the :mod:`sys` and - :mod:`os` modules, the CGI environment and I/O streams are specified explicitly. - The *multithread* and *multiprocess* values are used to set the - ``wsgi.multithread`` and ``wsgi.multiprocess`` flags for any applications run by - the handler instance. - - This class is a subclass of :class:`SimpleHandler` intended for use with - software other than HTTP "origin servers". If you are writing a gateway - protocol implementation (such as CGI, FastCGI, SCGI, etc.) that uses a - ``Status:`` header to send an HTTP status, you probably want to subclass this - instead of :class:`SimpleHandler`. - - -.. class:: SimpleHandler(stdin, stdout, stderr, environ, multithread=True, multiprocess=False) - - Similar to :class:`BaseCGIHandler`, but designed for use with HTTP origin - servers. If you are writing an HTTP server implementation, you will probably - want to subclass this instead of :class:`BaseCGIHandler`. - - This class is a subclass of :class:`BaseHandler`. It overrides the - :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, - :meth:`_write`, and :meth:`_flush` methods to support explicitly setting the - environment and streams via the constructor. The supplied environment and - streams are stored in the :attr:`stdin`, :attr:`stdout`, :attr:`stderr`, and - :attr:`environ` attributes. - - The :meth:`~io.BufferedIOBase.write` method of *stdout* should write - each chunk in full, like :class:`io.BufferedIOBase`. - - -.. class:: BaseHandler() - - This is an abstract base class for running WSGI applications. Each instance - will handle a single HTTP request, although in principle you could create a - subclass that was reusable for multiple requests. - - :class:`BaseHandler` instances have only one method intended for external use: - - - .. method:: BaseHandler.run(app) - - Run the specified WSGI application, *app*. - - All of the other :class:`BaseHandler` methods are invoked by this method in the - process of running the application, and thus exist primarily to allow - customizing the process. - - The following methods MUST be overridden in a subclass: - - - .. method:: BaseHandler._write(data) - - Buffer the bytes *data* for transmission to the client. It's okay if this - method actually transmits the data; :class:`BaseHandler` just separates write - and flush operations for greater efficiency when the underlying system actually - has such a distinction. - - - .. method:: BaseHandler._flush() - - Force buffered data to be transmitted to the client. It's okay if this method - is a no-op (i.e., if :meth:`_write` actually sends the data). - - - .. method:: BaseHandler.get_stdin() - - Return an input stream object suitable for use as the ``wsgi.input`` of the - request currently being processed. - - - .. method:: BaseHandler.get_stderr() - - Return an output stream object suitable for use as the ``wsgi.errors`` of the - request currently being processed. - - - .. method:: BaseHandler.add_cgi_vars() - - Insert CGI variables for the current request into the :attr:`environ` attribute. - - Here are some other methods and attributes you may wish to override. This list - is only a summary, however, and does not include every method that can be - overridden. You should consult the docstrings and source code for additional - information before attempting to create a customized :class:`BaseHandler` - subclass. - - Attributes and methods for customizing the WSGI environment: - - - .. attribute:: BaseHandler.wsgi_multithread - - The value to be used for the ``wsgi.multithread`` environment variable. It - defaults to true in :class:`BaseHandler`, but may have a different default (or - be set by the constructor) in the other subclasses. - - - .. attribute:: BaseHandler.wsgi_multiprocess - - The value to be used for the ``wsgi.multiprocess`` environment variable. It - defaults to true in :class:`BaseHandler`, but may have a different default (or - be set by the constructor) in the other subclasses. - - - .. attribute:: BaseHandler.wsgi_run_once - - The value to be used for the ``wsgi.run_once`` environment variable. It - defaults to false in :class:`BaseHandler`, but :class:`CGIHandler` sets it to - true by default. - - - .. attribute:: BaseHandler.os_environ - - The default environment variables to be included in every request's WSGI - environment. By default, this is a copy of ``os.environ`` at the time that - :mod:`wsgiref.handlers` was imported, but subclasses can either create their own - at the class or instance level. Note that the dictionary should be considered - read-only, since the default value is shared between multiple classes and - instances. - - - .. attribute:: BaseHandler.server_software - - If the :attr:`origin_server` attribute is set, this attribute's value is used to - set the default ``SERVER_SOFTWARE`` WSGI environment variable, and also to set a - default ``Server:`` header in HTTP responses. It is ignored for handlers (such - as :class:`BaseCGIHandler` and :class:`CGIHandler`) that are not HTTP origin - servers. - - .. versionchanged:: 3.3 - The term "Python" is replaced with implementation specific term like - "CPython", "Jython" etc. - - .. method:: BaseHandler.get_scheme() - - Return the URL scheme being used for the current request. The default - implementation uses the :func:`guess_scheme` function from :mod:`wsgiref.util` - to guess whether the scheme should be "http" or "https", based on the current - request's :attr:`environ` variables. - - - .. method:: BaseHandler.setup_environ() - - Set the :attr:`environ` attribute to a fully-populated WSGI environment. The - default implementation uses all of the above methods and attributes, plus the - :meth:`get_stdin`, :meth:`get_stderr`, and :meth:`add_cgi_vars` methods and the - :attr:`wsgi_file_wrapper` attribute. It also inserts a ``SERVER_SOFTWARE`` key - if not present, as long as the :attr:`origin_server` attribute is a true value - and the :attr:`server_software` attribute is set. - - Methods and attributes for customizing exception handling: - - - .. method:: BaseHandler.log_exception(exc_info) - - Log the *exc_info* tuple in the server log. *exc_info* is a ``(type, value, - traceback)`` tuple. The default implementation simply writes the traceback to - the request's ``wsgi.errors`` stream and flushes it. Subclasses can override - this method to change the format or retarget the output, mail the traceback to - an administrator, or whatever other action may be deemed suitable. - - - .. attribute:: BaseHandler.traceback_limit - - The maximum number of frames to include in tracebacks output by the default - :meth:`log_exception` method. If ``None``, all frames are included. - - - .. method:: BaseHandler.error_output(environ, start_response) - - This method is a WSGI application to generate an error page for the user. It is - only invoked if an error occurs before headers are sent to the client. - - This method can access the current error information using ``sys.exc_info()``, - and should pass that information to *start_response* when calling it (as - described in the "Error Handling" section of :pep:`3333`). - - The default implementation just uses the :attr:`error_status`, - :attr:`error_headers`, and :attr:`error_body` attributes to generate an output - page. Subclasses can override this to produce more dynamic error output. - - Note, however, that it's not recommended from a security perspective to spit out - diagnostics to any old user; ideally, you should have to do something special to - enable diagnostic output, which is why the default implementation doesn't - include any. - - - .. attribute:: BaseHandler.error_status - - The HTTP status used for error responses. This should be a status string as - defined in :pep:`3333`; it defaults to a 500 code and message. - - - .. attribute:: BaseHandler.error_headers - - The HTTP headers used for error responses. This should be a list of WSGI - response headers (``(name, value)`` tuples), as described in :pep:`3333`. The - default list just sets the content type to ``text/plain``. - - - .. attribute:: BaseHandler.error_body - - The error response body. This should be an HTTP response body bytestring. It - defaults to the plain text, "A server error occurred. Please contact the - administrator." - - Methods and attributes for :pep:`3333`'s "Optional Platform-Specific File - Handling" feature: - - - .. attribute:: BaseHandler.wsgi_file_wrapper - - A ``wsgi.file_wrapper`` factory, or ``None``. The default value of this - attribute is the :class:`wsgiref.util.FileWrapper` class. - - - .. method:: BaseHandler.sendfile() - - Override to implement platform-specific file transmission. This method is - called only if the application's return value is an instance of the class - specified by the :attr:`wsgi_file_wrapper` attribute. It should return a true - value if it was able to successfully transmit the file, so that the default - transmission code will not be executed. The default implementation of this - method just returns a false value. - - Miscellaneous methods and attributes: - - - .. attribute:: BaseHandler.origin_server - - This attribute should be set to a true value if the handler's :meth:`_write` and - :meth:`_flush` are being used to communicate directly to the client, rather than - via a CGI-like gateway protocol that wants the HTTP status in a special - ``Status:`` header. - - This attribute's default value is true in :class:`BaseHandler`, but false in - :class:`BaseCGIHandler` and :class:`CGIHandler`. - - - .. attribute:: BaseHandler.http_version - - If :attr:`origin_server` is true, this string attribute is used to set the HTTP - version of the response set to the client. It defaults to ``"1.0"``. - - -.. function:: read_environ() - - Transcode CGI variables from ``os.environ`` to PEP 3333 "bytes in unicode" - strings, returning a new dictionary. This function is used by - :class:`CGIHandler` and :class:`IISCGIHandler` in place of directly using - ``os.environ``, which is not necessarily WSGI-compliant on all platforms - and web servers using Python 3 -- specifically, ones where the OS's - actual environment is Unicode (i.e. Windows), or ones where the environment - is bytes, but the system encoding used by Python to decode it is anything - other than ISO-8859-1 (e.g. Unix systems using UTF-8). - - If you are implementing a CGI-based handler of your own, you probably want - to use this routine instead of just copying values out of ``os.environ`` - directly. - - .. versionadded:: 3.2 - - -Examples --------- - -This is a working "Hello World" WSGI application:: - - from wsgiref.simple_server import make_server - - # Every WSGI application must have an application object - a callable - # object that accepts two arguments. For that purpose, we're going to - # use a function (note that you're not limited to a function, you can - # use a class for example). The first argument passed to the function - # is a dictionary containing CGI-style environment variables and the - # second variable is the callable object (see PEP 333). - def hello_world_app(environ, start_response): - status = '200 OK' # HTTP Status - headers = [('Content-type', 'text/plain; charset=utf-8')] # HTTP Headers - start_response(status, headers) - - # The returned object is going to be printed - return [b"Hello World"] - - with make_server('', 8000, hello_world_app) as httpd: - print("Serving on port 8000...") - - # Serve until process is killed - httpd.serve_forever() diff --git a/third_party/python/Doc/library/xdrlib.rst b/third_party/python/Doc/library/xdrlib.rst deleted file mode 100644 index 42a03a467..000000000 --- a/third_party/python/Doc/library/xdrlib.rst +++ /dev/null @@ -1,278 +0,0 @@ -:mod:`xdrlib` --- Encode and decode XDR data -============================================ - -.. module:: xdrlib - :synopsis: Encoders and decoders for the External Data Representation (XDR). - -**Source code:** :source:`Lib/xdrlib.py` - -.. index:: - single: XDR - single: External Data Representation - --------------- - -The :mod:`xdrlib` module supports the External Data Representation Standard as -described in :rfc:`1014`, written by Sun Microsystems, Inc. June 1987. It -supports most of the data types described in the RFC. - -The :mod:`xdrlib` module defines two classes, one for packing variables into XDR -representation, and another for unpacking from XDR representation. There are -also two exception classes. - - -.. class:: Packer() - - :class:`Packer` is the class for packing data into XDR representation. The - :class:`Packer` class is instantiated with no arguments. - - -.. class:: Unpacker(data) - - ``Unpacker`` is the complementary class which unpacks XDR data values from a - string buffer. The input buffer is given as *data*. - - -.. seealso:: - - :rfc:`1014` - XDR: External Data Representation Standard - This RFC defined the encoding of data which was XDR at the time this module was - originally written. It has apparently been obsoleted by :rfc:`1832`. - - :rfc:`1832` - XDR: External Data Representation Standard - Newer RFC that provides a revised definition of XDR. - - -.. _xdr-packer-objects: - -Packer Objects --------------- - -:class:`Packer` instances have the following methods: - - -.. method:: Packer.get_buffer() - - Returns the current pack buffer as a string. - - -.. method:: Packer.reset() - - Resets the pack buffer to the empty string. - -In general, you can pack any of the most common XDR data types by calling the -appropriate ``pack_type()`` method. Each method takes a single argument, the -value to pack. The following simple data type packing methods are supported: -:meth:`pack_uint`, :meth:`pack_int`, :meth:`pack_enum`, :meth:`pack_bool`, -:meth:`pack_uhyper`, and :meth:`pack_hyper`. - - -.. method:: Packer.pack_float(value) - - Packs the single-precision floating point number *value*. - - -.. method:: Packer.pack_double(value) - - Packs the double-precision floating point number *value*. - -The following methods support packing strings, bytes, and opaque data: - - -.. method:: Packer.pack_fstring(n, s) - - Packs a fixed length string, *s*. *n* is the length of the string but it is - *not* packed into the data buffer. The string is padded with null bytes if - necessary to guaranteed 4 byte alignment. - - -.. method:: Packer.pack_fopaque(n, data) - - Packs a fixed length opaque data stream, similarly to :meth:`pack_fstring`. - - -.. method:: Packer.pack_string(s) - - Packs a variable length string, *s*. The length of the string is first packed - as an unsigned integer, then the string data is packed with - :meth:`pack_fstring`. - - -.. method:: Packer.pack_opaque(data) - - Packs a variable length opaque data string, similarly to :meth:`pack_string`. - - -.. method:: Packer.pack_bytes(bytes) - - Packs a variable length byte stream, similarly to :meth:`pack_string`. - -The following methods support packing arrays and lists: - - -.. method:: Packer.pack_list(list, pack_item) - - Packs a *list* of homogeneous items. This method is useful for lists with an - indeterminate size; i.e. the size is not available until the entire list has - been walked. For each item in the list, an unsigned integer ``1`` is packed - first, followed by the data value from the list. *pack_item* is the function - that is called to pack the individual item. At the end of the list, an unsigned - integer ``0`` is packed. - - For example, to pack a list of integers, the code might appear like this:: - - import xdrlib - p = xdrlib.Packer() - p.pack_list([1, 2, 3], p.pack_int) - - -.. method:: Packer.pack_farray(n, array, pack_item) - - Packs a fixed length list (*array*) of homogeneous items. *n* is the length of - the list; it is *not* packed into the buffer, but a :exc:`ValueError` exception - is raised if ``len(array)`` is not equal to *n*. As above, *pack_item* is the - function used to pack each element. - - -.. method:: Packer.pack_array(list, pack_item) - - Packs a variable length *list* of homogeneous items. First, the length of the - list is packed as an unsigned integer, then each element is packed as in - :meth:`pack_farray` above. - - -.. _xdr-unpacker-objects: - -Unpacker Objects ----------------- - -The :class:`Unpacker` class offers the following methods: - - -.. method:: Unpacker.reset(data) - - Resets the string buffer with the given *data*. - - -.. method:: Unpacker.get_position() - - Returns the current unpack position in the data buffer. - - -.. method:: Unpacker.set_position(position) - - Sets the data buffer unpack position to *position*. You should be careful about - using :meth:`get_position` and :meth:`set_position`. - - -.. method:: Unpacker.get_buffer() - - Returns the current unpack data buffer as a string. - - -.. method:: Unpacker.done() - - Indicates unpack completion. Raises an :exc:`Error` exception if all of the - data has not been unpacked. - -In addition, every data type that can be packed with a :class:`Packer`, can be -unpacked with an :class:`Unpacker`. Unpacking methods are of the form -``unpack_type()``, and take no arguments. They return the unpacked object. - - -.. method:: Unpacker.unpack_float() - - Unpacks a single-precision floating point number. - - -.. method:: Unpacker.unpack_double() - - Unpacks a double-precision floating point number, similarly to - :meth:`unpack_float`. - -In addition, the following methods unpack strings, bytes, and opaque data: - - -.. method:: Unpacker.unpack_fstring(n) - - Unpacks and returns a fixed length string. *n* is the number of characters - expected. Padding with null bytes to guaranteed 4 byte alignment is assumed. - - -.. method:: Unpacker.unpack_fopaque(n) - - Unpacks and returns a fixed length opaque data stream, similarly to - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_string() - - Unpacks and returns a variable length string. The length of the string is first - unpacked as an unsigned integer, then the string data is unpacked with - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_opaque() - - Unpacks and returns a variable length opaque data string, similarly to - :meth:`unpack_string`. - - -.. method:: Unpacker.unpack_bytes() - - Unpacks and returns a variable length byte stream, similarly to - :meth:`unpack_string`. - -The following methods support unpacking arrays and lists: - - -.. method:: Unpacker.unpack_list(unpack_item) - - Unpacks and returns a list of homogeneous items. The list is unpacked one - element at a time by first unpacking an unsigned integer flag. If the flag is - ``1``, then the item is unpacked and appended to the list. A flag of ``0`` - indicates the end of the list. *unpack_item* is the function that is called to - unpack the items. - - -.. method:: Unpacker.unpack_farray(n, unpack_item) - - Unpacks and returns (as a list) a fixed length array of homogeneous items. *n* - is number of list elements to expect in the buffer. As above, *unpack_item* is - the function used to unpack each element. - - -.. method:: Unpacker.unpack_array(unpack_item) - - Unpacks and returns a variable length *list* of homogeneous items. First, the - length of the list is unpacked as an unsigned integer, then each element is - unpacked as in :meth:`unpack_farray` above. - - -.. _xdr-exceptions: - -Exceptions ----------- - -Exceptions in this module are coded as class instances: - - -.. exception:: Error - - The base exception class. :exc:`Error` has a single public attribute - :attr:`msg` containing the description of the error. - - -.. exception:: ConversionError - - Class derived from :exc:`Error`. Contains no additional instance variables. - -Here is an example of how you would catch one of these exceptions:: - - import xdrlib - p = xdrlib.Packer() - try: - p.pack_double(8.01) - except xdrlib.ConversionError as instance: - print('packing the double failed:', instance.msg) - diff --git a/third_party/python/Doc/library/xml.dom.minidom.rst b/third_party/python/Doc/library/xml.dom.minidom.rst deleted file mode 100644 index 15b1cb0cb..000000000 --- a/third_party/python/Doc/library/xml.dom.minidom.rst +++ /dev/null @@ -1,256 +0,0 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== - -.. module:: xml.dom.minidom - :synopsis: Minimal Document Object Model (DOM) implementation. - -.. moduleauthor:: Paul Prescod <paul@prescod.net> -.. sectionauthor:: Paul Prescod <paul@prescod.net> -.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> - -**Source code:** :source:`Lib/xml/dom/minidom.py` - --------------- - -:mod:`xml.dom.minidom` is a minimal implementation of the Document Object -Model interface, with an API similar to that in other languages. It is intended -to be simpler than the full DOM and also significantly smaller. Users who are -not already proficient with the DOM should consider using the -:mod:`xml.etree.ElementTree` module for their XML processing instead. - - -.. warning:: - - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - - -DOM applications typically start by parsing some XML into a DOM. With -:mod:`xml.dom.minidom`, this is done through the parse functions:: - - from xml.dom.minidom import parse, parseString - - dom1 = parse('c:\\temp\\mydata.xml') # parse an XML file by name - - datasource = open('c:\\temp\\mydata.xml') - dom2 = parse(datasource) # parse an open file - - dom3 = parseString('<myxml>Some data<empty/> some more data</myxml>') - -The :func:`parse` function can take either a filename or an open file object. - - -.. function:: parse(filename_or_file, parser=None, bufsize=None) - - Return a :class:`Document` from the given input. *filename_or_file* may be - either a file name, or a file-like object. *parser*, if given, must be a SAX2 - parser object. This function will change the document handler of the parser and - activate namespace support; other parser configuration (like setting an entity - resolver) must have been done in advance. - -If you have XML in a string, you can use the :func:`parseString` function -instead: - - -.. function:: parseString(string, parser=None) - - Return a :class:`Document` that represents the *string*. This method creates an - :class:`io.StringIO` object for the string and passes that on to :func:`parse`. - -Both functions return a :class:`Document` object representing the content of the -document. - -What the :func:`parse` and :func:`parseString` functions do is connect an XML -parser with a "DOM builder" that can accept parse events from any SAX parser and -convert them into a DOM tree. The name of the functions are perhaps misleading, -but are easy to grasp when learning the interfaces. The parsing of the document -will be completed before these functions return; it's simply that these -functions do not provide a parser implementation themselves. - -You can also create a :class:`Document` by calling a method on a "DOM -Implementation" object. You can get this object either by calling the -:func:`getDOMImplementation` function in the :mod:`xml.dom` package or the -:mod:`xml.dom.minidom` module. Once you have a :class:`Document`, you -can add child nodes to it to populate the DOM:: - - from xml.dom.minidom import getDOMImplementation - - impl = getDOMImplementation() - - newdoc = impl.createDocument(None, "some_tag", None) - top_element = newdoc.documentElement - text = newdoc.createTextNode('Some textual content.') - top_element.appendChild(text) - -Once you have a DOM document object, you can access the parts of your XML -document through its properties and methods. These properties are defined in -the DOM specification. The main property of the document object is the -:attr:`documentElement` property. It gives you the main element in the XML -document: the one that holds all others. Here is an example program:: - - dom3 = parseString("<myxml>Some data</myxml>") - assert dom3.documentElement.tagName == "myxml" - -When you are finished with a DOM tree, you may optionally call the -:meth:`unlink` method to encourage early cleanup of the now-unneeded -objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific -extension to the DOM API that renders the node and its descendants are -essentially useless. Otherwise, Python's garbage collector will -eventually take care of the objects in the tree. - -.. seealso:: - - `Document Object Model (DOM) Level 1 Specification <https://www.w3.org/TR/REC-DOM-Level-1/>`_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - - -.. _minidom-objects: - -DOM Objects ------------ - -The definition of the DOM API for Python is given as part of the :mod:`xml.dom` -module documentation. This section lists the differences between the API and -:mod:`xml.dom.minidom`. - - -.. method:: Node.unlink() - - Break internal references within the DOM so that it will be garbage collected on - versions of Python without cyclic GC. Even when cyclic GC is available, using - this can make large amounts of memory available sooner, so calling this on DOM - objects as soon as they are no longer needed is good practice. This only needs - to be called on the :class:`Document` object, but may be called on child nodes - to discard children of that node. - - You can avoid calling this method explicitly by using the :keyword:`with` - statement. The following code will automatically unlink *dom* when the - :keyword:`with` block is exited:: - - with xml.dom.minidom.parse(datasource) as dom: - ... # Work with dom. - - -.. method:: Node.writexml(writer, indent="", addindent="", newl="") - - Write XML to the writer object. The writer should have a :meth:`write` method - which matches that of the file object interface. The *indent* parameter is the - indentation of the current node. The *addindent* parameter is the incremental - indentation to use for subnodes of the current one. The *newl* parameter - specifies the string to use to terminate newlines. - - For the :class:`Document` node, an additional keyword argument *encoding* can - be used to specify the encoding field of the XML header. - - -.. method:: Node.toxml(encoding=None) - - Return a string or byte string containing the XML represented by - the DOM node. - - With an explicit *encoding* [1]_ argument, the result is a byte - string in the specified encoding. - With no *encoding* argument, the result is a Unicode string, and the - XML declaration in the resulting string does not specify an - encoding. Encoding this string in an encoding other than UTF-8 is - likely incorrect, since UTF-8 is the default encoding of XML. - -.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None) - - Return a pretty-printed version of the document. *indent* specifies the - indentation string and defaults to a tabulator; *newl* specifies the string - emitted at the end of each line and defaults to ``\n``. - - The *encoding* argument behaves like the corresponding argument of - :meth:`toxml`. - - -.. _dom-example: - -DOM Example ------------ - -This example program is a fairly realistic example of a simple program. In this -particular case, we do not take much advantage of the flexibility of the DOM. - -.. literalinclude:: ../includes/minidom-example.py - - -.. _minidom-and-dom: - -minidom and the DOM standard ----------------------------- - -The :mod:`xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with -some DOM 2 features (primarily namespace features). - -Usage of the DOM interface in Python is straight-forward. The following mapping -rules apply: - -* Interfaces are accessed through instance objects. Applications should not - instantiate the classes themselves; they should use the creator functions - available on the :class:`Document` object. Derived interfaces support all - operations (and attributes) from the base interfaces, plus any new operations. - -* Operations are used as methods. Since the DOM uses only :keyword:`in` - parameters, the arguments are passed in normal order (from left to right). - There are no optional arguments. ``void`` operations return ``None``. - -* IDL attributes map to instance attributes. For compatibility with the OMG IDL - language mapping for Python, an attribute ``foo`` can also be accessed through - accessor methods :meth:`_get_foo` and :meth:`_set_foo`. ``readonly`` - attributes must not be changed; this is not enforced at runtime. - -* The types ``short int``, ``unsigned int``, ``unsigned long long``, and - ``boolean`` all map to Python integer objects. - -* The type ``DOMString`` maps to Python strings. :mod:`xml.dom.minidom` supports - either bytes or strings, but will normally produce strings. - Values of type ``DOMString`` may also be ``None`` where allowed to have the IDL - ``null`` value by the DOM specification from the W3C. - -* ``const`` declarations map to variables in their respective scope (e.g. - ``xml.dom.minidom.Node.PROCESSING_INSTRUCTION_NODE``); they must not be changed. - -* ``DOMException`` is currently not supported in :mod:`xml.dom.minidom`. - Instead, :mod:`xml.dom.minidom` uses standard Python exceptions such as - :exc:`TypeError` and :exc:`AttributeError`. - -* :class:`NodeList` objects are implemented using Python's built-in list type. - These objects provide the interface defined in the DOM specification, but with - earlier versions of Python they do not support the official API. They are, - however, much more "Pythonic" than the interface defined in the W3C - recommendations. - -The following interfaces have no implementation in :mod:`xml.dom.minidom`: - -* :class:`DOMTimeStamp` - -* :class:`DocumentType` - -* :class:`DOMImplementation` - -* :class:`CharacterData` - -* :class:`CDATASection` - -* :class:`Notation` - -* :class:`Entity` - -* :class:`EntityReference` - -* :class:`DocumentFragment` - -Most of these reflect information in the XML document that is not of general -utility to most DOM users. - -.. rubric:: Footnotes - -.. [1] The encoding name included in the XML output should conform to - the appropriate standards. For example, "UTF-8" is valid, but - "UTF8" is not valid in an XML document's declaration, even though - Python accepts it as an encoding name. - See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/third_party/python/Doc/library/xml.dom.pulldom.rst b/third_party/python/Doc/library/xml.dom.pulldom.rst deleted file mode 100644 index 2504409e7..000000000 --- a/third_party/python/Doc/library/xml.dom.pulldom.rst +++ /dev/null @@ -1,145 +0,0 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= - -.. module:: xml.dom.pulldom - :synopsis: Support for building partial DOM trees from SAX events. - -.. moduleauthor:: Paul Prescod <paul@prescod.net> - -**Source code:** :source:`Lib/xml/dom/pulldom.py` - --------------- - -The :mod:`xml.dom.pulldom` module provides a "pull parser" which can also be -asked to produce DOM-accessible fragments of the document where necessary. The -basic concept involves pulling "events" from a stream of incoming XML and -processing them. In contrast to SAX which also employs an event-driven -processing model together with callbacks, the user of a pull parser is -responsible for explicitly pulling events from the stream, looping over those -events until either processing is finished or an error condition occurs. - - -.. warning:: - - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.6.7 - - The SAX parser no longer processes general external entities by default to - increase security by default. To enable processing of external entities, - pass a custom parser instance in:: - - from xml.dom.pulldom import parse - from xml.sax import make_parser - from xml.sax.handler import feature_external_ges - - parser = make_parser() - parser.setFeature(feature_external_ges, True) - parse(filename, parser=parser) - - -Example:: - - from xml.dom import pulldom - - doc = pulldom.parse('sales_items.xml') - for event, node in doc: - if event == pulldom.START_ELEMENT and node.tagName == 'item': - if int(node.getAttribute('price')) > 50: - doc.expandNode(node) - print(node.toxml()) - -``event`` is a constant and can be one of: - -* :data:`START_ELEMENT` -* :data:`END_ELEMENT` -* :data:`COMMENT` -* :data:`START_DOCUMENT` -* :data:`END_DOCUMENT` -* :data:`CHARACTERS` -* :data:`PROCESSING_INSTRUCTION` -* :data:`IGNORABLE_WHITESPACE` - -``node`` is an object of type :class:`xml.dom.minidom.Document`, -:class:`xml.dom.minidom.Element` or :class:`xml.dom.minidom.Text`. - -Since the document is treated as a "flat" stream of events, the document "tree" -is implicitly traversed and the desired elements are found regardless of their -depth in the tree. In other words, one does not need to consider hierarchical -issues such as recursive searching of the document nodes, although if the -context of elements were important, one would either need to maintain some -context-related state (i.e. remembering where one is in the document at any -given point) or to make use of the :func:`DOMEventStream.expandNode` method -and switch to DOM-related processing. - - -.. class:: PullDom(documentFactory=None) - - Subclass of :class:`xml.sax.handler.ContentHandler`. - - -.. class:: SAX2DOM(documentFactory=None) - - Subclass of :class:`xml.sax.handler.ContentHandler`. - - -.. function:: parse(stream_or_string, parser=None, bufsize=None) - - Return a :class:`DOMEventStream` from the given input. *stream_or_string* may be - either a file name, or a file-like object. *parser*, if given, must be an - :class:`~xml.sax.xmlreader.XMLReader` object. This function will change the - document handler of the - parser and activate namespace support; other parser configuration (like - setting an entity resolver) must have been done in advance. - -If you have XML in a string, you can use the :func:`parseString` function instead: - -.. function:: parseString(string, parser=None) - - Return a :class:`DOMEventStream` that represents the (Unicode) *string*. - -.. data:: default_bufsize - - Default value for the *bufsize* parameter to :func:`parse`. - - The value of this variable can be changed before calling :func:`parse` and - the new value will take effect. - -.. _domeventstream-objects: - -DOMEventStream Objects ----------------------- - -.. class:: DOMEventStream(stream, parser, bufsize) - - - .. method:: getEvent() - - Return a tuple containing *event* and the current *node* as - :class:`xml.dom.minidom.Document` if event equals :data:`START_DOCUMENT`, - :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or - :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals - :data:`CHARACTERS`. - The current node does not contain information about its children, unless - :func:`expandNode` is called. - - .. method:: expandNode(node) - - Expands all children of *node* into *node*. Example:: - - from xml.dom import pulldom - - xml = '<html><title>Foo

    Some text

    and more

    ' - doc = pulldom.parseString(xml) - for event, node in doc: - if event == pulldom.START_ELEMENT and node.tagName == 'p': - # Following statement only prints '

    ' - print(node.toxml()) - doc.expandNode(node) - # Following statement prints node with all its children '

    Some text

    and more

    ' - print(node.toxml()) - - .. method:: DOMEventStream.reset() - diff --git a/third_party/python/Doc/library/xml.dom.rst b/third_party/python/Doc/library/xml.dom.rst deleted file mode 100644 index de334af23..000000000 --- a/third_party/python/Doc/library/xml.dom.rst +++ /dev/null @@ -1,1033 +0,0 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ - -.. module:: xml.dom - :synopsis: Document Object Model API for Python. - -.. sectionauthor:: Paul Prescod -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/dom/__init__.py` - --------------- - -The Document Object Model, or "DOM," is a cross-language API from the World Wide -Web Consortium (W3C) for accessing and modifying XML documents. A DOM -implementation presents an XML document as a tree structure, or allows client -code to build such a structure from scratch. It then gives access to the -structure through a set of objects which provided well-known interfaces. - -The DOM is extremely useful for random-access applications. SAX only allows you -a view of one bit of the document at a time. If you are looking at one SAX -element, you have no access to another. If you are looking at a text node, you -have no access to a containing element. When you write a SAX application, you -need to keep track of your program's position in the document somewhere in your -own code. SAX does not do it for you. Also, if you need to look ahead in the -XML document, you are just out of luck. - -Some applications are simply impossible in an event driven model with no access -to a tree. Of course you could build some sort of tree yourself in SAX events, -but the DOM allows you to avoid writing that code. The DOM is a standard tree -representation for XML data. - -The Document Object Model is being defined by the W3C in stages, or "levels" in -their terminology. The Python mapping of the API is substantially based on the -DOM Level 2 recommendation. - -.. What if your needs are somewhere between SAX and the DOM? Perhaps - you cannot afford to load the entire tree in memory but you find the - SAX model somewhat cumbersome and low-level. There is also a module - called xml.dom.pulldom that allows you to build trees of only the - parts of a document that you need structured access to. It also has - features that allow you to find your way around the DOM. - See http://www.prescod.net/python/pulldom - -DOM applications typically start by parsing some XML into a DOM. How this is -accomplished is not covered at all by DOM Level 1, and Level 2 provides only -limited improvements: There is a :class:`DOMImplementation` object class which -provides access to :class:`Document` creation methods, but no way to access an -XML reader/parser/Document builder in an implementation-independent way. There -is also no well-defined way to access these methods without an existing -:class:`Document` object. In Python, each DOM implementation will provide a -function :func:`getDOMImplementation`. DOM Level 3 adds a Load/Store -specification, which defines an interface to the reader, but this is not yet -available in the Python standard library. - -Once you have a DOM document object, you can access the parts of your XML -document through its properties and methods. These properties are defined in -the DOM specification; this portion of the reference manual describes the -interpretation of the specification in Python. - -The specification provided by the W3C defines the DOM API for Java, ECMAScript, -and OMG IDL. The Python mapping defined here is based in large part on the IDL -version of the specification, but strict compliance is not required (though -implementations are free to support the strict mapping from IDL). See section -:ref:`dom-conformance` for a detailed discussion of mapping requirements. - - -.. seealso:: - - `Document Object Model (DOM) Level 2 Specification `_ - The W3C recommendation upon which the Python DOM API is based. - - `Document Object Model (DOM) Level 1 Specification `_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - - `Python Language Mapping Specification `_ - This specifies the mapping from OMG IDL to Python. - - -Module Contents ---------------- - -The :mod:`xml.dom` contains the following functions: - - -.. function:: registerDOMImplementation(name, factory) - - Register the *factory* function with the name *name*. The factory function - should return an object which implements the :class:`DOMImplementation` - interface. The factory function can return the same object every time, or a new - one for each call, as appropriate for the specific implementation (e.g. if that - implementation supports some customization). - - -.. function:: getDOMImplementation(name=None, features=()) - - Return a suitable DOM implementation. The *name* is either well-known, the - module name of a DOM implementation, or ``None``. If it is not ``None``, imports - the corresponding module and returns a :class:`DOMImplementation` object if the - import succeeds. If no name is given, and if the environment variable - :envvar:`PYTHON_DOM` is set, this variable is used to find the implementation. - - If name is not given, this examines the available implementations to find one - with the required feature set. If no implementation can be found, raise an - :exc:`ImportError`. The features list must be a sequence of ``(feature, - version)`` pairs which are passed to the :meth:`hasFeature` method on available - :class:`DOMImplementation` objects. - -Some convenience constants are also provided: - - -.. data:: EMPTY_NAMESPACE - - The value used to indicate that no namespace is associated with a node in the - DOM. This is typically found as the :attr:`namespaceURI` of a node, or used as - the *namespaceURI* parameter to a namespaces-specific method. - - -.. data:: XML_NAMESPACE - - The namespace URI associated with the reserved prefix ``xml``, as defined by - `Namespaces in XML `_ (section 4). - - -.. data:: XMLNS_NAMESPACE - - The namespace URI for namespace declarations, as defined by `Document Object - Model (DOM) Level 2 Core Specification - `_ (section 1.1.8). - - -.. data:: XHTML_NAMESPACE - - The URI of the XHTML namespace as defined by `XHTML 1.0: The Extensible - HyperText Markup Language `_ (section 3.1.1). - - -In addition, :mod:`xml.dom` contains a base :class:`Node` class and the DOM -exception classes. The :class:`Node` class provided by this module does not -implement any of the methods or attributes defined by the DOM specification; -concrete DOM implementations must provide those. The :class:`Node` class -provided as part of this module does provide the constants used for the -:attr:`nodeType` attribute on concrete :class:`Node` objects; they are located -within the class rather than at the module level to conform with the DOM -specifications. - -.. Should the Node documentation go here? - - -.. _dom-objects: - -Objects in the DOM ------------------- - -The definitive documentation for the DOM is the DOM specification from the W3C. - -Note that DOM attributes may also be manipulated as nodes instead of as simple -strings. It is fairly rare that you must do this, however, so this usage is not -yet documented. - -+--------------------------------+-----------------------------------+---------------------------------+ -| Interface | Section | Purpose | -+================================+===================================+=================================+ -| :class:`DOMImplementation` | :ref:`dom-implementation-objects` | Interface to the underlying | -| | | implementation. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Node` | :ref:`dom-node-objects` | Base interface for most objects | -| | | in a document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`NodeList` | :ref:`dom-nodelist-objects` | Interface for a sequence of | -| | | nodes. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`DocumentType` | :ref:`dom-documenttype-objects` | Information about the | -| | | declarations needed to process | -| | | a document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Document` | :ref:`dom-document-objects` | Object which represents an | -| | | entire document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Element` | :ref:`dom-element-objects` | Element nodes in the document | -| | | hierarchy. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Attr` | :ref:`dom-attr-objects` | Attribute value nodes on | -| | | element nodes. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Comment` | :ref:`dom-comment-objects` | Representation of comments in | -| | | the source document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Text` | :ref:`dom-text-objects` | Nodes containing textual | -| | | content from the document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`ProcessingInstruction` | :ref:`dom-pi-objects` | Processing instruction | -| | | representation. | -+--------------------------------+-----------------------------------+---------------------------------+ - -An additional section describes the exceptions defined for working with the DOM -in Python. - - -.. _dom-implementation-objects: - -DOMImplementation Objects -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`DOMImplementation` interface provides a way for applications to -determine the availability of particular features in the DOM they are using. -DOM Level 2 added the ability to create new :class:`Document` and -:class:`DocumentType` objects using the :class:`DOMImplementation` as well. - - -.. method:: DOMImplementation.hasFeature(feature, version) - - Return true if the feature identified by the pair of strings *feature* and - *version* is implemented. - - -.. method:: DOMImplementation.createDocument(namespaceUri, qualifiedName, doctype) - - Return a new :class:`Document` object (the root of the DOM), with a child - :class:`Element` object having the given *namespaceUri* and *qualifiedName*. The - *doctype* must be a :class:`DocumentType` object created by - :meth:`createDocumentType`, or ``None``. In the Python DOM API, the first two - arguments can also be ``None`` in order to indicate that no :class:`Element` - child is to be created. - - -.. method:: DOMImplementation.createDocumentType(qualifiedName, publicId, systemId) - - Return a new :class:`DocumentType` object that encapsulates the given - *qualifiedName*, *publicId*, and *systemId* strings, representing the - information contained in an XML document type declaration. - - -.. _dom-node-objects: - -Node Objects -^^^^^^^^^^^^ - -All of the components of an XML document are subclasses of :class:`Node`. - - -.. attribute:: Node.nodeType - - An integer representing the node type. Symbolic constants for the types are on - the :class:`Node` object: :const:`ELEMENT_NODE`, :const:`ATTRIBUTE_NODE`, - :const:`TEXT_NODE`, :const:`CDATA_SECTION_NODE`, :const:`ENTITY_NODE`, - :const:`PROCESSING_INSTRUCTION_NODE`, :const:`COMMENT_NODE`, - :const:`DOCUMENT_NODE`, :const:`DOCUMENT_TYPE_NODE`, :const:`NOTATION_NODE`. - This is a read-only attribute. - - -.. attribute:: Node.parentNode - - The parent of the current node, or ``None`` for the document node. The value is - always a :class:`Node` object or ``None``. For :class:`Element` nodes, this - will be the parent element, except for the root element, in which case it will - be the :class:`Document` object. For :class:`Attr` nodes, this is always - ``None``. This is a read-only attribute. - - -.. attribute:: Node.attributes - - A :class:`NamedNodeMap` of attribute objects. Only elements have actual values - for this; others provide ``None`` for this attribute. This is a read-only - attribute. - - -.. attribute:: Node.previousSibling - - The node that immediately precedes this one with the same parent. For - instance the element with an end-tag that comes just before the *self* - element's start-tag. Of course, XML documents are made up of more than just - elements so the previous sibling could be text, a comment, or something else. - If this node is the first child of the parent, this attribute will be - ``None``. This is a read-only attribute. - - -.. attribute:: Node.nextSibling - - The node that immediately follows this one with the same parent. See also - :attr:`previousSibling`. If this is the last child of the parent, this - attribute will be ``None``. This is a read-only attribute. - - -.. attribute:: Node.childNodes - - A list of nodes contained within this node. This is a read-only attribute. - - -.. attribute:: Node.firstChild - - The first child of the node, if there are any, or ``None``. This is a read-only - attribute. - - -.. attribute:: Node.lastChild - - The last child of the node, if there are any, or ``None``. This is a read-only - attribute. - - -.. attribute:: Node.localName - - The part of the :attr:`tagName` following the colon if there is one, else the - entire :attr:`tagName`. The value is a string. - - -.. attribute:: Node.prefix - - The part of the :attr:`tagName` preceding the colon if there is one, else the - empty string. The value is a string, or ``None``. - - -.. attribute:: Node.namespaceURI - - The namespace associated with the element name. This will be a string or - ``None``. This is a read-only attribute. - - -.. attribute:: Node.nodeName - - This has a different meaning for each node type; see the DOM specification for - details. You can always get the information you would get here from another - property such as the :attr:`tagName` property for elements or the :attr:`name` - property for attributes. For all node types, the value of this attribute will be - either a string or ``None``. This is a read-only attribute. - - -.. attribute:: Node.nodeValue - - This has a different meaning for each node type; see the DOM specification for - details. The situation is similar to that with :attr:`nodeName`. The value is - a string or ``None``. - - -.. method:: Node.hasAttributes() - - Returns true if the node has any attributes. - - -.. method:: Node.hasChildNodes() - - Returns true if the node has any child nodes. - - -.. method:: Node.isSameNode(other) - - Returns true if *other* refers to the same node as this node. This is especially - useful for DOM implementations which use any sort of proxy architecture (because - more than one object can refer to the same node). - - .. note:: - - This is based on a proposed DOM Level 3 API which is still in the "working - draft" stage, but this particular interface appears uncontroversial. Changes - from the W3C will not necessarily affect this method in the Python DOM interface - (though any new W3C API for this would also be supported). - - -.. method:: Node.appendChild(newChild) - - Add a new child node to this node at the end of the list of - children, returning *newChild*. If the node was already in - the tree, it is removed first. - - -.. method:: Node.insertBefore(newChild, refChild) - - Insert a new child node before an existing child. It must be the case that - *refChild* is a child of this node; if not, :exc:`ValueError` is raised. - *newChild* is returned. If *refChild* is ``None``, it inserts *newChild* at the - end of the children's list. - - -.. method:: Node.removeChild(oldChild) - - Remove a child node. *oldChild* must be a child of this node; if not, - :exc:`ValueError` is raised. *oldChild* is returned on success. If *oldChild* - will not be used further, its :meth:`unlink` method should be called. - - -.. method:: Node.replaceChild(newChild, oldChild) - - Replace an existing node with a new node. It must be the case that *oldChild* - is a child of this node; if not, :exc:`ValueError` is raised. - - -.. method:: Node.normalize() - - Join adjacent text nodes so that all stretches of text are stored as single - :class:`Text` instances. This simplifies processing text from a DOM tree for - many applications. - - -.. method:: Node.cloneNode(deep) - - Clone this node. Setting *deep* means to clone all child nodes as well. This - returns the clone. - - -.. _dom-nodelist-objects: - -NodeList Objects -^^^^^^^^^^^^^^^^ - -A :class:`NodeList` represents a sequence of nodes. These objects are used in -two ways in the DOM Core recommendation: an :class:`Element` object provides -one as its list of child nodes, and the :meth:`getElementsByTagName` and -:meth:`getElementsByTagNameNS` methods of :class:`Node` return objects with this -interface to represent query results. - -The DOM Level 2 recommendation defines one method and one attribute for these -objects: - - -.. method:: NodeList.item(i) - - Return the *i*'th item from the sequence, if there is one, or ``None``. The - index *i* is not allowed to be less than zero or greater than or equal to the - length of the sequence. - - -.. attribute:: NodeList.length - - The number of nodes in the sequence. - -In addition, the Python DOM interface requires that some additional support is -provided to allow :class:`NodeList` objects to be used as Python sequences. All -:class:`NodeList` implementations must include support for -:meth:`~object.__len__` and -:meth:`~object.__getitem__`; this allows iteration over the :class:`NodeList` in -:keyword:`for` statements and proper support for the :func:`len` built-in -function. - -If a DOM implementation supports modification of the document, the -:class:`NodeList` implementation must also support the -:meth:`~object.__setitem__` and :meth:`~object.__delitem__` methods. - - -.. _dom-documenttype-objects: - -DocumentType Objects -^^^^^^^^^^^^^^^^^^^^ - -Information about the notations and entities declared by a document (including -the external subset if the parser uses it and can provide the information) is -available from a :class:`DocumentType` object. The :class:`DocumentType` for a -document is available from the :class:`Document` object's :attr:`doctype` -attribute; if there is no ``DOCTYPE`` declaration for the document, the -document's :attr:`doctype` attribute will be set to ``None`` instead of an -instance of this interface. - -:class:`DocumentType` is a specialization of :class:`Node`, and adds the -following attributes: - - -.. attribute:: DocumentType.publicId - - The public identifier for the external subset of the document type definition. - This will be a string or ``None``. - - -.. attribute:: DocumentType.systemId - - The system identifier for the external subset of the document type definition. - This will be a URI as a string, or ``None``. - - -.. attribute:: DocumentType.internalSubset - - A string giving the complete internal subset from the document. This does not - include the brackets which enclose the subset. If the document has no internal - subset, this should be ``None``. - - -.. attribute:: DocumentType.name - - The name of the root element as given in the ``DOCTYPE`` declaration, if - present. - - -.. attribute:: DocumentType.entities - - This is a :class:`NamedNodeMap` giving the definitions of external entities. - For entity names defined more than once, only the first definition is provided - (others are ignored as required by the XML recommendation). This may be - ``None`` if the information is not provided by the parser, or if no entities are - defined. - - -.. attribute:: DocumentType.notations - - This is a :class:`NamedNodeMap` giving the definitions of notations. For - notation names defined more than once, only the first definition is provided - (others are ignored as required by the XML recommendation). This may be - ``None`` if the information is not provided by the parser, or if no notations - are defined. - - -.. _dom-document-objects: - -Document Objects -^^^^^^^^^^^^^^^^ - -A :class:`Document` represents an entire XML document, including its constituent -elements, attributes, processing instructions, comments etc. Remember that it -inherits properties from :class:`Node`. - - -.. attribute:: Document.documentElement - - The one and only root element of the document. - - -.. method:: Document.createElement(tagName) - - Create and return a new element node. The element is not inserted into the - document when it is created. You need to explicitly insert it with one of the - other methods such as :meth:`insertBefore` or :meth:`appendChild`. - - -.. method:: Document.createElementNS(namespaceURI, tagName) - - Create and return a new element with a namespace. The *tagName* may have a - prefix. The element is not inserted into the document when it is created. You - need to explicitly insert it with one of the other methods such as - :meth:`insertBefore` or :meth:`appendChild`. - - -.. method:: Document.createTextNode(data) - - Create and return a text node containing the data passed as a parameter. As - with the other creation methods, this one does not insert the node into the - tree. - - -.. method:: Document.createComment(data) - - Create and return a comment node containing the data passed as a parameter. As - with the other creation methods, this one does not insert the node into the - tree. - - -.. method:: Document.createProcessingInstruction(target, data) - - Create and return a processing instruction node containing the *target* and - *data* passed as parameters. As with the other creation methods, this one does - not insert the node into the tree. - - -.. method:: Document.createAttribute(name) - - Create and return an attribute node. This method does not associate the - attribute node with any particular element. You must use - :meth:`setAttributeNode` on the appropriate :class:`Element` object to use the - newly created attribute instance. - - -.. method:: Document.createAttributeNS(namespaceURI, qualifiedName) - - Create and return an attribute node with a namespace. The *tagName* may have a - prefix. This method does not associate the attribute node with any particular - element. You must use :meth:`setAttributeNode` on the appropriate - :class:`Element` object to use the newly created attribute instance. - - -.. method:: Document.getElementsByTagName(tagName) - - Search for all descendants (direct children, children's children, etc.) with a - particular element type name. - - -.. method:: Document.getElementsByTagNameNS(namespaceURI, localName) - - Search for all descendants (direct children, children's children, etc.) with a - particular namespace URI and localname. The localname is the part of the - namespace after the prefix. - - -.. _dom-element-objects: - -Element Objects -^^^^^^^^^^^^^^^ - -:class:`Element` is a subclass of :class:`Node`, so inherits all the attributes -of that class. - - -.. attribute:: Element.tagName - - The element type name. In a namespace-using document it may have colons in it. - The value is a string. - - -.. method:: Element.getElementsByTagName(tagName) - - Same as equivalent method in the :class:`Document` class. - - -.. method:: Element.getElementsByTagNameNS(namespaceURI, localName) - - Same as equivalent method in the :class:`Document` class. - - -.. method:: Element.hasAttribute(name) - - Returns true if the element has an attribute named by *name*. - - -.. method:: Element.hasAttributeNS(namespaceURI, localName) - - Returns true if the element has an attribute named by *namespaceURI* and - *localName*. - - -.. method:: Element.getAttribute(name) - - Return the value of the attribute named by *name* as a string. If no such - attribute exists, an empty string is returned, as if the attribute had no value. - - -.. method:: Element.getAttributeNode(attrname) - - Return the :class:`Attr` node for the attribute named by *attrname*. - - -.. method:: Element.getAttributeNS(namespaceURI, localName) - - Return the value of the attribute named by *namespaceURI* and *localName* as a - string. If no such attribute exists, an empty string is returned, as if the - attribute had no value. - - -.. method:: Element.getAttributeNodeNS(namespaceURI, localName) - - Return an attribute value as a node, given a *namespaceURI* and *localName*. - - -.. method:: Element.removeAttribute(name) - - Remove an attribute by name. If there is no matching attribute, a - :exc:`NotFoundErr` is raised. - - -.. method:: Element.removeAttributeNode(oldAttr) - - Remove and return *oldAttr* from the attribute list, if present. If *oldAttr* is - not present, :exc:`NotFoundErr` is raised. - - -.. method:: Element.removeAttributeNS(namespaceURI, localName) - - Remove an attribute by name. Note that it uses a localName, not a qname. No - exception is raised if there is no matching attribute. - - -.. method:: Element.setAttribute(name, value) - - Set an attribute value from a string. - - -.. method:: Element.setAttributeNode(newAttr) - - Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`name` attribute matches. If a replacement occurs, the - old attribute node will be returned. If *newAttr* is already in use, - :exc:`InuseAttributeErr` will be raised. - - -.. method:: Element.setAttributeNodeNS(newAttr) - - Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`namespaceURI` and :attr:`localName` attributes match. - If a replacement occurs, the old attribute node will be returned. If *newAttr* - is already in use, :exc:`InuseAttributeErr` will be raised. - - -.. method:: Element.setAttributeNS(namespaceURI, qname, value) - - Set an attribute value from a string, given a *namespaceURI* and a *qname*. - Note that a qname is the whole attribute name. This is different than above. - - -.. _dom-attr-objects: - -Attr Objects -^^^^^^^^^^^^ - -:class:`Attr` inherits from :class:`Node`, so inherits all its attributes. - - -.. attribute:: Attr.name - - The attribute name. - In a namespace-using document it may include a colon. - - -.. attribute:: Attr.localName - - The part of the name following the colon if there is one, else the - entire name. - This is a read-only attribute. - - -.. attribute:: Attr.prefix - - The part of the name preceding the colon if there is one, else the - empty string. - - -.. attribute:: Attr.value - - The text value of the attribute. This is a synonym for the - :attr:`nodeValue` attribute. - - -.. _dom-attributelist-objects: - -NamedNodeMap Objects -^^^^^^^^^^^^^^^^^^^^ - -:class:`NamedNodeMap` does *not* inherit from :class:`Node`. - - -.. attribute:: NamedNodeMap.length - - The length of the attribute list. - - -.. method:: NamedNodeMap.item(index) - - Return an attribute with a particular index. The order you get the attributes - in is arbitrary but will be consistent for the life of a DOM. Each item is an - attribute node. Get its value with the :attr:`value` attribute. - -There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`getAttribute\*` family -of methods on the :class:`Element` objects. - - -.. _dom-comment-objects: - -Comment Objects -^^^^^^^^^^^^^^^ - -:class:`Comment` represents a comment in the XML document. It is a subclass of -:class:`Node`, but cannot have child nodes. - - -.. attribute:: Comment.data - - The content of the comment as a string. The attribute contains all characters - between the leading ````, but does not - include them. - - -.. _dom-text-objects: - -Text and CDATASection Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`Text` interface represents text in the XML document. If the parser -and DOM implementation support the DOM's XML extension, portions of the text -enclosed in CDATA marked sections are stored in :class:`CDATASection` objects. -These two interfaces are identical, but provide different values for the -:attr:`nodeType` attribute. - -These interfaces extend the :class:`Node` interface. They cannot have child -nodes. - - -.. attribute:: Text.data - - The content of the text node as a string. - -.. note:: - - The use of a :class:`CDATASection` node does not indicate that the node - represents a complete CDATA marked section, only that the content of the node - was part of a CDATA section. A single CDATA section may be represented by more - than one node in the document tree. There is no way to determine whether two - adjacent :class:`CDATASection` nodes represent different CDATA marked sections. - - -.. _dom-pi-objects: - -ProcessingInstruction Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Represents a processing instruction in the XML document; this inherits from the -:class:`Node` interface and cannot have child nodes. - - -.. attribute:: ProcessingInstruction.target - - The content of the processing instruction up to the first whitespace character. - This is a read-only attribute. - - -.. attribute:: ProcessingInstruction.data - - The content of the processing instruction following the first whitespace - character. - - -.. _dom-exceptions: - -Exceptions -^^^^^^^^^^ - -The DOM Level 2 recommendation defines a single exception, :exc:`DOMException`, -and a number of constants that allow applications to determine what sort of -error occurred. :exc:`DOMException` instances carry a :attr:`code` attribute -that provides the appropriate value for the specific exception. - -The Python DOM interface provides the constants, but also expands the set of -exceptions so that a specific exception exists for each of the exception codes -defined by the DOM. The implementations must raise the appropriate specific -exception, each of which carries the appropriate value for the :attr:`code` -attribute. - - -.. exception:: DOMException - - Base exception class used for all specific DOM exceptions. This exception class - cannot be directly instantiated. - - -.. exception:: DomstringSizeErr - - Raised when a specified range of text does not fit into a string. This is not - known to be used in the Python DOM implementations, but may be received from DOM - implementations not written in Python. - - -.. exception:: HierarchyRequestErr - - Raised when an attempt is made to insert a node where the node type is not - allowed. - - -.. exception:: IndexSizeErr - - Raised when an index or size parameter to a method is negative or exceeds the - allowed values. - - -.. exception:: InuseAttributeErr - - Raised when an attempt is made to insert an :class:`Attr` node that is already - present elsewhere in the document. - - -.. exception:: InvalidAccessErr - - Raised if a parameter or an operation is not supported on the underlying object. - - -.. exception:: InvalidCharacterErr - - This exception is raised when a string parameter contains a character that is - not permitted in the context it's being used in by the XML 1.0 recommendation. - For example, attempting to create an :class:`Element` node with a space in the - element type name will cause this error to be raised. - - -.. exception:: InvalidModificationErr - - Raised when an attempt is made to modify the type of a node. - - -.. exception:: InvalidStateErr - - Raised when an attempt is made to use an object that is not defined or is no - longer usable. - - -.. exception:: NamespaceErr - - If an attempt is made to change any object in a way that is not permitted with - regard to the `Namespaces in XML `_ - recommendation, this exception is raised. - - -.. exception:: NotFoundErr - - Exception when a node does not exist in the referenced context. For example, - :meth:`NamedNodeMap.removeNamedItem` will raise this if the node passed in does - not exist in the map. - - -.. exception:: NotSupportedErr - - Raised when the implementation does not support the requested type of object or - operation. - - -.. exception:: NoDataAllowedErr - - This is raised if data is specified for a node which does not support data. - - .. XXX a better explanation is needed! - - -.. exception:: NoModificationAllowedErr - - Raised on attempts to modify an object where modifications are not allowed (such - as for read-only nodes). - - -.. exception:: SyntaxErr - - Raised when an invalid or illegal string is specified. - - .. XXX how is this different from InvalidCharacterErr? - - -.. exception:: WrongDocumentErr - - Raised when a node is inserted in a different document than it currently belongs - to, and the implementation does not support migrating the node from one document - to the other. - -The exception codes defined in the DOM recommendation map to the exceptions -described above according to this table: - -+--------------------------------------+---------------------------------+ -| Constant | Exception | -+======================================+=================================+ -| :const:`DOMSTRING_SIZE_ERR` | :exc:`DomstringSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`HIERARCHY_REQUEST_ERR` | :exc:`HierarchyRequestErr` | -+--------------------------------------+---------------------------------+ -| :const:`INDEX_SIZE_ERR` | :exc:`IndexSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INUSE_ATTRIBUTE_ERR` | :exc:`InuseAttributeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_ACCESS_ERR` | :exc:`InvalidAccessErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_CHARACTER_ERR` | :exc:`InvalidCharacterErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_MODIFICATION_ERR` | :exc:`InvalidModificationErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_STATE_ERR` | :exc:`InvalidStateErr` | -+--------------------------------------+---------------------------------+ -| :const:`NAMESPACE_ERR` | :exc:`NamespaceErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_FOUND_ERR` | :exc:`NotFoundErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_SUPPORTED_ERR` | :exc:`NotSupportedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_DATA_ALLOWED_ERR` | :exc:`NoDataAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_MODIFICATION_ALLOWED_ERR` | :exc:`NoModificationAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`SYNTAX_ERR` | :exc:`SyntaxErr` | -+--------------------------------------+---------------------------------+ -| :const:`WRONG_DOCUMENT_ERR` | :exc:`WrongDocumentErr` | -+--------------------------------------+---------------------------------+ - - -.. _dom-conformance: - -Conformance ------------ - -This section describes the conformance requirements and relationships between -the Python DOM API, the W3C DOM recommendations, and the OMG IDL mapping for -Python. - - -.. _dom-type-mapping: - -Type Mapping -^^^^^^^^^^^^ - -The IDL types used in the DOM specification are mapped to Python types -according to the following table. - -+------------------+-------------------------------------------+ -| IDL Type | Python Type | -+==================+===========================================+ -| ``boolean`` | ``bool`` or ``int`` | -+------------------+-------------------------------------------+ -| ``int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``long int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``unsigned int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``DOMString`` | ``str`` or ``bytes`` | -+------------------+-------------------------------------------+ -| ``null`` | ``None`` | -+------------------+-------------------------------------------+ - -.. _dom-accessor-methods: - -Accessor Methods -^^^^^^^^^^^^^^^^ - -The mapping from OMG IDL to Python defines accessor functions for IDL -``attribute`` declarations in much the way the Java mapping does. -Mapping the IDL declarations :: - - readonly attribute string someValue; - attribute string anotherValue; - -yields three accessor functions: a "get" method for :attr:`someValue` -(:meth:`_get_someValue`), and "get" and "set" methods for :attr:`anotherValue` -(:meth:`_get_anotherValue` and :meth:`_set_anotherValue`). The mapping, in -particular, does not require that the IDL attributes are accessible as normal -Python attributes: ``object.someValue`` is *not* required to work, and may -raise an :exc:`AttributeError`. - -The Python DOM API, however, *does* require that normal attribute access work. -This means that the typical surrogates generated by Python IDL compilers are not -likely to work, and wrapper objects may be needed on the client if the DOM -objects are accessed via CORBA. While this does require some additional -consideration for CORBA DOM clients, the implementers with experience using DOM -over CORBA from Python do not consider this a problem. Attributes that are -declared ``readonly`` may not restrict write access in all DOM -implementations. - -In the Python DOM API, accessor functions are not required. If provided, they -should take the form defined by the Python IDL mapping, but these methods are -considered unnecessary since the attributes are accessible directly from Python. -"Set" accessors should never be provided for ``readonly`` attributes. - -The IDL definitions do not fully embody the requirements of the W3C DOM API, -such as the notion of certain objects, such as the return value of -:meth:`getElementsByTagName`, being "live". The Python DOM API does not require -implementations to enforce such requirements. - diff --git a/third_party/python/Doc/library/xml.etree.elementtree.rst b/third_party/python/Doc/library/xml.etree.elementtree.rst deleted file mode 100644 index 80505ea64..000000000 --- a/third_party/python/Doc/library/xml.etree.elementtree.rst +++ /dev/null @@ -1,1198 +0,0 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== - -.. module:: xml.etree.ElementTree - :synopsis: Implementation of the ElementTree API. - -.. moduleauthor:: Fredrik Lundh - -**Source code:** :source:`Lib/xml/etree/ElementTree.py` - --------------- - -The :mod:`xml.etree.ElementTree` module implements a simple and efficient API -for parsing and creating XML data. - -.. versionchanged:: 3.3 - This module will use a fast implementation whenever available. - The :mod:`xml.etree.cElementTree` module is deprecated. - - -.. warning:: - - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - -Tutorial --------- - -This is a short tutorial for using :mod:`xml.etree.ElementTree` (``ET`` in -short). The goal is to demonstrate some of the building blocks and basic -concepts of the module. - -XML tree and elements -^^^^^^^^^^^^^^^^^^^^^ - -XML is an inherently hierarchical data format, and the most natural way to -represent it is with a tree. ``ET`` has two classes for this purpose - -:class:`ElementTree` represents the whole XML document as a tree, and -:class:`Element` represents a single node in this tree. Interactions with -the whole document (reading and writing to/from files) are usually done -on the :class:`ElementTree` level. Interactions with a single XML element -and its sub-elements are done on the :class:`Element` level. - -.. _elementtree-parsing-xml: - -Parsing XML -^^^^^^^^^^^ - -We'll be using the following XML document as the sample data for this section: - -.. code-block:: xml - - - - - 1 - 2008 - 141100 - - - - - 4 - 2011 - 59900 - - - - 68 - 2011 - 13600 - - - - - -We can import this data by reading from a file:: - - import xml.etree.ElementTree as ET - tree = ET.parse('country_data.xml') - root = tree.getroot() - -Or directly from a string:: - - root = ET.fromstring(country_data_as_string) - -:func:`fromstring` parses XML from a string directly into an :class:`Element`, -which is the root element of the parsed tree. Other parsing functions may -create an :class:`ElementTree`. Check the documentation to be sure. - -As an :class:`Element`, ``root`` has a tag and a dictionary of attributes:: - - >>> root.tag - 'data' - >>> root.attrib - {} - -It also has children nodes over which we can iterate:: - - >>> for child in root: - ... print(child.tag, child.attrib) - ... - country {'name': 'Liechtenstein'} - country {'name': 'Singapore'} - country {'name': 'Panama'} - -Children are nested, and we can access specific child nodes by index:: - - >>> root[0][1].text - '2008' - - -.. note:: - - Not all elements of the XML input will end up as elements of the - parsed tree. Currently, this module skips over any XML comments, - processing instructions, and document type declarations in the - input. Nevertheless, trees built using this module's API rather - than parsing from XML text can have comments and processing - instructions in them; they will be included when generating XML - output. A document type declaration may be accessed by passing a - custom :class:`TreeBuilder` instance to the :class:`XMLParser` - constructor. - - -.. _elementtree-pull-parsing: - -Pull API for non-blocking parsing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most parsing functions provided by this module require the whole document -to be read at once before returning any result. It is possible to use an -:class:`XMLParser` and feed data into it incrementally, but it is a push API that -calls methods on a callback target, which is too low-level and inconvenient for -most needs. Sometimes what the user really wants is to be able to parse XML -incrementally, without blocking operations, while enjoying the convenience of -fully constructed :class:`Element` objects. - -The most powerful tool for doing this is :class:`XMLPullParser`. It does not -require a blocking read to obtain the XML data, and is instead fed with data -incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML -elements, call :meth:`XMLPullParser.read_events`. Here is an example:: - - >>> parser = ET.XMLPullParser(['start', 'end']) - >>> parser.feed('sometext') - >>> list(parser.read_events()) - [('start', )] - >>> parser.feed(' more text') - >>> for event, elem in parser.read_events(): - ... print(event) - ... print(elem.tag, 'text=', elem.text) - ... - end - -The obvious use case is applications that operate in a non-blocking fashion -where the XML data is being received from a socket or read incrementally from -some storage device. In such cases, blocking reads are unacceptable. - -Because it's so flexible, :class:`XMLPullParser` can be inconvenient to use for -simpler use-cases. If you don't mind your application blocking on reading XML -data but would still like to have incremental parsing capabilities, take a look -at :func:`iterparse`. It can be useful when you're reading a large XML document -and don't want to hold it wholly in memory. - -Finding interesting elements -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`Element` has some useful methods that help iterate recursively over all -the sub-tree below it (its children, their children, and so on). For example, -:meth:`Element.iter`:: - - >>> for neighbor in root.iter('neighbor'): - ... print(neighbor.attrib) - ... - {'name': 'Austria', 'direction': 'E'} - {'name': 'Switzerland', 'direction': 'W'} - {'name': 'Malaysia', 'direction': 'N'} - {'name': 'Costa Rica', 'direction': 'W'} - {'name': 'Colombia', 'direction': 'E'} - -:meth:`Element.findall` finds only elements with a tag which are direct -children of the current element. :meth:`Element.find` finds the *first* child -with a particular tag, and :attr:`Element.text` accesses the element's text -content. :meth:`Element.get` accesses the element's attributes:: - - >>> for country in root.findall('country'): - ... rank = country.find('rank').text - ... name = country.get('name') - ... print(name, rank) - ... - Liechtenstein 1 - Singapore 4 - Panama 68 - -More sophisticated specification of which elements to look for is possible by -using :ref:`XPath `. - -Modifying an XML File -^^^^^^^^^^^^^^^^^^^^^ - -:class:`ElementTree` provides a simple way to build XML documents and write them to files. -The :meth:`ElementTree.write` method serves this purpose. - -Once created, an :class:`Element` object may be manipulated by directly changing -its fields (such as :attr:`Element.text`), adding and modifying attributes -(:meth:`Element.set` method), as well as adding new children (for example -with :meth:`Element.append`). - -Let's say we want to add one to each country's rank, and add an ``updated`` -attribute to the rank element:: - - >>> for rank in root.iter('rank'): - ... new_rank = int(rank.text) + 1 - ... rank.text = str(new_rank) - ... rank.set('updated', 'yes') - ... - >>> tree.write('output.xml') - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - 69 - 2011 - 13600 - - - - - -We can remove elements using :meth:`Element.remove`. Let's say we want to -remove all countries with a rank higher than 50:: - - >>> for country in root.findall('country'): - ... rank = int(country.find('rank').text) - ... if rank > 50: - ... root.remove(country) - ... - >>> tree.write('output.xml') - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - -Building XML documents -^^^^^^^^^^^^^^^^^^^^^^ - -The :func:`SubElement` function also provides a convenient way to create new -sub-elements for a given element:: - - >>> a = ET.Element('a') - >>> b = ET.SubElement(a, 'b') - >>> c = ET.SubElement(a, 'c') - >>> d = ET.SubElement(c, 'd') - >>> ET.dump(a) -     - -Parsing XML with Namespaces -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If the XML input has `namespaces -`__, tags and attributes -with prefixes in the form ``prefix:sometag`` get expanded to -``{uri}sometag`` where the *prefix* is replaced by the full *URI*. -Also, if there is a `default namespace -`__, -that full URI gets prepended to all of the non-prefixed tags. - -Here is an XML example that incorporates two namespaces, one with the -prefix "fictional" and the other serving as the default namespace: - -.. code-block:: xml - - - - - John Cleese - Lancelot - Archie Leach - - - Eric Idle - Sir Robin - Gunther - Commander Clement - - - -One way to search and explore this XML example is to manually add the -URI to every tag or attribute in the xpath of a -:meth:`~Element.find` or :meth:`~Element.findall`:: - - root = fromstring(xml_text) - for actor in root.findall('{http://people.example.com}actor'): - name = actor.find('{http://people.example.com}name') - print(name.text) - for char in actor.findall('{http://characters.example.com}character'): - print(' |-->', char.text) - -A better way to search the namespaced XML example is to create a -dictionary with your own prefixes and use those in the search functions:: - - ns = {'real_person': 'http://people.example.com', - 'role': 'http://characters.example.com'} - - for actor in root.findall('real_person:actor', ns): - name = actor.find('real_person:name', ns) - print(name.text) - for char in actor.findall('role:character', ns): - print(' |-->', char.text) - -These two approaches both output:: - - John Cleese - |--> Lancelot - |--> Archie Leach - Eric Idle - |--> Sir Robin - |--> Gunther - |--> Commander Clement - - -Additional resources -^^^^^^^^^^^^^^^^^^^^ - -See http://effbot.org/zone/element-index.htm for tutorials and links to other -docs. - - -.. _elementtree-xpath: - -XPath support -------------- - -This module provides limited support for -`XPath expressions `_ for locating elements in a -tree. The goal is to support a small subset of the abbreviated syntax; a full -XPath engine is outside the scope of the module. - -Example -^^^^^^^ - -Here's an example that demonstrates some of the XPath capabilities of the -module. We'll be using the ``countrydata`` XML document from the -:ref:`Parsing XML ` section:: - - import xml.etree.ElementTree as ET - - root = ET.fromstring(countrydata) - - # Top-level elements - root.findall(".") - - # All 'neighbor' grand-children of 'country' children of the top-level - # elements - root.findall("./country/neighbor") - - # Nodes with name='Singapore' that have a 'year' child - root.findall(".//year/..[@name='Singapore']") - - # 'year' nodes that are children of nodes with name='Singapore' - root.findall(".//*[@name='Singapore']/year") - - # All 'neighbor' nodes that are the second child of their parent - root.findall(".//neighbor[2]") - -Supported XPath syntax -^^^^^^^^^^^^^^^^^^^^^^ - -.. tabularcolumns:: |l|L| - -+-----------------------+------------------------------------------------------+ -| Syntax | Meaning | -+=======================+======================================================+ -| ``tag`` | Selects all child elements with the given tag. | -| | For example, ``spam`` selects all child elements | -| | named ``spam``, and ``spam/egg`` selects all | -| | grandchildren named ``egg`` in all children named | -| | ``spam``. | -+-----------------------+------------------------------------------------------+ -| ``*`` | Selects all child elements. For example, ``*/egg`` | -| | selects all grandchildren named ``egg``. | -+-----------------------+------------------------------------------------------+ -| ``.`` | Selects the current node. This is mostly useful | -| | at the beginning of the path, to indicate that it's | -| | a relative path. | -+-----------------------+------------------------------------------------------+ -| ``//`` | Selects all subelements, on all levels beneath the | -| | current element. For example, ``.//egg`` selects | -| | all ``egg`` elements in the entire tree. | -+-----------------------+------------------------------------------------------+ -| ``..`` | Selects the parent element. Returns ``None`` if the | -| | path attempts to reach the ancestors of the start | -| | element (the element ``find`` was called on). | -+-----------------------+------------------------------------------------------+ -| ``[@attrib]`` | Selects all elements that have the given attribute. | -+-----------------------+------------------------------------------------------+ -| ``[@attrib='value']`` | Selects all elements for which the given attribute | -| | has the given value. The value cannot contain | -| | quotes. | -+-----------------------+------------------------------------------------------+ -| ``[tag]`` | Selects all elements that have a child named | -| | ``tag``. Only immediate children are supported. | -+-----------------------+------------------------------------------------------+ -| ``[tag='text']`` | Selects all elements that have a child named | -| | ``tag`` whose complete text content, including | -| | descendants, equals the given ``text``. | -+-----------------------+------------------------------------------------------+ -| ``[position]`` | Selects all elements that are located at the given | -| | position. The position can be either an integer | -| | (1 is the first position), the expression ``last()`` | -| | (for the last position), or a position relative to | -| | the last position (e.g. ``last()-1``). | -+-----------------------+------------------------------------------------------+ - -Predicates (expressions within square brackets) must be preceded by a tag -name, an asterisk, or another predicate. ``position`` predicates must be -preceded by a tag name. - -Reference ---------- - -.. _elementtree-functions: - -Functions -^^^^^^^^^ - - -.. function:: Comment(text=None) - - Comment element factory. This factory function creates a special element - that will be serialized as an XML comment by the standard serializer. The - comment string can be either a bytestring or a Unicode string. *text* is a - string containing the comment string. Returns an element instance - representing a comment. - - Note that :class:`XMLParser` skips over comments in the input - instead of creating comment objects for them. An :class:`ElementTree` will - only contain comment nodes if they have been inserted into to - the tree using one of the :class:`Element` methods. - -.. function:: dump(elem) - - Writes an element tree or element structure to sys.stdout. This function - should be used for debugging only. - - The exact output format is implementation dependent. In this version, it's - written as an ordinary XML file. - - *elem* is an element tree or an individual element. - - -.. function:: fromstring(text) - - Parses an XML section from a string constant. Same as :func:`XML`. *text* - is a string containing XML data. Returns an :class:`Element` instance. - - -.. function:: fromstringlist(sequence, parser=None) - - Parses an XML document from a sequence of string fragments. *sequence* is a - list or other sequence containing XML data fragments. *parser* is an - optional parser instance. If not given, the standard :class:`XMLParser` - parser is used. Returns an :class:`Element` instance. - - .. versionadded:: 3.2 - - -.. function:: iselement(element) - - Checks if an object appears to be a valid element object. *element* is an - element instance. Returns a true value if this is an element object. - - -.. function:: iterparse(source, events=None, parser=None) - - Parses an XML section into an element tree incrementally, and reports what's - going on to the user. *source* is a filename or :term:`file object` - containing XML data. *events* is a sequence of events to report back. The - supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and - ``"end-ns"`` (the "ns" events are used to get detailed namespace - information). If *events* is omitted, only ``"end"`` events are reported. - *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. *parser* must be a subclass of - :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a - target. Returns an :term:`iterator` providing ``(event, elem)`` pairs. - - Note that while :func:`iterparse` builds the tree incrementally, it issues - blocking reads on *source* (or the file it names). As such, it's unsuitable - for applications where blocking reads can't be made. For fully non-blocking - parsing, see :class:`XMLPullParser`. - - .. note:: - - :func:`iterparse` only guarantees that it has seen the ">" character of a - starting tag when it emits a "start" event, so the attributes are defined, - but the contents of the text and tail attributes are undefined at that - point. The same applies to the element children; they may or may not be - present. - - If you need a fully populated element, look for "end" events instead. - - .. deprecated:: 3.4 - The *parser* argument. - -.. function:: parse(source, parser=None) - - Parses an XML section into an element tree. *source* is a filename or file - object containing XML data. *parser* is an optional parser instance. If - not given, the standard :class:`XMLParser` parser is used. Returns an - :class:`ElementTree` instance. - - -.. function:: ProcessingInstruction(target, text=None) - - PI element factory. This factory function creates a special element that - will be serialized as an XML processing instruction. *target* is a string - containing the PI target. *text* is a string containing the PI contents, if - given. Returns an element instance, representing a processing instruction. - - Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An - :class:`ElementTree` will only contain processing instruction nodes if - they have been inserted into to the tree using one of the - :class:`Element` methods. - -.. function:: register_namespace(prefix, uri) - - Registers a namespace prefix. The registry is global, and any existing - mapping for either the given prefix or the namespace URI will be removed. - *prefix* is a namespace prefix. *uri* is a namespace uri. Tags and - attributes in this namespace will be serialized with the given prefix, if at - all possible. - - .. versionadded:: 3.2 - - -.. function:: SubElement(parent, tag, attrib={}, **extra) - - Subelement factory. This function creates an element instance, and appends - it to an existing element. - - The element name, attribute names, and attribute values can be either - bytestrings or Unicode strings. *parent* is the parent element. *tag* is - the subelement name. *attrib* is an optional dictionary, containing element - attributes. *extra* contains additional attributes, given as keyword - arguments. Returns an element instance. - - -.. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - short_empty_elements=True) - - Generates a string representation of an XML element, including all - subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is - the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. - Returns an (optionally) encoded string containing the XML data. - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - -.. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - short_empty_elements=True) - - Generates a string representation of an XML element, including all - subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is - the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. - Returns a list of (optionally) encoded strings containing the XML data. - It does not guarantee any specific sequence, except that - ``b"".join(tostringlist(element)) == tostring(element)``. - - .. versionadded:: 3.2 - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - -.. function:: XML(text, parser=None) - - Parses an XML section from a string constant. This function can be used to - embed "XML literals" in Python code. *text* is a string containing XML - data. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. Returns an :class:`Element` instance. - - -.. function:: XMLID(text, parser=None) - - Parses an XML section from a string constant, and also returns a dictionary - which maps from element id:s to elements. *text* is a string containing XML - data. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. Returns a tuple containing an - :class:`Element` instance and a dictionary. - - -.. _elementtree-element-objects: - -Element Objects -^^^^^^^^^^^^^^^ - -.. class:: Element(tag, attrib={}, **extra) - - Element class. This class defines the Element interface, and provides a - reference implementation of this interface. - - The element name, attribute names, and attribute values can be either - bytestrings or Unicode strings. *tag* is the element name. *attrib* is - an optional dictionary, containing element attributes. *extra* contains - additional attributes, given as keyword arguments. - - - .. attribute:: tag - - A string identifying what kind of data this element represents (the - element type, in other words). - - - .. attribute:: text - tail - - These attributes can be used to hold additional data associated with - the element. Their values are usually strings but may be any - application-specific object. If the element is created from - an XML file, the *text* attribute holds either the text between - the element's start tag and its first child or end tag, or ``None``, and - the *tail* attribute holds either the text between the element's - end tag and the next tag, or ``None``. For the XML data - - .. code-block:: xml - - 1234 - - the *a* element has ``None`` for both *text* and *tail* attributes, - the *b* element has *text* ``"1"`` and *tail* ``"4"``, - the *c* element has *text* ``"2"`` and *tail* ``None``, - and the *d* element has *text* ``None`` and *tail* ``"3"``. - - To collect the inner text of an element, see :meth:`itertext`, for - example ``"".join(element.itertext())``. - - Applications may store arbitrary objects in these attributes. - - - .. attribute:: attrib - - A dictionary containing the element's attributes. Note that while the - *attrib* value is always a real mutable Python dictionary, an ElementTree - implementation may choose to use another internal representation, and - create the dictionary only if someone asks for it. To take advantage of - such implementations, use the dictionary methods below whenever possible. - - The following dictionary-like methods work on the element attributes. - - - .. method:: clear() - - Resets an element. This function removes all subelements, clears all - attributes, and sets the text and tail attributes to ``None``. - - - .. method:: get(key, default=None) - - Gets the element attribute named *key*. - - Returns the attribute value, or *default* if the attribute was not found. - - - .. method:: items() - - Returns the element attributes as a sequence of (name, value) pairs. The - attributes are returned in an arbitrary order. - - - .. method:: keys() - - Returns the elements attribute names as a list. The names are returned - in an arbitrary order. - - - .. method:: set(key, value) - - Set the attribute *key* on the element to *value*. - - The following methods work on the element's children (subelements). - - - .. method:: append(subelement) - - Adds the element *subelement* to the end of this element's internal list - of subelements. Raises :exc:`TypeError` if *subelement* is not an - :class:`Element`. - - - .. method:: extend(subelements) - - Appends *subelements* from a sequence object with zero or more elements. - Raises :exc:`TypeError` if a subelement is not an :class:`Element`. - - .. versionadded:: 3.2 - - - .. method:: find(match, namespaces=None) - - Finds the first subelement matching *match*. *match* may be a tag name - or a :ref:`path `. Returns an element instance - or ``None``. *namespaces* is an optional mapping from namespace prefix - to full name. - - - .. method:: findall(match, namespaces=None) - - Finds all matching subelements, by tag name or - :ref:`path `. Returns a list containing all matching - elements in document order. *namespaces* is an optional mapping from - namespace prefix to full name. - - - .. method:: findtext(match, default=None, namespaces=None) - - Finds text for the first subelement matching *match*. *match* may be - a tag name or a :ref:`path `. Returns the text content - of the first matching element, or *default* if no element was found. - Note that if the matching element has no text content an empty string - is returned. *namespaces* is an optional mapping from namespace prefix - to full name. - - - .. method:: getchildren() - - .. deprecated:: 3.2 - Use ``list(elem)`` or iteration. - - - .. method:: getiterator(tag=None) - - .. deprecated:: 3.2 - Use method :meth:`Element.iter` instead. - - - .. method:: insert(index, subelement) - - Inserts *subelement* at the given position in this element. Raises - :exc:`TypeError` if *subelement* is not an :class:`Element`. - - - .. method:: iter(tag=None) - - Creates a tree :term:`iterator` with the current element as the root. - The iterator iterates over this element and all elements below it, in - document (depth first) order. If *tag* is not ``None`` or ``'*'``, only - elements whose tag equals *tag* are returned from the iterator. If the - tree structure is modified during iteration, the result is undefined. - - .. versionadded:: 3.2 - - - .. method:: iterfind(match, namespaces=None) - - Finds all matching subelements, by tag name or - :ref:`path `. Returns an iterable yielding all - matching elements in document order. *namespaces* is an optional mapping - from namespace prefix to full name. - - - .. versionadded:: 3.2 - - - .. method:: itertext() - - Creates a text iterator. The iterator loops over this element and all - subelements, in document order, and returns all inner text. - - .. versionadded:: 3.2 - - - .. method:: makeelement(tag, attrib) - - Creates a new element object of the same type as this element. Do not - call this method, use the :func:`SubElement` factory function instead. - - - .. method:: remove(subelement) - - Removes *subelement* from the element. Unlike the find\* methods this - method compares elements based on the instance identity, not on tag value - or contents. - - :class:`Element` objects also support the following sequence type methods - for working with subelements: :meth:`~object.__delitem__`, - :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, - :meth:`~object.__len__`. - - Caution: Elements with no subelements will test as ``False``. This behavior - will change in future versions. Use specific ``len(elem)`` or ``elem is - None`` test instead. :: - - element = root.find('foo') - - if not element: # careful! - print("element not found, or element has no subelements") - - if element is None: - print("element not found") - - -.. _elementtree-elementtree-objects: - -ElementTree Objects -^^^^^^^^^^^^^^^^^^^ - - -.. class:: ElementTree(element=None, file=None) - - ElementTree wrapper class. This class represents an entire element - hierarchy, and adds some extra support for serialization to and from - standard XML. - - *element* is the root element. The tree is initialized with the contents - of the XML *file* if given. - - - .. method:: _setroot(element) - - Replaces the root element for this tree. This discards the current - contents of the tree, and replaces it with the given element. Use with - care. *element* is an element instance. - - - .. method:: find(match, namespaces=None) - - Same as :meth:`Element.find`, starting at the root of the tree. - - - .. method:: findall(match, namespaces=None) - - Same as :meth:`Element.findall`, starting at the root of the tree. - - - .. method:: findtext(match, default=None, namespaces=None) - - Same as :meth:`Element.findtext`, starting at the root of the tree. - - - .. method:: getiterator(tag=None) - - .. deprecated:: 3.2 - Use method :meth:`ElementTree.iter` instead. - - - .. method:: getroot() - - Returns the root element for this tree. - - - .. method:: iter(tag=None) - - Creates and returns a tree iterator for the root element. The iterator - loops over all elements in this tree, in section order. *tag* is the tag - to look for (default is to return all elements). - - - .. method:: iterfind(match, namespaces=None) - - Same as :meth:`Element.iterfind`, starting at the root of the tree. - - .. versionadded:: 3.2 - - - .. method:: parse(source, parser=None) - - Loads an external XML section into this element tree. *source* is a file - name or :term:`file object`. *parser* is an optional parser instance. - If not given, the standard :class:`XMLParser` parser is used. Returns the - section root element. - - - .. method:: write(file, encoding="us-ascii", xml_declaration=None, \ - default_namespace=None, method="xml", *, \ - short_empty_elements=True) - - Writes the element tree to a file, as XML. *file* is a file name, or a - :term:`file object` opened for writing. *encoding* [1]_ is the output - encoding (default is US-ASCII). - *xml_declaration* controls if an XML declaration should be added to the - file. Use ``False`` for never, ``True`` for always, ``None`` - for only if not US-ASCII or UTF-8 or Unicode (default is ``None``). - *default_namespace* sets the default XML namespace (for "xmlns"). - *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is - ``"xml"``). - The keyword-only *short_empty_elements* parameter controls the formatting - of elements that contain no content. If ``True`` (the default), they are - emitted as a single self-closed tag, otherwise they are emitted as a pair - of start/end tags. - - The output is either a string (:class:`str`) or binary (:class:`bytes`). - This is controlled by the *encoding* argument. If *encoding* is - ``"unicode"``, the output is a string; otherwise, it's binary. Note that - this may conflict with the type of *file* if it's an open - :term:`file object`; make sure you do not try to write a string to a - binary stream and vice versa. - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - -This is the XML file that is going to be manipulated:: - - - - Example page - - -

    Moved to example.org - or example.com.

    - - - -Example of changing the attribute "target" of every link in first paragraph:: - - >>> from xml.etree.ElementTree import ElementTree - >>> tree = ElementTree() - >>> tree.parse("index.xhtml") - - >>> p = tree.find("body/p") # Finds first occurrence of tag p in body - >>> p - - >>> links = list(p.iter("a")) # Returns list of all links - >>> links - [, ] - >>> for i in links: # Iterates through all found links - ... i.attrib["target"] = "blank" - >>> tree.write("output.xhtml") - -.. _elementtree-qname-objects: - -QName Objects -^^^^^^^^^^^^^ - - -.. class:: QName(text_or_uri, tag=None) - - QName wrapper. This can be used to wrap a QName attribute value, in order - to get proper namespace handling on output. *text_or_uri* is a string - containing the QName value, in the form {uri}local, or, if the tag argument - is given, the URI part of a QName. If *tag* is given, the first argument is - interpreted as a URI, and this argument is interpreted as a local name. - :class:`QName` instances are opaque. - - - -.. _elementtree-treebuilder-objects: - -TreeBuilder Objects -^^^^^^^^^^^^^^^^^^^ - - -.. class:: TreeBuilder(element_factory=None) - - Generic element structure builder. This builder converts a sequence of - start, data, and end method calls to a well-formed element structure. You - can use this class to build an element structure using a custom XML parser, - or a parser for some other XML-like format. *element_factory*, when given, - must be a callable accepting two positional arguments: a tag and - a dict of attributes. It is expected to return a new element instance. - - .. method:: close() - - Flushes the builder buffers, and returns the toplevel document - element. Returns an :class:`Element` instance. - - - .. method:: data(data) - - Adds text to the current element. *data* is a string. This should be - either a bytestring, or a Unicode string. - - - .. method:: end(tag) - - Closes the current element. *tag* is the element name. Returns the - closed element. - - - .. method:: start(tag, attrs) - - Opens a new element. *tag* is the element name. *attrs* is a dictionary - containing element attributes. Returns the opened element. - - - In addition, a custom :class:`TreeBuilder` object can provide the - following method: - - .. method:: doctype(name, pubid, system) - - Handles a doctype declaration. *name* is the doctype name. *pubid* is - the public identifier. *system* is the system identifier. This method - does not exist on the default :class:`TreeBuilder` class. - - .. versionadded:: 3.2 - - -.. _elementtree-xmlparser-objects: - -XMLParser Objects -^^^^^^^^^^^^^^^^^ - - -.. class:: XMLParser(html=0, target=None, encoding=None) - - This class is the low-level building block of the module. It uses - :mod:`xml.parsers.expat` for efficient, event-based parsing of XML. It can - be fed XML data incrementally with the :meth:`feed` method, and parsing - events are translated to a push API - by invoking callbacks on the *target* - object. If *target* is omitted, the standard :class:`TreeBuilder` is used. - The *html* argument was historically used for backwards compatibility and is - now deprecated. If *encoding* [1]_ is given, the value overrides the - encoding specified in the XML file. - - .. deprecated:: 3.4 - The *html* argument. The remaining arguments should be passed via - keyword to prepare for the removal of the *html* argument. - - .. method:: close() - - Finishes feeding data to the parser. Returns the result of calling the - ``close()`` method of the *target* passed during construction; by default, - this is the toplevel document element. - - - .. method:: doctype(name, pubid, system) - - .. deprecated:: 3.2 - Define the :meth:`TreeBuilder.doctype` method on a custom TreeBuilder - target. - - - .. method:: feed(data) - - Feeds data to the parser. *data* is encoded data. - - :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method - for each opening tag, its ``end(tag)`` method for each closing tag, and data - is processed by method ``data(data)``. :meth:`XMLParser.close` calls - *target*\'s method ``close()``. :class:`XMLParser` can be used not only for - building a tree structure. This is an example of counting the maximum depth - of an XML file:: - - >>> from xml.etree.ElementTree import XMLParser - >>> class MaxDepth: # The target object of the parser - ... maxDepth = 0 - ... depth = 0 - ... def start(self, tag, attrib): # Called for each opening tag. - ... self.depth += 1 - ... if self.depth > self.maxDepth: - ... self.maxDepth = self.depth - ... def end(self, tag): # Called for each closing tag. - ... self.depth -= 1 - ... def data(self, data): - ... pass # We do not need to do anything with data. - ... def close(self): # Called when all data has been parsed. - ... return self.maxDepth - ... - >>> target = MaxDepth() - >>> parser = XMLParser(target=target) - >>> exampleXml = """ - ... - ... - ... - ... - ... - ... - ... - ... - ... - ... """ - >>> parser.feed(exampleXml) - >>> parser.close() - 4 - - -.. _elementtree-xmlpullparser-objects: - -XMLPullParser Objects -^^^^^^^^^^^^^^^^^^^^^ - -.. class:: XMLPullParser(events=None) - - A pull parser suitable for non-blocking applications. Its input-side API is - similar to that of :class:`XMLParser`, but instead of pushing calls to a - callback target, :class:`XMLPullParser` collects an internal list of parsing - events and lets the user read from it. *events* is a sequence of events to - report back. The supported events are the strings ``"start"``, ``"end"``, - ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed - namespace information). If *events* is omitted, only ``"end"`` events are - reported. - - .. method:: feed(data) - - Feed the given bytes data to the parser. - - .. method:: close() - - Signal the parser that the data stream is terminated. Unlike - :meth:`XMLParser.close`, this method always returns :const:`None`. - Any events not yet retrieved when the parser is closed can still be - read with :meth:`read_events`. - - .. method:: read_events() - - Return an iterator over the events which have been encountered in the - data fed to the - parser. The iterator yields ``(event, elem)`` pairs, where *event* is a - string representing the type of event (e.g. ``"end"``) and *elem* is the - encountered :class:`Element` object. - - Events provided in a previous call to :meth:`read_events` will not be - yielded again. Events are consumed from the internal queue only when - they are retrieved from the iterator, so multiple readers iterating in - parallel over iterators obtained from :meth:`read_events` will have - unpredictable results. - - .. note:: - - :class:`XMLPullParser` only guarantees that it has seen the ">" - character of a starting tag when it emits a "start" event, so the - attributes are defined, but the contents of the text and tail attributes - are undefined at that point. The same applies to the element children; - they may or may not be present. - - If you need a fully populated element, look for "end" events instead. - - .. versionadded:: 3.4 - -Exceptions -^^^^^^^^^^ - -.. class:: ParseError - - XML parse error, raised by the various parsing methods in this module when - parsing fails. The string representation of an instance of this exception - will contain a user-friendly error message. In addition, it will have - the following attributes available: - - .. attribute:: code - - A numeric error code from the expat parser. See the documentation of - :mod:`xml.parsers.expat` for the list of error codes and their meanings. - - .. attribute:: position - - A tuple of *line*, *column* numbers, specifying where the error occurred. - -.. rubric:: Footnotes - -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/third_party/python/Doc/library/xml.rst b/third_party/python/Doc/library/xml.rst deleted file mode 100644 index 9b8ba6b17..000000000 --- a/third_party/python/Doc/library/xml.rst +++ /dev/null @@ -1,139 +0,0 @@ -.. _xml: - -XML Processing Modules -====================== - -.. module:: xml - :synopsis: Package containing XML processing modules - -.. sectionauthor:: Christian Heimes -.. sectionauthor:: Georg Brandl - -**Source code:** :source:`Lib/xml/` - --------------- - -Python's interfaces for processing XML are grouped in the ``xml`` package. - -.. warning:: - - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defused-packages` sections. - -It is important to note that modules in the :mod:`xml` package require that -there be at least one SAX-compliant XML parser available. The Expat parser is -included with Python, so the :mod:`xml.parsers.expat` module will always be -available. - -The documentation for the :mod:`xml.dom` and :mod:`xml.sax` packages are the -definition of the Python bindings for the DOM and SAX interfaces. - -The XML handling submodules are: - -* :mod:`xml.etree.ElementTree`: the ElementTree API, a simple and lightweight - XML processor - -.. - -* :mod:`xml.dom`: the DOM API definition -* :mod:`xml.dom.minidom`: a minimal DOM implementation -* :mod:`xml.dom.pulldom`: support for building partial DOM trees - -.. - -* :mod:`xml.sax`: SAX2 base classes and convenience functions -* :mod:`xml.parsers.expat`: the Expat parser binding - - -.. _xml-vulnerabilities: - -XML vulnerabilities -------------------- - -The XML processing modules are not secure against maliciously constructed data. -An attacker can abuse XML features to carry out denial of service attacks, -access local files, generate network connections to other machines, or -circumvent firewalls. - -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. - -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.8.0, external general entities are no longer processed by - default since Python. - - -billion laughs / exponential entity expansion - The `Billion Laughs`_ attack -- also known as exponential entity expansion -- - uses multiple levels of nested entities. Each entity refers to another entity - several times, and the final entity definition contains a small string. - The exponential expansion results in several gigabytes of text and - consumes lots of memory and CPU time. - -quadratic blowup entity expansion - A quadratic blowup attack is similar to a `Billion Laughs`_ attack; it abuses - entity expansion, too. Instead of nested entities it repeats one large entity - with a couple of thousand chars over and over again. The attack isn't as - efficient as the exponential case but it avoids triggering parser countermeasures - that forbid deeply-nested entities. - -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -`DTD`_ retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - -decompression bomb - Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries - that can parse compressed XML streams such as gzipped HTTP streams or - LZMA-compressed - files. For an attacker it can reduce the amount of transmitted data by three - magnitudes or more. - -The documentation for `defusedxml`_ on PyPI has further information about -all known attack vectors with examples and references. - -.. _defused-packages: - -The :mod:`defusedxml` and :mod:`defusedexpat` Packages ------------------------------------------------------- - -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - -`defusedexpat`_ provides a modified libexpat and a patched -:mod:`pyexpat` module that have countermeasures against entity expansion -DoS attacks. The :mod:`defusedexpat` module still allows a sane and configurable amount of entity -expansions. The modifications may be included in some future release of Python, -but will not be included in any bugfix releases of -Python because they break backward compatibility. - - -.. _defusedxml: https://pypi.org/project/defusedxml/ -.. _defusedexpat: https://pypi.org/project/defusedexpat/ -.. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs -.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/third_party/python/Doc/library/xml.sax.handler.rst b/third_party/python/Doc/library/xml.sax.handler.rst deleted file mode 100644 index ae0877ca9..000000000 --- a/third_party/python/Doc/library/xml.sax.handler.rst +++ /dev/null @@ -1,415 +0,0 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== - -.. module:: xml.sax.handler - :synopsis: Base classes for SAX event handlers. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/handler.py` - --------------- - -The SAX API defines four kinds of handlers: content handlers, DTD handlers, -error handlers, and entity resolvers. Applications normally only need to -implement those interfaces whose events they are interested in; they can -implement the interfaces in a single object or in multiple objects. Handler -implementations should inherit from the base classes provided in the module -:mod:`xml.sax.handler`, so that all methods get default implementations. - - -.. class:: ContentHandler - - This is the main callback interface in SAX, and the one most important to - applications. The order of events in this interface mirrors the order of the - information in the document. - - -.. class:: DTDHandler - - Handle DTD events. - - This interface specifies only those DTD events required for basic parsing - (unparsed entities and attributes). - - -.. class:: EntityResolver - - Basic interface for resolving entities. If you create an object implementing - this interface, then register the object with your Parser, the parser will call - the method in your object to resolve all external entities. - - -.. class:: ErrorHandler - - Interface used by the parser to present error and warning messages to the - application. The methods of this object control whether errors are immediately - converted to exceptions or are handled in some other way. - -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants -for the feature and property names. - - -.. data:: feature_namespaces - - | value: ``"http://xml.org/sax/features/namespaces"`` - | true: Perform Namespace processing. - | false: Optionally do not perform Namespace processing (implies - namespace-prefixes; default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_namespace_prefixes - - | value: ``"http://xml.org/sax/features/namespace-prefixes"`` - | true: Report the original prefixed names and attributes used for Namespace - declarations. - | false: Do not report attributes used for Namespace declarations, and - optionally do not report original prefixed names (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_string_interning - - | value: ``"http://xml.org/sax/features/string-interning"`` - | true: All element names, prefixes, attribute names, Namespace URIs, and - local names are interned using the built-in intern function. - | false: Names are not necessarily interned, although they may be (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_validation - - | value: ``"http://xml.org/sax/features/validation"`` - | true: Report all validation errors (implies external-general-entities and - external-parameter-entities). - | false: Do not report validation errors. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_ges - - | value: ``"http://xml.org/sax/features/external-general-entities"`` - | true: Include all external general (text) entities. - | false: Do not include external general entities. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_pes - - | value: ``"http://xml.org/sax/features/external-parameter-entities"`` - | true: Include all external parameter entities, including the external DTD - subset. - | false: Do not include any external parameter entities, even the external - DTD subset. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: all_features - - List of all features. - - -.. data:: property_lexical_handler - - | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.sax2lib.LexicalHandler (not supported in Python 2) - | description: An optional extension handler for lexical events like - comments. - | access: read/write - - -.. data:: property_declaration_handler - - | value: ``"http://xml.org/sax/properties/declaration-handler"`` - | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) - | description: An optional extension handler for DTD-related events other - than notations and unparsed entities. - | access: read/write - - -.. data:: property_dom_node - - | value: ``"http://xml.org/sax/properties/dom-node"`` - | data type: org.w3c.dom.Node (not supported in Python 2) - | description: When parsing, the current DOM node being visited if this is - a DOM iterator; when not parsing, the root DOM node for iteration. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: property_xml_string - - | value: ``"http://xml.org/sax/properties/xml-string"`` - | data type: String - | description: The literal string of characters that was the source for the - current event. - | access: read-only - - -.. data:: all_properties - - List of all known property names. - - -.. _content-handler-objects: - -ContentHandler Objects ----------------------- - -Users are expected to subclass :class:`ContentHandler` to support their -application. The following methods are called by the parser on the appropriate -events in the input document: - - -.. method:: ContentHandler.setDocumentLocator(locator) - - Called by the parser to give the application a locator for locating the origin - of document events. - - SAX parsers are strongly encouraged (though not absolutely required) to supply a - locator: if it does so, it must supply the locator to the application by - invoking this method before invoking any of the other methods in the - DocumentHandler interface. - - The locator allows the application to determine the end position of any - document-related event, even if the parser is not reporting an error. Typically, - the application will use this information for reporting its own errors (such as - character content that does not match an application's business rules). The - information returned by the locator is probably not sufficient for use with a - search engine. - - Note that the locator will return correct information only during the invocation - of the events in this interface. The application should not attempt to use it at - any other time. - - -.. method:: ContentHandler.startDocument() - - Receive notification of the beginning of a document. - - The SAX parser will invoke this method only once, before any other methods in - this interface or in DTDHandler (except for :meth:`setDocumentLocator`). - - -.. method:: ContentHandler.endDocument() - - Receive notification of the end of a document. - - The SAX parser will invoke this method only once, and it will be the last method - invoked during the parse. The parser shall not invoke this method until it has - either abandoned parsing (because of an unrecoverable error) or reached the end - of input. - - -.. method:: ContentHandler.startPrefixMapping(prefix, uri) - - Begin the scope of a prefix-URI Namespace mapping. - - The information from this event is not necessary for normal Namespace - processing: the SAX XML reader will automatically replace prefixes for element - and attribute names when the ``feature_namespaces`` feature is enabled (the - default). - - There are cases, however, when applications need to use prefixes in character - data or in attribute values, where they cannot safely be expanded automatically; - the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the - information to the application to expand prefixes in those contexts itself, if - necessary. - - .. XXX This is not really the default, is it? MvL - - Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not - guaranteed to be properly nested relative to each-other: all - :meth:`startPrefixMapping` events will occur before the corresponding - :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur - after the corresponding :meth:`endElement` event, but their order is not - guaranteed. - - -.. method:: ContentHandler.endPrefixMapping(prefix) - - End the scope of a prefix-URI mapping. - - See :meth:`startPrefixMapping` for details. This event will always occur after - the corresponding :meth:`endElement` event, but the order of - :meth:`endPrefixMapping` events is not otherwise guaranteed. - - -.. method:: ContentHandler.startElement(name, attrs) - - Signals the start of an element in non-namespace mode. - - The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of - the element. The object passed as *attrs* may be re-used by the parser; holding - on to a reference to it is not a reliable way to keep a copy of the attributes. - To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* - object. - - -.. method:: ContentHandler.endElement(name) - - Signals the end of an element in non-namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElement` event. - - -.. method:: ContentHandler.startElementNS(name, qname, attrs) - - Signals the start of an element in namespace mode. - - The *name* parameter contains the name of the element type as a ``(uri, - localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in - the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) - containing the attributes of the element. If no namespace is associated with - the element, the *uri* component of *name* will be ``None``. The object passed - as *attrs* may be re-used by the parser; holding on to a reference to it is not - a reliable way to keep a copy of the attributes. To keep a copy of the - attributes, use the :meth:`copy` method of the *attrs* object. - - Parsers may set the *qname* parameter to ``None``, unless the - ``feature_namespace_prefixes`` feature is activated. - - -.. method:: ContentHandler.endElementNS(name, qname) - - Signals the end of an element in namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElementNS` method, likewise the *qname* parameter. - - -.. method:: ContentHandler.characters(content) - - Receive notification of character data. - - The Parser will call this method to report each chunk of character data. SAX - parsers may return all contiguous character data in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity so that the Locator provides useful - information. - - *content* may be a string or bytes instance; the ``expat`` reader module - always produces strings. - - .. note:: - - The earlier SAX 1 interface provided by the Python XML Special Interest Group - used a more Java-like interface for this method. Since most parsers used from - Python did not take advantage of the older interface, the simpler signature was - chosen to replace it. To convert old code to the new interface, use *content* - instead of slicing content with the old *offset* and *length* parameters. - - -.. method:: ContentHandler.ignorableWhitespace(whitespace) - - Receive notification of ignorable whitespace in element content. - - Validating Parsers must use this method to report each chunk of ignorable - whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating - parsers may also use this method if they are capable of parsing and using - content models. - - SAX parsers may return all contiguous whitespace in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity, so that the Locator provides useful - information. - - -.. method:: ContentHandler.processingInstruction(target, data) - - Receive notification of a processing instruction. - - The Parser will invoke this method once for each processing instruction found: - note that processing instructions may occur before or after the main document - element. - - A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a - text declaration (XML 1.0, section 4.3.1) using this method. - - -.. method:: ContentHandler.skippedEntity(name) - - Receive notification of a skipped entity. - - The Parser will invoke this method once for each entity skipped. Non-validating - processors may skip entities if they have not seen the declarations (because, - for example, the entity was declared in an external DTD subset). All processors - may skip external entities, depending on the values of the - ``feature_external_ges`` and the ``feature_external_pes`` properties. - - -.. _dtd-handler-objects: - -DTDHandler Objects ------------------- - -:class:`DTDHandler` instances provide the following methods: - - -.. method:: DTDHandler.notationDecl(name, publicId, systemId) - - Handle a notation declaration event. - - -.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) - - Handle an unparsed entity declaration event. - - -.. _entity-resolver-objects: - -EntityResolver Objects ----------------------- - - -.. method:: EntityResolver.resolveEntity(publicId, systemId) - - Resolve the system identifier of an entity and return either the system - identifier to read from as a string, or an InputSource to read from. The default - implementation returns *systemId*. - - -.. _sax-error-handler: - -ErrorHandler Objects --------------------- - -Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser -will call the methods in your object to report all warnings and errors. There -are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`SAXParseException` as the -only parameter. Errors and warnings may be converted to an exception by raising -the passed-in exception object. - - -.. method:: ErrorHandler.error(exception) - - Called when the parser encounters a recoverable error. If this method does not - raise an exception, parsing may continue, but further document information - should not be expected by the application. Allowing the parser to continue may - allow additional errors to be discovered in the input document. - - -.. method:: ErrorHandler.fatalError(exception) - - Called when the parser encounters an error it cannot recover from; parsing is - expected to terminate when this method returns. - - -.. method:: ErrorHandler.warning(exception) - - Called when the parser presents minor warning information to the application. - Parsing is expected to continue when this method returns, and document - information will continue to be passed to the application. Raising an exception - in this method will cause parsing to end. - diff --git a/third_party/python/Doc/library/xml.sax.reader.rst b/third_party/python/Doc/library/xml.sax.reader.rst deleted file mode 100644 index 1b6e43145..000000000 --- a/third_party/python/Doc/library/xml.sax.reader.rst +++ /dev/null @@ -1,393 +0,0 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== - -.. module:: xml.sax.xmlreader - :synopsis: Interface which SAX-compliant XML parsers must implement. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/xmlreader.py` - --------------- - -SAX parsers implement the :class:`XMLReader` interface. They are implemented in -a Python module, which must provide a function :func:`create_parser`. This -function is invoked by :func:`xml.sax.make_parser` with no arguments to create -a new parser object. - - -.. class:: XMLReader() - - Base class which can be inherited by SAX parsers. - - -.. class:: IncrementalParser() - - In some cases, it is desirable not to parse an input source at once, but to feed - chunks of the document as they get available. Note that the reader will normally - not read the entire file, but read it in chunks as well; still :meth:`parse` - won't return until the entire document is processed. So these interfaces should - be used if the blocking behaviour of :meth:`parse` is not desirable. - - When the parser is instantiated it is ready to begin accepting data from the - feed method immediately. After parsing has been finished with a call to close - the reset method must be called to make the parser ready to accept new data, - either from feed or using the parse method. - - Note that these methods must *not* be called during parsing, that is, after - parse has been called and before it returns. - - By default, the class also implements the parse method of the XMLReader - interface using the feed, close and reset methods of the IncrementalParser - interface as a convenience to SAX 2.0 driver writers. - - -.. class:: Locator() - - Interface for associating a SAX event with a document location. A locator object - will return valid results only during calls to DocumentHandler methods; at any - other time, the results are unpredictable. If information is not available, - methods may return ``None``. - - -.. class:: InputSource(system_id=None) - - Encapsulation of the information needed by the :class:`XMLReader` to read - entities. - - This class may include information about the public identifier, system - identifier, byte stream (possibly with character encoding information) and/or - the character stream of an entity. - - Applications will create objects of this class for use in the - :meth:`XMLReader.parse` method and for returning from - EntityResolver.resolveEntity. - - An :class:`InputSource` belongs to the application, the :class:`XMLReader` is - not allowed to modify :class:`InputSource` objects passed to it from the - application, although it may make copies and modify those. - - -.. class:: AttributesImpl(attrs) - - This is an implementation of the :class:`Attributes` interface (see section - :ref:`attributes-objects`). This is a dictionary-like object which - represents the element attributes in a :meth:`startElement` call. In addition - to the most useful dictionary operations, it supports a number of other - methods as described by the interface. Objects of this class should be - instantiated by readers; *attrs* must be a dictionary-like object containing - a mapping from attribute names to attribute values. - - -.. class:: AttributesNSImpl(attrs, qnames) - - Namespace-aware variant of :class:`AttributesImpl`, which will be passed to - :meth:`startElementNS`. It is derived from :class:`AttributesImpl`, but - understands attribute names as two-tuples of *namespaceURI* and - *localname*. In addition, it provides a number of methods expecting qualified - names as they appear in the original document. This class implements the - :class:`AttributesNS` interface (see section :ref:`attributes-ns-objects`). - - -.. _xmlreader-objects: - -XMLReader Objects ------------------ - -The :class:`XMLReader` interface supports the following methods: - - -.. method:: XMLReader.parse(source) - - Process an input source, producing SAX events. The *source* object can be a - system identifier (a string identifying the input source -- typically a file - name or a URL), a file-like object, or an :class:`InputSource` object. When - :meth:`parse` returns, the input is completely processed, and the parser object - can be discarded or reset. - - .. versionchanged:: 3.5 - Added support of character streams. - - -.. method:: XMLReader.getContentHandler() - - Return the current :class:`~xml.sax.handler.ContentHandler`. - - -.. method:: XMLReader.setContentHandler(handler) - - Set the current :class:`~xml.sax.handler.ContentHandler`. If no - :class:`~xml.sax.handler.ContentHandler` is set, content events will be - discarded. - - -.. method:: XMLReader.getDTDHandler() - - Return the current :class:`~xml.sax.handler.DTDHandler`. - - -.. method:: XMLReader.setDTDHandler(handler) - - Set the current :class:`~xml.sax.handler.DTDHandler`. If no - :class:`~xml.sax.handler.DTDHandler` is set, DTD - events will be discarded. - - -.. method:: XMLReader.getEntityResolver() - - Return the current :class:`~xml.sax.handler.EntityResolver`. - - -.. method:: XMLReader.setEntityResolver(handler) - - Set the current :class:`~xml.sax.handler.EntityResolver`. If no - :class:`~xml.sax.handler.EntityResolver` is set, - attempts to resolve an external entity will result in opening the system - identifier for the entity, and fail if it is not available. - - -.. method:: XMLReader.getErrorHandler() - - Return the current :class:`~xml.sax.handler.ErrorHandler`. - - -.. method:: XMLReader.setErrorHandler(handler) - - Set the current error handler. If no :class:`~xml.sax.handler.ErrorHandler` - is set, errors will be raised as exceptions, and warnings will be printed. - - -.. method:: XMLReader.setLocale(locale) - - Allow an application to set the locale for errors and warnings. - - SAX parsers are not required to provide localization for errors and warnings; if - they cannot support the requested locale, however, they must raise a SAX - exception. Applications may request a locale change in the middle of a parse. - - -.. method:: XMLReader.getFeature(featurename) - - Return the current setting for feature *featurename*. If the feature is not - recognized, :exc:`SAXNotRecognizedException` is raised. The well-known - featurenames are listed in the module :mod:`xml.sax.handler`. - - -.. method:: XMLReader.setFeature(featurename, value) - - Set the *featurename* to *value*. If the feature is not recognized, - :exc:`SAXNotRecognizedException` is raised. If the feature or its setting is not - supported by the parser, *SAXNotSupportedException* is raised. - - -.. method:: XMLReader.getProperty(propertyname) - - Return the current setting for property *propertyname*. If the property is not - recognized, a :exc:`SAXNotRecognizedException` is raised. The well-known - propertynames are listed in the module :mod:`xml.sax.handler`. - - -.. method:: XMLReader.setProperty(propertyname, value) - - Set the *propertyname* to *value*. If the property is not recognized, - :exc:`SAXNotRecognizedException` is raised. If the property or its setting is - not supported by the parser, *SAXNotSupportedException* is raised. - - -.. _incremental-parser-objects: - -IncrementalParser Objects -------------------------- - -Instances of :class:`IncrementalParser` offer the following additional methods: - - -.. method:: IncrementalParser.feed(data) - - Process a chunk of *data*. - - -.. method:: IncrementalParser.close() - - Assume the end of the document. That will check well-formedness conditions that - can be checked only at the end, invoke handlers, and may clean up resources - allocated during parsing. - - -.. method:: IncrementalParser.reset() - - This method is called after close has been called to reset the parser so that it - is ready to parse new documents. The results of calling parse or feed after - close without calling reset are undefined. - - -.. _locator-objects: - -Locator Objects ---------------- - -Instances of :class:`Locator` provide these methods: - - -.. method:: Locator.getColumnNumber() - - Return the column number where the current event begins. - - -.. method:: Locator.getLineNumber() - - Return the line number where the current event begins. - - -.. method:: Locator.getPublicId() - - Return the public identifier for the current event. - - -.. method:: Locator.getSystemId() - - Return the system identifier for the current event. - - -.. _input-source-objects: - -InputSource Objects -------------------- - - -.. method:: InputSource.setPublicId(id) - - Sets the public identifier of this :class:`InputSource`. - - -.. method:: InputSource.getPublicId() - - Returns the public identifier of this :class:`InputSource`. - - -.. method:: InputSource.setSystemId(id) - - Sets the system identifier of this :class:`InputSource`. - - -.. method:: InputSource.getSystemId() - - Returns the system identifier of this :class:`InputSource`. - - -.. method:: InputSource.setEncoding(encoding) - - Sets the character encoding of this :class:`InputSource`. - - The encoding must be a string acceptable for an XML encoding declaration (see - section 4.3.3 of the XML recommendation). - - The encoding attribute of the :class:`InputSource` is ignored if the - :class:`InputSource` also contains a character stream. - - -.. method:: InputSource.getEncoding() - - Get the character encoding of this InputSource. - - -.. method:: InputSource.setByteStream(bytefile) - - Set the byte stream (a :term:`binary file`) for this input source. - - The SAX parser will ignore this if there is also a character stream specified, - but it will use a byte stream in preference to opening a URI connection itself. - - If the application knows the character encoding of the byte stream, it should - set it with the setEncoding method. - - -.. method:: InputSource.getByteStream() - - Get the byte stream for this input source. - - The getEncoding method will return the character encoding for this byte stream, - or ``None`` if unknown. - - -.. method:: InputSource.setCharacterStream(charfile) - - Set the character stream (a :term:`text file`) for this input source. - - If there is a character stream specified, the SAX parser will ignore any byte - stream and will not attempt to open a URI connection to the system identifier. - - -.. method:: InputSource.getCharacterStream() - - Get the character stream for this input source. - - -.. _attributes-objects: - -The :class:`Attributes` Interface ---------------------------------- - -:class:`Attributes` objects implement a portion of the :term:`mapping protocol -`, including the methods :meth:`~collections.abc.Mapping.copy`, -:meth:`~collections.abc.Mapping.get`, :meth:`~object.__contains__`, -:meth:`~collections.abc.Mapping.items`, :meth:`~collections.abc.Mapping.keys`, -and :meth:`~collections.abc.Mapping.values`. The following methods -are also provided: - - -.. method:: Attributes.getLength() - - Return the number of attributes. - - -.. method:: Attributes.getNames() - - Return the names of the attributes. - - -.. method:: Attributes.getType(name) - - Returns the type of the attribute *name*, which is normally ``'CDATA'``. - - -.. method:: Attributes.getValue(name) - - Return the value of attribute *name*. - -.. getValueByQName, getNameByQName, getQNameByName, getQNames available -.. here already, but documented only for derived class. - - -.. _attributes-ns-objects: - -The :class:`AttributesNS` Interface ------------------------------------ - -This interface is a subtype of the :class:`Attributes` interface (see section -:ref:`attributes-objects`). All methods supported by that interface are also -available on :class:`AttributesNS` objects. - -The following methods are also available: - - -.. method:: AttributesNS.getValueByQName(name) - - Return the value for a qualified name. - - -.. method:: AttributesNS.getNameByQName(name) - - Return the ``(namespace, localname)`` pair for a qualified *name*. - - -.. method:: AttributesNS.getQNameByName(name) - - Return the qualified name for a ``(namespace, localname)`` pair. - - -.. method:: AttributesNS.getQNames() - - Return the qualified names of all attributes. - diff --git a/third_party/python/Doc/library/xml.sax.rst b/third_party/python/Doc/library/xml.sax.rst deleted file mode 100644 index 254b539e7..000000000 --- a/third_party/python/Doc/library/xml.sax.rst +++ /dev/null @@ -1,173 +0,0 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== - -.. module:: xml.sax - :synopsis: Package containing SAX2 base classes and convenience functions. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/__init__.py` - --------------- - -The :mod:`xml.sax` package provides a number of modules which implement the -Simple API for XML (SAX) interface for Python. The package itself provides the -SAX exceptions and the convenience functions which will be most used by users of -the SAX API. - - -.. warning:: - - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.6.7 - - The SAX parser no longer processes general external entities by default - to increase security. Before, the parser created network connections - to fetch remote files or loaded local files from the file - system for DTD and entities. The feature can be enabled again with method - :meth:`~xml.sax.xmlreader.XMLReader.setFeature` on the parser object - and argument :data:`~xml.sax.handler.feature_external_ges`. - -The convenience functions are: - - -.. function:: make_parser(parser_list=[]) - - Create and return a SAX :class:`~xml.sax.xmlreader.XMLReader` object. The - first parser found will - be used. If *parser_list* is provided, it must be a list of strings which - name modules that have a function named :func:`create_parser`. Modules listed - in *parser_list* will be used before modules in the default list of parsers. - - -.. function:: parse(filename_or_stream, handler, error_handler=handler.ErrorHandler()) - - Create a SAX parser and use it to parse a document. The document, passed in as - *filename_or_stream*, can be a filename or a file object. The *handler* - parameter needs to be a SAX :class:`~handler.ContentHandler` instance. If - *error_handler* is given, it must be a SAX :class:`~handler.ErrorHandler` - instance; if - omitted, :exc:`SAXParseException` will be raised on all errors. There is no - return value; all work must be done by the *handler* passed in. - - -.. function:: parseString(string, handler, error_handler=handler.ErrorHandler()) - - Similar to :func:`parse`, but parses from a buffer *string* received as a - parameter. *string* must be a :class:`str` instance or a - :term:`bytes-like object`. - - .. versionchanged:: 3.5 - Added support of :class:`str` instances. - -A typical SAX application uses three kinds of objects: readers, handlers and -input sources. "Reader" in this context is another term for parser, i.e. some -piece of code that reads the bytes or characters from the input source, and -produces a sequence of events. The events then get distributed to the handler -objects, i.e. the reader invokes a method on the handler. A SAX application -must therefore obtain a reader object, create or open the input sources, create -the handlers, and connect these objects all together. As the final step of -preparation, the reader is called to parse the input. During parsing, methods on -the handler objects are called based on structural and syntactic events from the -input data. - -For these objects, only the interfaces are relevant; they are normally not -instantiated by the application itself. Since Python does not have an explicit -notion of interface, they are formally introduced as classes, but applications -may use implementations which do not inherit from the provided classes. The -:class:`~xml.sax.xmlreader.InputSource`, :class:`~xml.sax.xmlreader.Locator`, -:class:`~xml.sax.xmlreader.Attributes`, :class:`~xml.sax.xmlreader.AttributesNS`, -and :class:`~xml.sax.xmlreader.XMLReader` interfaces are defined in the -module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in -:mod:`xml.sax.handler`. For convenience, -:class:`~xml.sax.xmlreader.InputSource` (which is often -instantiated directly) and the handler classes are also available from -:mod:`xml.sax`. These interfaces are described below. - -In addition to these classes, :mod:`xml.sax` provides the following exception -classes. - - -.. exception:: SAXException(msg, exception=None) - - Encapsulate an XML error or warning. This class can contain basic error or - warning information from either the XML parser or the application: it can be - subclassed to provide additional functionality or to add localization. Note - that although the handlers defined in the - :class:`~xml.sax.handler.ErrorHandler` interface - receive instances of this exception, it is not required to actually raise the - exception --- it is also useful as a container for information. - - When instantiated, *msg* should be a human-readable description of the error. - The optional *exception* parameter, if given, should be ``None`` or an exception - that was caught by the parsing code and is being passed along as information. - - This is the base class for the other SAX exception classes. - - -.. exception:: SAXParseException(msg, exception, locator) - - Subclass of :exc:`SAXException` raised on parse errors. Instances of this - class are passed to the methods of the SAX - :class:`~xml.sax.handler.ErrorHandler` interface to provide information - about the parse error. This class supports the SAX - :class:`~xml.sax.xmlreader.Locator` interface as well as the - :class:`SAXException` interface. - - -.. exception:: SAXNotRecognizedException(msg, exception=None) - - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is - confronted with an unrecognized feature or property. SAX applications and - extensions may use this class for similar purposes. - - -.. exception:: SAXNotSupportedException(msg, exception=None) - - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is asked to - enable a feature that is not supported, or to set a property to a value that the - implementation does not support. SAX applications and extensions may use this - class for similar purposes. - - -.. seealso:: - - `SAX: The Simple API for XML `_ - This site is the focal point for the definition of the SAX API. It provides a - Java implementation and online documentation. Links to implementations and - historical information are also available. - - Module :mod:`xml.sax.handler` - Definitions of the interfaces for application-provided objects. - - Module :mod:`xml.sax.saxutils` - Convenience functions for use in SAX applications. - - Module :mod:`xml.sax.xmlreader` - Definitions of the interfaces for parser-provided objects. - - -.. _sax-exception-objects: - -SAXException Objects --------------------- - -The :class:`SAXException` exception class supports the following methods: - - -.. method:: SAXException.getMessage() - - Return a human-readable message describing the error condition. - - -.. method:: SAXException.getException() - - Return an encapsulated exception object, or ``None``. - diff --git a/third_party/python/Doc/library/xml.sax.utils.rst b/third_party/python/Doc/library/xml.sax.utils.rst deleted file mode 100644 index e46fefdf9..000000000 --- a/third_party/python/Doc/library/xml.sax.utils.rst +++ /dev/null @@ -1,91 +0,0 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= - -.. module:: xml.sax.saxutils - :synopsis: Convenience functions and classes for use with SAX. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/saxutils.py` - --------------- - -The module :mod:`xml.sax.saxutils` contains a number of classes and functions -that are commonly useful when creating SAX applications, either in direct use, -or as base classes. - - -.. function:: escape(data, entities={}) - - Escape ``'&'``, ``'<'``, and ``'>'`` in a string of data. - - You can escape other strings of data by passing a dictionary as the optional - *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. The characters ``'&'``, ``'<'`` and - ``'>'`` are always escaped, even if *entities* is provided. - - -.. function:: unescape(data, entities={}) - - Unescape ``'&'``, ``'<'``, and ``'>'`` in a string of data. - - You can unescape other strings of data by passing a dictionary as the optional - *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` - are always unescaped, even if *entities* is provided. - - -.. function:: quoteattr(data, entities={}) - - Similar to :func:`escape`, but also prepares *data* to be used as an - attribute value. The return value is a quoted version of *data* with any - additional required replacements. :func:`quoteattr` will select a quote - character based on the content of *data*, attempting to avoid encoding any - quote characters in the string. If both single- and double-quote characters - are already in *data*, the double-quote characters will be encoded and *data* - will be wrapped in double-quotes. The resulting string can be used directly - as an attribute value:: - - >>> print("" % quoteattr("ab ' cd \" ef")) - - - This function is useful when generating attribute values for HTML or any SGML - using the reference concrete syntax. - - -.. class:: XMLGenerator(out=None, encoding='iso-8859-1', short_empty_elements=False) - - This class implements the :class:`~xml.sax.handler.ContentHandler` interface - by writing SAX - events back into an XML document. In other words, using an :class:`XMLGenerator` - as the content handler will reproduce the original document being parsed. *out* - should be a file-like object which will default to *sys.stdout*. *encoding* is - the encoding of the output stream which defaults to ``'iso-8859-1'``. - *short_empty_elements* controls the formatting of elements that contain no - content: if ``False`` (the default) they are emitted as a pair of start/end - tags, if set to ``True`` they are emitted as a single self-closed tag. - - .. versionadded:: 3.2 - The *short_empty_elements* parameter. - - -.. class:: XMLFilterBase(base) - - This class is designed to sit between an - :class:`~xml.sax.xmlreader.XMLReader` and the client - application's event handlers. By default, it does nothing but pass requests up - to the reader and events on to the handlers unmodified, but subclasses can - override specific methods to modify the event stream or the configuration - requests as they pass through. - - -.. function:: prepare_input_source(source, base='') - - This function takes an input source and an optional base URL and returns a - fully resolved :class:`~xml.sax.xmlreader.InputSource` object ready for - reading. The input source can be given as a string, a file-like object, or - an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this - function to implement the polymorphic *source* argument to their - :meth:`parse` method. - diff --git a/third_party/python/Doc/library/xmlrpc.client.rst b/third_party/python/Doc/library/xmlrpc.client.rst deleted file mode 100644 index 27d92e324..000000000 --- a/third_party/python/Doc/library/xmlrpc.client.rst +++ /dev/null @@ -1,606 +0,0 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== - -.. module:: xmlrpc.client - :synopsis: XML-RPC client access. - -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/xmlrpc/client.py` - -.. XXX Not everything is documented yet. It might be good to describe - Marshaller, Unmarshaller, getparser and Transport. - --------------- - -XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP(S) as a -transport. With it, a client can call methods with parameters on a remote -server (the server is named by a URI) and get back structured data. This module -supports writing XML-RPC client code; it handles all the details of translating -between conformable Python objects and XML on the wire. - - -.. warning:: - - The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.5 - - For HTTPS URIs, :mod:`xmlrpc.client` now performs all the necessary - certificate and hostname checks by default. - -.. class:: ServerProxy(uri, transport=None, encoding=None, verbose=False, \ - allow_none=False, use_datetime=False, \ - use_builtin_types=False, *, context=None) - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - A :class:`ServerProxy` instance is an object that manages communication with a - remote XML-RPC server. The required first argument is a URI (Uniform Resource - Indicator), and will normally be the URL of the server. The optional second - argument is a transport factory instance; by default it is an internal - :class:`SafeTransport` instance for https: URLs and an internal HTTP - :class:`Transport` instance otherwise. The optional third argument is an - encoding, by default UTF-8. The optional fourth argument is a debugging flag. - - The following parameters govern the use of the returned proxy instance. - If *allow_none* is true, the Python constant ``None`` will be translated into - XML; the default behaviour is for ``None`` to raise a :exc:`TypeError`. This is - a commonly-used extension to the XML-RPC specification, but isn't supported by - all clients and servers; see `http://ontosys.com/xml-rpc/extensions.php - `_ - for a description. - The *use_builtin_types* flag can be used to cause date/time values - to be presented as :class:`datetime.datetime` objects and binary data to be - presented as :class:`bytes` objects; this flag is false by default. - :class:`datetime.datetime`, :class:`bytes` and :class:`bytearray` objects - may be passed to calls. - The obsolete *use_datetime* flag is similar to *use_builtin_types* but it - applies only to date/time values. - - Both the HTTP and HTTPS transports support the URL syntax extension for HTTP - Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` - portion will be base64-encoded as an HTTP 'Authorization' header, and sent to - the remote server as part of the connection process when invoking an XML-RPC - method. You only need to use this if the remote server requires a Basic - Authentication user and password. If an HTTPS URL is provided, *context* may - be :class:`ssl.SSLContext` and configures the SSL settings of the underlying - HTTPS connection. - - The returned instance is a proxy object with methods that can be used to invoke - corresponding RPC calls on the remote server. If the remote server supports the - introspection API, the proxy can also be used to query the remote server for the - methods it supports (service discovery) and fetch other server-associated - metadata. - - Types that are conformable (e.g. that can be marshalled through XML), - include the following (and except where noted, they are unmarshalled - as the same Python type): - - .. tabularcolumns:: |l|L| - - +----------------------+-------------------------------------------------------+ - | XML-RPC type | Python type | - +======================+=======================================================+ - | ``boolean`` | :class:`bool` | - +----------------------+-------------------------------------------------------+ - | ``int``, ``i1``, | :class:`int` in range from -2147483648 to 2147483647. | - | ``i2``, ``i4``, | Values get the ```` tag. | - | ``i8`` or | | - | ``biginteger`` | | - +----------------------+-------------------------------------------------------+ - | ``double`` or | :class:`float`. Values get the ```` tag. | - | ``float`` | | - +----------------------+-------------------------------------------------------+ - | ``string`` | :class:`str` | - +----------------------+-------------------------------------------------------+ - | ``array`` | :class:`list` or :class:`tuple` containing | - | | conformable elements. Arrays are returned as | - | | :class:`lists `. | - +----------------------+-------------------------------------------------------+ - | ``struct`` | :class:`dict`. Keys must be strings, values may be | - | | any conformable type. Objects of user-defined | - | | classes can be passed in; only their | - | | :attr:`~object.__dict__` attribute is transmitted. | - +----------------------+-------------------------------------------------------+ - | ``dateTime.iso8601`` | :class:`DateTime` or :class:`datetime.datetime`. | - | | Returned type depends on values of | - | | *use_builtin_types* and *use_datetime* flags. | - +----------------------+-------------------------------------------------------+ - | ``base64`` | :class:`Binary`, :class:`bytes` or | - | | :class:`bytearray`. Returned type depends on the | - | | value of the *use_builtin_types* flag. | - +----------------------+-------------------------------------------------------+ - | ``nil`` | The ``None`` constant. Passing is allowed only if | - | | *allow_none* is true. | - +----------------------+-------------------------------------------------------+ - | ``bigdecimal`` | :class:`decimal.Decimal`. Returned type only. | - +----------------------+-------------------------------------------------------+ - - This is the full set of data types supported by XML-RPC. Method calls may also - raise a special :exc:`Fault` instance, used to signal XML-RPC server errors, or - :exc:`ProtocolError` used to signal an error in the HTTP/HTTPS transport layer. - Both :exc:`Fault` and :exc:`ProtocolError` derive from a base class called - :exc:`Error`. Note that the xmlrpc client module currently does not marshal - instances of subclasses of built-in types. - - When passing strings, characters special to XML such as ``<``, ``>``, and ``&`` - will be automatically escaped. However, it's the caller's responsibility to - ensure that the string is free of characters that aren't allowed in XML, such as - the control characters with ASCII values between 0 and 31 (except, of course, - tab, newline and carriage return); failing to do this will result in an XML-RPC - request that isn't well-formed XML. If you have to pass arbitrary bytes - via XML-RPC, use :class:`bytes` or :class:`bytearray` classes or the - :class:`Binary` wrapper class described below. - - :class:`Server` is retained as an alias for :class:`ServerProxy` for backwards - compatibility. New code should use :class:`ServerProxy`. - - .. versionchanged:: 3.5 - Added the *context* argument. - - .. versionchanged:: 3.6 - Added support of type tags with prefixes (e.g. ``ex:nil``). - Added support of unmarshalling additional types used by Apache XML-RPC - implementation for numerics: ``i1``, ``i2``, ``i8``, ``biginteger``, - ``float`` and ``bigdecimal``. - See http://ws.apache.org/xmlrpc/types.html for a description. - - -.. seealso:: - - `XML-RPC HOWTO `_ - A good description of XML-RPC operation and client software in several languages. - Contains pretty much everything an XML-RPC client developer needs to know. - - `XML-RPC Introspection `_ - Describes the XML-RPC protocol extension for introspection. - - `XML-RPC Specification `_ - The official specification. - - `Unofficial XML-RPC Errata `_ - Fredrik Lundh's "unofficial errata, intended to clarify certain - details in the XML-RPC specification, as well as hint at - 'best practices' to use when designing your own XML-RPC - implementations." - -.. _serverproxy-objects: - -ServerProxy Objects -------------------- - -A :class:`ServerProxy` instance has a method corresponding to each remote -procedure call accepted by the XML-RPC server. Calling the method performs an -RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. - -Servers that support the XML introspection API support some common methods -grouped under the reserved :attr:`~ServerProxy.system` attribute: - - -.. method:: ServerProxy.system.listMethods() - - This method returns a list of strings, one for each (non-system) method - supported by the XML-RPC server. - - -.. method:: ServerProxy.system.methodSignature(name) - - This method takes one parameter, the name of a method implemented by the XML-RPC - server. It returns an array of possible signatures for this method. A signature - is an array of types. The first of these types is the return type of the method, - the rest are parameters. - - Because multiple signatures (ie. overloading) is permitted, this method returns - a list of signatures rather than a singleton. - - Signatures themselves are restricted to the top level parameters expected by a - method. For instance if a method expects one array of structs as a parameter, - and it returns a string, its signature is simply "string, array". If it expects - three integers and returns a string, its signature is "string, int, int, int". - - If no signature is defined for the method, a non-array value is returned. In - Python this means that the type of the returned value will be something other - than list. - - -.. method:: ServerProxy.system.methodHelp(name) - - This method takes one parameter, the name of a method implemented by the XML-RPC - server. It returns a documentation string describing the use of that method. If - no such string is available, an empty string is returned. The documentation - string may contain HTML markup. - -.. versionchanged:: 3.5 - - Instances of :class:`ServerProxy` support the :term:`context manager` protocol - for closing the underlying transport. - - -A working example follows. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - def is_even(n): - return n % 2 == 0 - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(is_even, "is_even") - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - with xmlrpc.client.ServerProxy("http://localhost:8000/") as proxy: - print("3 is even: %s" % str(proxy.is_even(3))) - print("100 is even: %s" % str(proxy.is_even(100))) - -.. _datetime-objects: - -DateTime Objects ----------------- - -.. class:: DateTime - - This class may be initialized with seconds since the epoch, a time - tuple, an ISO 8601 time/date string, or a :class:`datetime.datetime` - instance. It has the following methods, supported mainly for internal - use by the marshalling/unmarshalling code: - - - .. method:: decode(string) - - Accept a string as the instance's new time value. - - - .. method:: encode(out) - - Write the XML-RPC encoding of this :class:`DateTime` item to the *out* stream - object. - - It also supports certain of Python's built-in operators through rich comparison - and :meth:`__repr__` methods. - -A working example follows. The server code:: - - import datetime - from xmlrpc.server import SimpleXMLRPCServer - import xmlrpc.client - - def today(): - today = datetime.datetime.today() - return xmlrpc.client.DateTime(today) - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(today, "today") - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - import datetime - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - - today = proxy.today() - # convert the ISO8601 string to a datetime object - converted = datetime.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") - print("Today: %s" % converted.strftime("%d.%m.%Y, %H:%M")) - -.. _binary-objects: - -Binary Objects --------------- - -.. class:: Binary - - This class may be initialized from bytes data (which may include NULs). The - primary access to the content of a :class:`Binary` object is provided by an - attribute: - - - .. attribute:: data - - The binary data encapsulated by the :class:`Binary` instance. The data is - provided as a :class:`bytes` object. - - :class:`Binary` objects have the following methods, supported mainly for - internal use by the marshalling/unmarshalling code: - - - .. method:: decode(bytes) - - Accept a base64 :class:`bytes` object and decode it as the instance's new data. - - - .. method:: encode(out) - - Write the XML-RPC base 64 encoding of this binary item to the *out* stream object. - - The encoded data will have newlines every 76 characters as per - :rfc:`RFC 2045 section 6.8 <2045#section-6.8>`, - which was the de facto standard base64 specification when the - XML-RPC spec was written. - - It also supports certain of Python's built-in operators through :meth:`__eq__` - and :meth:`__ne__` methods. - -Example usage of the binary objects. We're going to transfer an image over -XMLRPC:: - - from xmlrpc.server import SimpleXMLRPCServer - import xmlrpc.client - - def python_logo(): - with open("python_logo.jpg", "rb") as handle: - return xmlrpc.client.Binary(handle.read()) - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(python_logo, 'python_logo') - - server.serve_forever() - -The client gets the image and saves it to a file:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - with open("fetched_python_logo.jpg", "wb") as handle: - handle.write(proxy.python_logo().data) - -.. _fault-objects: - -Fault Objects -------------- - -.. class:: Fault - - A :class:`Fault` object encapsulates the content of an XML-RPC fault tag. Fault - objects have the following attributes: - - - .. attribute:: faultCode - - A string indicating the fault type. - - - .. attribute:: faultString - - A string containing a diagnostic message associated with the fault. - -In the following example we're going to intentionally cause a :exc:`Fault` by -returning a complex type object. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - # A marshalling error is going to occur because we're returning a - # complex number - def add(x, y): - return x+y+0j - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(add, 'add') - - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - try: - proxy.add(2, 5) - except xmlrpc.client.Fault as err: - print("A fault occurred") - print("Fault code: %d" % err.faultCode) - print("Fault string: %s" % err.faultString) - - - -.. _protocol-error-objects: - -ProtocolError Objects ---------------------- - -.. class:: ProtocolError - - A :class:`ProtocolError` object describes a protocol error in the underlying - transport layer (such as a 404 'not found' error if the server named by the URI - does not exist). It has the following attributes: - - - .. attribute:: url - - The URI or URL that triggered the error. - - - .. attribute:: errcode - - The error code. - - - .. attribute:: errmsg - - The error message or diagnostic string. - - - .. attribute:: headers - - A dict containing the headers of the HTTP/HTTPS request that triggered the - error. - -In the following example we're going to intentionally cause a :exc:`ProtocolError` -by providing an invalid URI:: - - import xmlrpc.client - - # create a ServerProxy with a URI that doesn't respond to XMLRPC requests - proxy = xmlrpc.client.ServerProxy("http://google.com/") - - try: - proxy.some_method() - except xmlrpc.client.ProtocolError as err: - print("A protocol error occurred") - print("URL: %s" % err.url) - print("HTTP/HTTPS headers: %s" % err.headers) - print("Error code: %d" % err.errcode) - print("Error message: %s" % err.errmsg) - -MultiCall Objects ------------------ - -The :class:`MultiCall` object provides a way to encapsulate multiple calls to a -remote server into a single request [#]_. - - -.. class:: MultiCall(server) - - Create an object used to boxcar method calls. *server* is the eventual target of - the call. Calls can be made to the result object, but they will immediately - return ``None``, and only store the call name and parameters in the - :class:`MultiCall` object. Calling the object itself causes all stored calls to - be transmitted as a single ``system.multicall`` request. The result of this call - is a :term:`generator`; iterating over this generator yields the individual - results. - -A usage example of this class follows. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - def add(x, y): - return x + y - - def subtract(x, y): - return x - y - - def multiply(x, y): - return x * y - - def divide(x, y): - return x // y - - # A simple server with simple arithmetic functions - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_multicall_functions() - server.register_function(add, 'add') - server.register_function(subtract, 'subtract') - server.register_function(multiply, 'multiply') - server.register_function(divide, 'divide') - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - multicall = xmlrpc.client.MultiCall(proxy) - multicall.add(7, 3) - multicall.subtract(7, 3) - multicall.multiply(7, 3) - multicall.divide(7, 3) - result = multicall() - - print("7+3=%d, 7-3=%d, 7*3=%d, 7//3=%d" % tuple(result)) - - -Convenience Functions ---------------------- - -.. function:: dumps(params, methodname=None, methodresponse=None, encoding=None, allow_none=False) - - Convert *params* into an XML-RPC request. or into a response if *methodresponse* - is true. *params* can be either a tuple of arguments or an instance of the - :exc:`Fault` exception class. If *methodresponse* is true, only a single value - can be returned, meaning that *params* must be of length 1. *encoding*, if - supplied, is the encoding to use in the generated XML; the default is UTF-8. - Python's :const:`None` value cannot be used in standard XML-RPC; to allow using - it via an extension, provide a true value for *allow_none*. - - -.. function:: loads(data, use_datetime=False, use_builtin_types=False) - - Convert an XML-RPC request or response into Python objects, a ``(params, - methodname)``. *params* is a tuple of argument; *methodname* is a string, or - ``None`` if no method name is present in the packet. If the XML-RPC packet - represents a fault condition, this function will raise a :exc:`Fault` exception. - The *use_builtin_types* flag can be used to cause date/time values to be - presented as :class:`datetime.datetime` objects and binary data to be - presented as :class:`bytes` objects; this flag is false by default. - - The obsolete *use_datetime* flag is similar to *use_builtin_types* but it - applies only to date/time values. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. _xmlrpc-client-example: - -Example of Client Usage ------------------------ - -:: - - # simple test program (from the XML-RPC specification) - from xmlrpc.client import ServerProxy, Error - - # server = ServerProxy("http://localhost:8000") # local server - with ServerProxy("http://betty.userland.com") as proxy: - - print(proxy) - - try: - print(proxy.examples.getStateName(41)) - except Error as v: - print("ERROR", v) - -To access an XML-RPC server through a HTTP proxy, you need to define a custom -transport. The following example shows how:: - - import http.client - import xmlrpc.client - - class ProxiedTransport(xmlrpc.client.Transport): - - def set_proxy(self, host, port=None, headers=None): - self.proxy = host, port - self.proxy_headers = headers - - def make_connection(self, host): - connection = http.client.HTTPConnection(*self.proxy) - connection.set_tunnel(host, headers=self.proxy_headers) - self._connection = host, connection - return connection - - transport = ProxiedTransport() - transport.set_proxy('proxy-server', 8080) - server = xmlrpc.client.ServerProxy('http://betty.userland.com', transport=transport) - print(server.examples.getStateName(41)) - - -Example of Client and Server Usage ----------------------------------- - -See :ref:`simplexmlrpcserver-example`. - - -.. rubric:: Footnotes - -.. [#] This approach has been first presented in `a discussion on xmlrpc.com - `_. -.. the link now points to webarchive since the one at -.. http://www.xmlrpc.com/discuss/msgReader%241208 is broken (and webadmin -.. doesn't reply) diff --git a/third_party/python/Doc/library/xmlrpc.rst b/third_party/python/Doc/library/xmlrpc.rst deleted file mode 100644 index ae68157b0..000000000 --- a/third_party/python/Doc/library/xmlrpc.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`xmlrpc` --- XMLRPC server and client modules -================================================== - -XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a -transport. With it, a client can call methods with parameters on a remote -server (the server is named by a URI) and get back structured data. - -``xmlrpc`` is a package that collects server and client modules implementing -XML-RPC. The modules are: - -* :mod:`xmlrpc.client` -* :mod:`xmlrpc.server` diff --git a/third_party/python/Doc/library/xmlrpc.server.rst b/third_party/python/Doc/library/xmlrpc.server.rst deleted file mode 100644 index 0511ddf47..000000000 --- a/third_party/python/Doc/library/xmlrpc.server.rst +++ /dev/null @@ -1,402 +0,0 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== - -.. module:: xmlrpc.server - :synopsis: Basic XML-RPC server implementations. - -.. moduleauthor:: Brian Quinlan -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/xmlrpc/server.py` - --------------- - -The :mod:`xmlrpc.server` module provides a basic server framework for XML-RPC -servers written in Python. Servers can either be free standing, using -:class:`SimpleXMLRPCServer`, or embedded in a CGI environment, using -:class:`CGIXMLRPCRequestHandler`. - - -.. warning:: - - The :mod:`xmlrpc.server` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - -.. class:: SimpleXMLRPCServer(addr, requestHandler=SimpleXMLRPCRequestHandler,\ - logRequests=True, allow_none=False, encoding=None,\ - bind_and_activate=True, use_builtin_types=False) - - Create a new server instance. This class provides methods for registration of - functions that can be called by the XML-RPC protocol. The *requestHandler* - parameter should be a factory for request handler instances; it defaults to - :class:`SimpleXMLRPCRequestHandler`. The *addr* and *requestHandler* parameters - are passed to the :class:`socketserver.TCPServer` constructor. If *logRequests* - is true (the default), requests will be logged; setting this parameter to false - will turn off logging. The *allow_none* and *encoding* parameters are passed - on to :mod:`xmlrpc.client` and control the XML-RPC responses that will be returned - from the server. The *bind_and_activate* parameter controls whether - :meth:`server_bind` and :meth:`server_activate` are called immediately by the - constructor; it defaults to true. Setting it to false allows code to manipulate - the *allow_reuse_address* class variable before the address is bound. - The *use_builtin_types* parameter is passed to the - :func:`~xmlrpc.client.loads` function and controls which types are processed - when date/times values or binary data are received; it defaults to false. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: CGIXMLRPCRequestHandler(allow_none=False, encoding=None,\ - use_builtin_types=False) - - Create a new instance to handle XML-RPC requests in a CGI environment. The - *allow_none* and *encoding* parameters are passed on to :mod:`xmlrpc.client` - and control the XML-RPC responses that will be returned from the server. - The *use_builtin_types* parameter is passed to the - :func:`~xmlrpc.client.loads` function and controls which types are processed - when date/times values or binary data are received; it defaults to false. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: SimpleXMLRPCRequestHandler() - - Create a new request handler instance. This request handler supports ``POST`` - requests and modifies logging so that the *logRequests* parameter to the - :class:`SimpleXMLRPCServer` constructor parameter is honored. - - -.. _simple-xmlrpc-servers: - -SimpleXMLRPCServer Objects --------------------------- - -The :class:`SimpleXMLRPCServer` class is based on -:class:`socketserver.TCPServer` and provides a means of creating simple, stand -alone XML-RPC servers. - - -.. method:: SimpleXMLRPCServer.register_function(function, name=None) - - Register a function that can respond to XML-RPC requests. If *name* is given, - it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* can be either a normal or Unicode - string, and may contain characters not legal in Python identifiers, including - the period character. - - -.. method:: SimpleXMLRPCServer.register_instance(instance, allow_dotted_names=False) - - Register an object which is used to expose method names which have not been - registered using :meth:`register_function`. If *instance* contains a - :meth:`_dispatch` method, it is called with the requested method name and the - parameters from the request. Its API is ``def _dispatch(self, method, params)`` - (note that *params* does not represent a variable argument list). If it calls - an underlying function to perform its task, that function is called as - ``func(*params)``, expanding the parameter list. The return value from - :meth:`_dispatch` is returned to the client as the result. If *instance* does - not have a :meth:`_dispatch` method, it is searched for an attribute matching - the name of the requested method. - - If the optional *allow_dotted_names* argument is true and the instance does not - have a :meth:`_dispatch` method, then if the requested method name contains - periods, each component of the method name is searched for individually, with - the effect that a simple hierarchical search is performed. The value found from - this search is then called with the parameters from the request, and the return - value is passed back to the client. - - .. warning:: - - Enabling the *allow_dotted_names* option allows intruders to access your - module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this option on a secure, closed network. - - -.. method:: SimpleXMLRPCServer.register_introspection_functions() - - Registers the XML-RPC introspection functions ``system.listMethods``, - ``system.methodHelp`` and ``system.methodSignature``. - - -.. method:: SimpleXMLRPCServer.register_multicall_functions() - - Registers the XML-RPC multicall function system.multicall. - - -.. attribute:: SimpleXMLRPCRequestHandler.rpc_paths - - An attribute value that must be a tuple listing valid path portions of the URL - for receiving XML-RPC requests. Requests posted to other paths will result in a - 404 "no such page" HTTP error. If this tuple is empty, all paths will be - considered valid. The default value is ``('/', '/RPC2')``. - - -.. _simplexmlrpcserver-example: - -SimpleXMLRPCServer Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Server code:: - - from xmlrpc.server import SimpleXMLRPCServer - from xmlrpc.server import SimpleXMLRPCRequestHandler - - # Restrict to a particular path. - class RequestHandler(SimpleXMLRPCRequestHandler): - rpc_paths = ('/RPC2',) - - # Create server - with SimpleXMLRPCServer(("localhost", 8000), - requestHandler=RequestHandler) as server: - server.register_introspection_functions() - - # Register pow() function; this will use the value of - # pow.__name__ as the name, which is just 'pow'. - server.register_function(pow) - - # Register a function under a different name - def adder_function(x,y): - return x + y - server.register_function(adder_function, 'add') - - # Register an instance; all the methods of the instance are - # published as XML-RPC methods (in this case, just 'mul'). - class MyFuncs: - def mul(self, x, y): - return x * y - - server.register_instance(MyFuncs()) - - # Run the server's main loop - server.serve_forever() - -The following client code will call the methods made available by the preceding -server:: - - import xmlrpc.client - - s = xmlrpc.client.ServerProxy('http://localhost:8000') - print(s.pow(2,3)) # Returns 2**3 = 8 - print(s.add(2,3)) # Returns 5 - print(s.mul(5,2)) # Returns 5*2 = 10 - - # Print list of available methods - print(s.system.listMethods()) - -The following example included in the :file:`Lib/xmlrpc/server.py` module shows -a server allowing dotted names and registering a multicall function. - -.. warning:: - - Enabling the *allow_dotted_names* option allows intruders to access your - module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this example only within a secure, closed network. - -:: - - import datetime - - class ExampleService: - def getData(self): - return '42' - - class currentTime: - @staticmethod - def getCurrentTime(): - return datetime.datetime.now() - - with SimpleXMLRPCServer(("localhost", 8000)) as server: - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - sys.exit(0) - -This ExampleService demo can be invoked from the command line:: - - python -m xmlrpc.server - - -The client that interacts with the above server is included in -`Lib/xmlrpc/client.py`:: - - server = ServerProxy("http://localhost:8000") - - try: - print(server.currentTime.getCurrentTime()) - except Error as v: - print("ERROR", v) - - multi = MultiCall(server) - multi.getData() - multi.pow(2,9) - multi.add(1,2) - try: - for response in multi(): - print(response) - except Error as v: - print("ERROR", v) - -This client which interacts with the demo XMLRPC server can be invoked as:: - - python -m xmlrpc.client - - -CGIXMLRPCRequestHandler ------------------------ - -The :class:`CGIXMLRPCRequestHandler` class can be used to handle XML-RPC -requests sent to Python CGI scripts. - - -.. method:: CGIXMLRPCRequestHandler.register_function(function, name=None) - - Register a function that can respond to XML-RPC requests. If *name* is given, - it will be the method name associated with function, otherwise - *function.__name__* will be used. *name* can be either a normal or Unicode - string, and may contain characters not legal in Python identifiers, including - the period character. - - -.. method:: CGIXMLRPCRequestHandler.register_instance(instance) - - Register an object which is used to expose method names which have not been - registered using :meth:`register_function`. If instance contains a - :meth:`_dispatch` method, it is called with the requested method name and the - parameters from the request; the return value is returned to the client as the - result. If instance does not have a :meth:`_dispatch` method, it is searched - for an attribute matching the name of the requested method; if the requested - method name contains periods, each component of the method name is searched for - individually, with the effect that a simple hierarchical search is performed. - The value found from this search is then called with the parameters from the - request, and the return value is passed back to the client. - - -.. method:: CGIXMLRPCRequestHandler.register_introspection_functions() - - Register the XML-RPC introspection functions ``system.listMethods``, - ``system.methodHelp`` and ``system.methodSignature``. - - -.. method:: CGIXMLRPCRequestHandler.register_multicall_functions() - - Register the XML-RPC multicall function ``system.multicall``. - - -.. method:: CGIXMLRPCRequestHandler.handle_request(request_text=None) - - Handle an XML-RPC request. If *request_text* is given, it should be the POST - data provided by the HTTP server, otherwise the contents of stdin will be used. - -Example:: - - class MyFuncs: - def mul(self, x, y): - return x * y - - - handler = CGIXMLRPCRequestHandler() - handler.register_function(pow) - handler.register_function(lambda x,y: x+y, 'add') - handler.register_introspection_functions() - handler.register_instance(MyFuncs()) - handler.handle_request() - - -Documenting XMLRPC server -------------------------- - -These classes extend the above classes to serve HTML documentation in response -to HTTP GET requests. Servers can either be free standing, using -:class:`DocXMLRPCServer`, or embedded in a CGI environment, using -:class:`DocCGIXMLRPCRequestHandler`. - - -.. class:: DocXMLRPCServer(addr, requestHandler=DocXMLRPCRequestHandler,\ - logRequests=True, allow_none=False, encoding=None,\ - bind_and_activate=True, use_builtin_types=True) - - Create a new server instance. All parameters have the same meaning as for - :class:`SimpleXMLRPCServer`; *requestHandler* defaults to - :class:`DocXMLRPCRequestHandler`. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: DocCGIXMLRPCRequestHandler() - - Create a new instance to handle XML-RPC requests in a CGI environment. - - -.. class:: DocXMLRPCRequestHandler() - - Create a new request handler instance. This request handler supports XML-RPC - POST requests, documentation GET requests, and modifies logging so that the - *logRequests* parameter to the :class:`DocXMLRPCServer` constructor parameter is - honored. - - -.. _doc-xmlrpc-servers: - -DocXMLRPCServer Objects ------------------------ - -The :class:`DocXMLRPCServer` class is derived from :class:`SimpleXMLRPCServer` -and provides a means of creating self-documenting, stand alone XML-RPC -servers. HTTP POST requests are handled as XML-RPC method calls. HTTP GET -requests are handled by generating pydoc-style HTML documentation. This allows a -server to provide its own web-based documentation. - - -.. method:: DocXMLRPCServer.set_server_title(server_title) - - Set the title used in the generated HTML documentation. This title will be used - inside the HTML "title" element. - - -.. method:: DocXMLRPCServer.set_server_name(server_name) - - Set the name used in the generated HTML documentation. This name will appear at - the top of the generated documentation inside a "h1" element. - - -.. method:: DocXMLRPCServer.set_server_documentation(server_documentation) - - Set the description used in the generated HTML documentation. This description - will appear as a paragraph, below the server name, in the documentation. - - -DocCGIXMLRPCRequestHandler --------------------------- - -The :class:`DocCGIXMLRPCRequestHandler` class is derived from -:class:`CGIXMLRPCRequestHandler` and provides a means of creating -self-documenting, XML-RPC CGI scripts. HTTP POST requests are handled as XML-RPC -method calls. HTTP GET requests are handled by generating pydoc-style HTML -documentation. This allows a server to provide its own web-based documentation. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_title(server_title) - - Set the title used in the generated HTML documentation. This title will be used - inside the HTML "title" element. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_name(server_name) - - Set the name used in the generated HTML documentation. This name will appear at - the top of the generated documentation inside a "h1" element. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_documentation(server_documentation) - - Set the description used in the generated HTML documentation. This description - will appear as a paragraph, below the server name, in the documentation. diff --git a/third_party/python/Doc/library/zipapp.rst b/third_party/python/Doc/library/zipapp.rst deleted file mode 100644 index 96586947c..000000000 --- a/third_party/python/Doc/library/zipapp.rst +++ /dev/null @@ -1,427 +0,0 @@ -:mod:`zipapp` --- Manage executable python zip archives -======================================================= - -.. module:: zipapp - :synopsis: Manage executable python zip archives - -.. versionadded:: 3.5 - -**Source code:** :source:`Lib/zipapp.py` - -.. index:: - single: Executable Zip Files - --------------- - -This module provides tools to manage the creation of zip files containing -Python code, which can be :ref:`executed directly by the Python interpreter -`. The module provides both a -:ref:`zipapp-command-line-interface` and a :ref:`zipapp-python-api`. - - -Basic Example -------------- - -The following example shows how the :ref:`zipapp-command-line-interface` -can be used to create an executable archive from a directory containing -Python code. When run, the archive will execute the ``main`` function from -the module ``myapp`` in the archive. - -.. code-block:: shell-session - - $ python -m zipapp myapp -m "myapp:main" - $ python myapp.pyz - - - -.. _zipapp-command-line-interface: - -Command-Line Interface ----------------------- - -When called as a program from the command line, the following form is used: - -.. code-block:: shell-session - - $ python -m zipapp source [options] - -If *source* is a directory, this will create an archive from the contents of -*source*. If *source* is a file, it should be an archive, and it will be -copied to the target archive (or the contents of its shebang line will be -displayed if the --info option is specified). - -The following options are understood: - -.. program:: zipapp - -.. cmdoption:: -o , --output= - - Write the output to a file named *output*. If this option is not specified, - the output filename will be the same as the input *source*, with the - extension ``.pyz`` added. If an explicit filename is given, it is used as - is (so a ``.pyz`` extension should be included if required). - - An output filename must be specified if the *source* is an archive (and in - that case, *output* must not be the same as *source*). - -.. cmdoption:: -p , --python= - - Add a ``#!`` line to the archive specifying *interpreter* as the command - to run. Also, on POSIX, make the archive executable. The default is to - write no ``#!`` line, and not make the file executable. - -.. cmdoption:: -m , --main= - - Write a ``__main__.py`` file to the archive that executes *mainfn*. The - *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a - package/module in the archive, and "fn" is a callable in the given module. - The ``__main__.py`` file will execute that callable. - - :option:`--main` cannot be specified when copying an archive. - -.. cmdoption:: --info - - Display the interpreter embedded in the archive, for diagnostic purposes. In - this case, any other options are ignored and SOURCE must be an archive, not a - directory. - -.. cmdoption:: -h, --help - - Print a short usage message and exit. - - -.. _zipapp-python-api: - -Python API ----------- - -The module defines two convenience functions: - - -.. function:: create_archive(source, target=None, interpreter=None, main=None) - - Create an application archive from *source*. The source can be any - of the following: - - * The name of a directory, or a :class:`pathlib.Path` object referring - to a directory, in which case a new application archive will be - created from the content of that directory. - * The name of an existing application archive file, or a :class:`pathlib.Path` - object referring to such a file, in which case the file is copied to - the target (modifying it to reflect the value given for the *interpreter* - argument). The file name should include the ``.pyz`` extension, if required. - * A file object open for reading in bytes mode. The content of the - file should be an application archive, and the file object is - assumed to be positioned at the start of the archive. - - The *target* argument determines where the resulting archive will be - written: - - * If it is the name of a file, or a :class:`pathlb.Path` object, - the archive will be written to that file. - * If it is an open file object, the archive will be written to that - file object, which must be open for writing in bytes mode. - * If the target is omitted (or ``None``), the source must be a directory - and the target will be a file with the same name as the source, with - a ``.pyz`` extension added. - - The *interpreter* argument specifies the name of the Python - interpreter with which the archive will be executed. It is written as - a "shebang" line at the start of the archive. On POSIX, this will be - interpreted by the OS, and on Windows it will be handled by the Python - launcher. Omitting the *interpreter* results in no shebang line being - written. If an interpreter is specified, and the target is a - filename, the executable bit of the target file will be set. - - The *main* argument specifies the name of a callable which will be - used as the main program for the archive. It can only be specified if - the source is a directory, and the source does not already contain a - ``__main__.py`` file. The *main* argument should take the form - "pkg.module:callable" and the archive will be run by importing - "pkg.module" and executing the given callable with no arguments. It - is an error to omit *main* if the source is a directory and does not - contain a ``__main__.py`` file, as otherwise the resulting archive - would not be executable. - - If a file object is specified for *source* or *target*, it is the - caller's responsibility to close it after calling create_archive. - - When copying an existing archive, file objects supplied only need - ``read`` and ``readline``, or ``write`` methods. When creating an - archive from a directory, if the target is a file object it will be - passed to the ``zipfile.ZipFile`` class, and must supply the methods - needed by that class. - -.. function:: get_interpreter(archive) - - Return the interpreter specified in the ``#!`` line at the start of the - archive. If there is no ``#!`` line, return :const:`None`. - The *archive* argument can be a filename or a file-like object open - for reading in bytes mode. It is assumed to be at the start of the archive. - - -.. _zipapp-examples: - -Examples --------- - -Pack up a directory into an archive, and run it. - -.. code-block:: shell-session - - $ python -m zipapp myapp - $ python myapp.pyz - - -The same can be done using the :func:`create_archive` functon:: - - >>> import zipapp - >>> zipapp.create_archive('myapp.pyz', 'myapp') - -To make the application directly executable on POSIX, specify an interpreter -to use. - -.. code-block:: shell-session - - $ python -m zipapp myapp -p "/usr/bin/env python" - $ ./myapp.pyz - - -To replace the shebang line on an existing archive, create a modified archive -using the :func:`create_archive` function:: - - >>> import zipapp - >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') - -To update the file in place, do the replacement in memory using a :class:`BytesIO` -object, and then overwrite the source afterwards. Note that there is a risk -when overwriting a file in place that an error will result in the loss of -the original file. This code does not protect against such errors, but -production code should do so. Also, this method will only work if the archive -fits in memory:: - - >>> import zipapp - >>> import io - >>> temp = io.BytesIO() - >>> zipapp.create_archive('myapp.pyz', temp, '/usr/bin/python2') - >>> with open('myapp.pyz', 'wb') as f: - >>> f.write(temp.getvalue()) - - -.. _zipapp-specifying-the-interpreter: - -Specifying the Interpreter --------------------------- - -Note that if you specify an interpreter and then distribute your application -archive, you need to ensure that the interpreter used is portable. The Python -launcher for Windows supports most common forms of POSIX ``#!`` line, but there -are other issues to consider: - -* If you use "/usr/bin/env python" (or other forms of the "python" command, - such as "/usr/bin/python"), you need to consider that your users may have - either Python 2 or Python 3 as their default, and write your code to work - under both versions. -* If you use an explicit version, for example "/usr/bin/env python3" your - application will not work for users who do not have that version. (This - may be what you want if you have not made your code Python 2 compatible). -* There is no way to say "python X.Y or later", so be careful of using an - exact version like "/usr/bin/env python3.4" as you will need to change your - shebang line for users of Python 3.5, for example. - -Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", -depending on whether your code is written for Python 2 or 3. - - -Creating Standalone Applications with zipapp --------------------------------------------- - -Using the :mod:`zipapp` module, it is possible to create self-contained Python -programs, which can be distributed to end users who only need to have a -suitable version of Python installed on their system. The key to doing this -is to bundle all of the application's dependencies into the archive, along -with the application code. - -The steps to create a standalone archive are as follows: - -1. Create your application in a directory as normal, so you have a ``myapp`` - directory containing a ``__main__.py`` file, and any supporting application - code. - -2. Install all of your application's dependencies into the ``myapp`` directory, - using pip: - - .. code-block:: shell-session - - $ python -m pip install -r requirements.txt --target myapp - - (this assumes you have your project requirements in a ``requirements.txt`` - file - if not, you can just list the dependencies manually on the pip command - line). - -3. Optionally, delete the ``.dist-info`` directories created by pip in the - ``myapp`` directory. These hold metadata for pip to manage the packages, and - as you won't be making any further use of pip they aren't required - - although it won't do any harm if you leave them. - -4. Package the application using: - - .. code-block:: shell-session - - $ python -m zipapp -p "interpreter" myapp - -This will produce a standalone executable, which can be run on any machine with -the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` -for details. It can be shipped to users as a single file. - -On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the -file to remove the ``.pyz`` extension if you prefer a "plain" command name. On -Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that -the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions -when installed. - - -Making a Windows executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Windows, registration of the ``.pyz`` extension is optional, and -furthermore, there are certain places that don't recognise registered -extensions "transparently" (the simplest example is that -``subprocess.run(['myapp'])`` won't find your application - you need to -explicitly specify the extension). - -On Windows, therefore, it is often preferable to create an executable from the -zipapp. This is relatively easy, although it does require a C compiler. The -basic approach relies on the fact that zipfiles can have arbitrary data -prepended, and Windows exe files can have arbitrary data appended. So by -creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, -you end up with a single-file executable that runs your application. - -A suitable launcher can be as simple as the following:: - - #define Py_LIMITED_API 1 - #include "Python.h" - - #define WIN32_LEAN_AND_MEAN - #include - - #ifdef WINDOWS - int WINAPI wWinMain( - HINSTANCE hInstance, /* handle to current instance */ - HINSTANCE hPrevInstance, /* handle to previous instance */ - LPWSTR lpCmdLine, /* pointer to command line */ - int nCmdShow /* show state of window */ - ) - #else - int wmain() - #endif - { - wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); - myargv[0] = __wargv[0]; - memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); - return Py_Main(__argc+1, myargv); - } - -If you define the ``WINDOWS`` preprocessor symbol, this will generate a -GUI executable, and without it, a console executable. - -To compile the executable, you can either just use the standard MSVC -command line tools, or you can take advantage of the fact that distutils -knows how to compile Python source:: - - >>> from distutils.ccompiler import new_compiler - >>> import distutils.sysconfig - >>> import sys - >>> import os - >>> from pathlib import Path - - >>> def compile(src): - >>> src = Path(src) - >>> cc = new_compiler() - >>> exe = src.stem - >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) - >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) - >>> # First the CLI executable - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe) - >>> # Now the GUI executable - >>> cc.define_macro('WINDOWS') - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe + 'w') - - >>> if __name__ == "__main__": - >>> compile("zastub.c") - -The resulting launcher uses the "Limited ABI", so it will run unchanged with -any version of Python 3.x. All it needs is for Python (``python3.dll``) to be -on the user's ``PATH``. - -For a fully standalone distribution, you can distribute the launcher with your -application appended, bundled with the Python "embedded" distribution. This -will run on any PC with the appropriate architecture (32 bit or 64 bit). - - -Caveats -~~~~~~~ - -There are some limitations to the process of bundling your application into -a single file. In most, if not all, cases they can be addressed without -needing major changes to your application. - -1. If your application depends on a package that includes a C extension, that - package cannot be run from a zip file (this is an OS limitation, as executable - code must be present in the filesystem for the OS loader to load it). In this - case, you can exclude that dependency from the zipfile, and either require - your users to have it installed, or ship it alongside your zipfile and add code - to your ``__main__.py`` to include the directory containing the unzipped - module in ``sys.path``. In this case, you will need to make sure to ship - appropriate binaries for your target architecture(s) (and potentially pick the - correct version to add to ``sys.path`` at runtime, based on the user's machine). - -2. If you are shipping a Windows executable as described above, you either need to - ensure that your users have ``python3.dll`` on their PATH (which is not the - default behaviour of the installer) or you should bundle your application with - the embedded distribution. - -3. The suggested launcher above uses the Python embedding API. This means that in - your application, ``sys.executable`` will be your application, and *not* a - conventional Python interpreter. Your code and its dependencies need to be - prepared for this possibility. For example, if your application uses the - :mod:`multiprocessing` module, it will need to call - :func:`multiprocessing.set_executable` to let the module know where to find the - standard Python interpreter. - - -The Python Zip Application Archive Format ------------------------------------------ - -Python has been able to execute zip files which contain a ``__main__.py`` file -since version 2.6. In order to be executed by Python, an application archive -simply has to be a standard zip file containing a ``__main__.py`` file which -will be run as the entry point for the application. As usual for any Python -script, the parent of the script (in this case the zip file) will be placed on -:data:`sys.path` and thus further modules can be imported from the zip file. - -The zip file format allows arbitrary data to be prepended to a zip file. The -zip application format uses this ability to prepend a standard POSIX "shebang" -line to the file (``#!/path/to/interpreter``). - -Formally, the Python zip application format is therefore: - -1. An optional shebang line, containing the characters ``b'#!'`` followed by an - interpreter name, and then a newline (``b'\n'``) character. The interpreter - name can be anything acceptable to the OS "shebang" processing, or the Python - launcher on Windows. The interpreter should be encoded in UTF-8 on Windows, - and in :func:`sys.getfilesystemencoding()` on POSIX. -2. Standard zipfile data, as generated by the :mod:`zipfile` module. The - zipfile content *must* include a file called ``__main__.py`` (which must be - in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The - zipfile data can be compressed or uncompressed. - -If an application archive has a shebang line, it may have the executable bit set -on POSIX systems, to allow it to be executed directly. - -There is no requirement that the tools in this module are used to create -application archives - the module is a convenience, but archives in the above -format created by any means are acceptable to Python. - diff --git a/third_party/python/Doc/library/zipfile.rst b/third_party/python/Doc/library/zipfile.rst deleted file mode 100644 index b65b61d8d..000000000 --- a/third_party/python/Doc/library/zipfile.rst +++ /dev/null @@ -1,707 +0,0 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= - -.. module:: zipfile - :synopsis: Read and write ZIP-format archive files. - -.. moduleauthor:: James C. Ahlstrom -.. sectionauthor:: James C. Ahlstrom - -**Source code:** :source:`Lib/zipfile.py` - --------------- - -The ZIP file format is a common archive and compression standard. This module -provides tools to create, read, write, append, and list a ZIP file. Any -advanced use of this module will require an understanding of the format, as -defined in `PKZIP Application Note`_. - -This module does not currently handle multi-disk ZIP files. -It can handle ZIP files that use the ZIP64 extensions -(that is ZIP files that are more than 4 GiB in size). It supports -decryption of encrypted files in ZIP archives, but it currently cannot -create an encrypted file. Decryption is extremely slow as it is -implemented in native Python rather than C. - -The module defines the following items: - -.. exception:: BadZipFile - - The error raised for bad ZIP files. - - .. versionadded:: 3.2 - - -.. exception:: BadZipfile - - Alias of :exc:`BadZipFile`, for compatibility with older Python versions. - - .. deprecated:: 3.2 - - -.. exception:: LargeZipFile - - The error raised when a ZIP file would require ZIP64 functionality but that has - not been enabled. - - -.. class:: ZipFile - :noindex: - - The class for reading and writing ZIP files. See section - :ref:`zipfile-objects` for constructor details. - - -.. class:: PyZipFile - :noindex: - - Class for creating ZIP archives containing Python libraries. - - -.. class:: ZipInfo(filename='NoName', date_time=(1980,1,1,0,0,0)) - - Class used to represent information about a member of an archive. Instances - of this class are returned by the :meth:`.getinfo` and :meth:`.infolist` - methods of :class:`ZipFile` objects. Most users of the :mod:`zipfile` module - will not need to create these, but only use those created by this - module. *filename* should be the full name of the archive member, and - *date_time* should be a tuple containing six fields which describe the time - of the last modification to the file; the fields are described in section - :ref:`zipinfo-objects`. - - -.. function:: is_zipfile(filename) - - Returns ``True`` if *filename* is a valid ZIP file based on its magic number, - otherwise returns ``False``. *filename* may be a file or file-like object too. - - .. versionchanged:: 3.1 - Support for file and file-like objects. - - -.. data:: ZIP_STORED - - The numeric constant for an uncompressed archive member. - - -.. data:: ZIP_DEFLATED - - The numeric constant for the usual ZIP compression method. This requires the - :mod:`zlib` module. - - -.. data:: ZIP_BZIP2 - - The numeric constant for the BZIP2 compression method. This requires the - :mod:`bz2` module. - - .. versionadded:: 3.3 - -.. data:: ZIP_LZMA - - The numeric constant for the LZMA compression method. This requires the - :mod:`lzma` module. - - .. versionadded:: 3.3 - - .. note:: - - The ZIP file format specification has included support for bzip2 compression - since 2001, and for LZMA compression since 2006. However, some tools - (including older Python releases) do not support these compression - methods, and may either refuse to process the ZIP file altogether, - or fail to extract individual files. - - -.. seealso:: - - `PKZIP Application Note`_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - `Info-ZIP Home Page `_ - Information about the Info-ZIP project's ZIP archive programs and development - libraries. - - -.. _zipfile-objects: - -ZipFile Objects ---------------- - - -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) - - Open a ZIP file, where *file* can be a path to a file (a string), a - file-like object or a :term:`path-like object`. - The *mode* parameter should be ``'r'`` to read an existing - file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an - existing file, or ``'x'`` to exclusively create and write a new file. - If *mode* is ``'x'`` and *file* refers to an existing file, - a :exc:`FileExistsError` will be raised. - If *mode* is ``'a'`` and *file* refers to an existing ZIP - file, then additional files are added to it. If *file* does not refer to a - ZIP file, then a new ZIP archive is appended to the file. This is meant for - adding a ZIP archive to another file (such as :file:`python.exe`). If - *mode* is ``'a'`` and the file does not exist at all, it is created. - If *mode* is ``'r'`` or ``'a'``, the file should be seekable. - *compression* is the ZIP compression method to use when writing the archive, - and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module - (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError` - is raised. The default is :const:`ZIP_STORED`. If *allowZip64* is - ``True`` (the default) zipfile will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 4 GiB. If it is false :mod:`zipfile` - will raise an exception when the ZIP file would require ZIP64 extensions. - - If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then - :meth:`closed ` without adding any files to the archive, the appropriate - ZIP structures for an empty archive will be written to the file. - - ZipFile is also a context manager and therefore supports the - :keyword:`with` statement. In the example, *myzip* is closed after the - :keyword:`with` statement's suite is finished---even if an exception occurs:: - - with ZipFile('spam.zip', 'w') as myzip: - myzip.write('eggs.txt') - - .. versionadded:: 3.2 - Added the ability to use :class:`ZipFile` as a context manager. - - .. versionchanged:: 3.3 - Added support for :mod:`bzip2 ` and :mod:`lzma` compression. - - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - - .. versionchanged:: 3.5 - Added support for writing to unseekable streams. - Added support for the ``'x'`` mode. - - .. versionchanged:: 3.6 - Previously, a plain :exc:`RuntimeError` was raised for unrecognized - compression values. - - .. versionchanged:: 3.6.2 - The *file* parameter accepts a :term:`path-like object`. - - -.. method:: ZipFile.close() - - Close the archive file. You must call :meth:`close` before exiting your program - or essential records will not be written. - - -.. method:: ZipFile.getinfo(name) - - Return a :class:`ZipInfo` object with information about the archive member - *name*. Calling :meth:`getinfo` for a name not currently contained in the - archive will raise a :exc:`KeyError`. - - -.. method:: ZipFile.infolist() - - Return a list containing a :class:`ZipInfo` object for each member of the - archive. The objects are in the same order as their entries in the actual ZIP - file on disk if an existing archive was opened. - - -.. method:: ZipFile.namelist() - - Return a list of archive members by name. - - -.. method:: ZipFile.open(name, mode='r', pwd=None, *, force_zip64=False) - - Access a member of the archive as a binary file-like object. *name* - can be either the name of a file within the archive or a :class:`ZipInfo` - object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files. - - :meth:`~ZipFile.open` is also a context manager and therefore supports the - :keyword:`with` statement:: - - with ZipFile('spam.zip') as myzip: - with myzip.open('eggs.txt') as myfile: - print(myfile.read()) - - With *mode* ``'r'`` the file-like object - (``ZipExtFile``) is read-only and provides the following methods: - :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. These objects can operate independently of - the ZipFile. - - With ``mode='w'``, a writable file handle is returned, which supports the - :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, - attempting to read or write other files in the ZIP file will raise a - :exc:`ValueError`. - - When writing a file, if the file size is not known in advance but may exceed - 2 GiB, pass ``force_zip64=True`` to ensure that the header format is - capable of supporting large files. If the file size is known in advance, - construct a :class:`ZipInfo` object with :attr:`~ZipInfo.file_size` set, and - use that as the *name* parameter. - - .. note:: - - The :meth:`.open`, :meth:`read` and :meth:`extract` methods can take a filename - or a :class:`ZipInfo` object. You will appreciate this when trying to read a - ZIP file that contains members with duplicate names. - - .. versionchanged:: 3.6 - Removed support of ``mode='U'``. Use :class:`io.TextIOWrapper` for reading - compressed text files in :term:`universal newlines` mode. - - .. versionchanged:: 3.6 - :meth:`open` can now be used to write files into the archive with the - ``mode='w'`` option. - - .. versionchanged:: 3.6 - Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. - Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.extract(member, path=None, pwd=None) - - Extract a member from the archive to the current working directory; *member* - must be its full name or a :class:`ZipInfo` object. Its file information is - extracted as accurately as possible. *path* specifies a different directory - to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files. - - Returns the normalized path created (a directory or new file). - - .. note:: - - If a member filename is an absolute path, a drive/UNC sharepoint and - leading (back)slashes will be stripped, e.g.: ``///foo/bar`` becomes - ``foo/bar`` on Unix, and ``C:\foo\bar`` becomes ``foo\bar`` on Windows. - And all ``".."`` components in a member filename will be removed, e.g.: - ``../../foo../../ba..r`` becomes ``foo../ba..r``. On Windows illegal - characters (``:``, ``<``, ``>``, ``|``, ``"``, ``?``, and ``*``) - replaced by underscore (``_``). - - .. versionchanged:: 3.6 - Calling :meth:`extract` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - .. versionchanged:: 3.6.2 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: ZipFile.extractall(path=None, members=None, pwd=None) - - Extract all members from the archive to the current working directory. *path* - specifies a different directory to extract to. *members* is optional and must - be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files. - - .. warning:: - - Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. This module attempts to prevent that. - See :meth:`extract` note. - - .. versionchanged:: 3.6 - Calling :meth:`extractall` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - .. versionchanged:: 3.6.2 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: ZipFile.printdir() - - Print a table of contents for the archive to ``sys.stdout``. - - -.. method:: ZipFile.setpassword(pwd) - - Set *pwd* as default password to extract encrypted files. - - -.. method:: ZipFile.read(name, pwd=None) - - Return the bytes of the file *name* in the archive. *name* is the name of the - file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files and, if specified, - it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a ZipFile that uses a compression method other than - :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or - :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also - be raised if the corresponding compression module is not available. - - .. versionchanged:: 3.6 - Calling :meth:`read` on a closed ZipFile will raise a :exc:`ValueError`. - Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.testzip() - - Read all the files in the archive and check their CRC's and file headers. - Return the name of the first bad file, or else return ``None``. - - .. versionchanged:: 3.6 - Calling :meth:`testzip` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.write(filename, arcname=None, compress_type=None) - - Write the file named *filename* to the archive, giving it the archive name - *arcname* (by default, this will be the same as *filename*, but without a drive - letter and with leading path separators removed). If given, *compress_type* - overrides the value given for the *compression* parameter to the constructor for - the new entry. - The archive must be open with mode ``'w'``, ``'x'`` or ``'a'``. - - .. note:: - - Archive names should be relative to the archive root, that is, they should not - start with a path separator. - - .. note:: - - If ``arcname`` (or ``filename``, if ``arcname`` is not given) contains a null - byte, the name of the file in the archive will be truncated at the null byte. - - .. versionchanged:: 3.6 - Calling :meth:`write` on a ZipFile created with mode ``'r'`` or - a closed ZipFile will raise a :exc:`ValueError`. Previously, - a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.writestr(zinfo_or_arcname, data[, compress_type]) - - Write a file into the archive. The contents is *data*, which may be either - a :class:`str` or a :class:`bytes` instance; if it is a :class:`str`, - it is encoded as UTF-8 first. *zinfo_or_arcname* is either the file - name it will be given in the archive, or a :class:`ZipInfo` instance. If it's - an instance, at least the filename, date, and time must be given. If it's a - name, the date and time is set to the current date and time. - The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'``. - - If given, *compress_type* overrides the value given for the *compression* - parameter to the constructor for the new entry, or in the *zinfo_or_arcname* - (if that is a :class:`ZipInfo` instance). - - .. note:: - - When passing a :class:`ZipInfo` instance as the *zinfo_or_arcname* parameter, - the compression method used will be that specified in the *compress_type* - member of the given :class:`ZipInfo` instance. By default, the - :class:`ZipInfo` constructor sets this member to :const:`ZIP_STORED`. - - .. versionchanged:: 3.2 - The *compress_type* argument. - - .. versionchanged:: 3.6 - Calling :meth:`writestr` on a ZipFile created with mode ``'r'`` or - a closed ZipFile will raise a :exc:`ValueError`. Previously, - a :exc:`RuntimeError` was raised. - - -The following data attributes are also available: - -.. attribute:: ZipFile.filename - - Name of the ZIP file. - -.. attribute:: ZipFile.debug - - The level of debug output to use. This may be set from ``0`` (the default, no - output) to ``3`` (the most output). Debugging information is written to - ``sys.stdout``. - -.. attribute:: ZipFile.comment - - The comment associated with the ZIP file as a :class:`bytes` object. - If assigning a comment to a - :class:`ZipFile` instance created with mode ``'w'``, ``'x'`` or ``'a'``, - it should be no longer than 65535 bytes. Comments longer than this will be - truncated. - - -.. _pyzipfile-objects: - -PyZipFile Objects ------------------ - -The :class:`PyZipFile` constructor takes the same parameters as the -:class:`ZipFile` constructor, and one additional parameter, *optimize*. - -.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ - optimize=-1) - - .. versionadded:: 3.2 - The *optimize* parameter. - - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - - Instances have one method in addition to those of :class:`ZipFile` objects: - - .. method:: PyZipFile.writepy(pathname, basename='', filterfunc=None) - - Search for files :file:`\*.py` and add the corresponding file to the - archive. - - If the *optimize* parameter to :class:`PyZipFile` was not given or ``-1``, - the corresponding file is a :file:`\*.pyc` file, compiling if necessary. - - If the *optimize* parameter to :class:`PyZipFile` was ``0``, ``1`` or - ``2``, only files with that optimization level (see :func:`compile`) are - added to the archive, compiling if necessary. - - If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.pyc`) file is added at the top level - (no path information). If *pathname* is a file that does not end with - :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, - and the directory is not a package directory, then all the files - :file:`\*.pyc` are added at the top level. If the directory is a - package directory, then all :file:`\*.pyc` are added under the package - name as a file path, and if any subdirectories are package directories, - all of these are added recursively. - - *basename* is intended for internal use only. - - *filterfunc*, if given, must be a function taking a single string - argument. It will be passed each path (including each individual full - file path) before it is added to the archive. If *filterfunc* returns a - false value, the path will not be added, and if it is a directory its - contents will be ignored. For example, if our test files are all either - in ``test`` directories or start with the string ``test_``, we can use a - *filterfunc* to exclude them:: - - >>> zf = PyZipFile('myprog.zip') - >>> def notests(s): - ... fn = os.path.basename(s) - ... return (not (fn == 'test' or fn.startswith('test_'))) - >>> zf.writepy('myprog', filterfunc=notests) - - The :meth:`writepy` method makes archives with file names like - this:: - - string.pyc # Top level name - test/__init__.pyc # Package directory - test/testall.pyc # Module test.testall - test/bogus/__init__.pyc # Subpackage directory - test/bogus/myfile.pyc # Submodule test.bogus.myfile - - .. versionadded:: 3.4 - The *filterfunc* parameter. - - .. versionchanged:: 3.6.2 - The *pathname* parameter accepts a :term:`path-like object`. - - -.. _zipinfo-objects: - -ZipInfo Objects ---------------- - -Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and -:meth:`.infolist` methods of :class:`ZipFile` objects. Each object stores -information about a single member of the ZIP archive. - -There is one classmethod to make a :class:`ZipInfo` instance for a filesystem -file: - -.. classmethod:: ZipInfo.from_file(filename, arcname=None) - - Construct a :class:`ZipInfo` instance for a file on the filesystem, in - preparation for adding it to a zip file. - - *filename* should be the path to a file or directory on the filesystem. - - If *arcname* is specified, it is used as the name within the archive. - If *arcname* is not specified, the name will be the same as *filename*, but - with any drive letter and leading path separators removed. - - .. versionadded:: 3.6 - - .. versionchanged:: 3.6.2 - The *filename* parameter accepts a :term:`path-like object`. - - -Instances have the following methods and attributes: - -.. method:: ZipInfo.is_dir() - - Return ``True`` if this archive member is a directory. - - This uses the entry's name: directories should always end with ``/``. - - .. versionadded:: 3.6 - - -.. attribute:: ZipInfo.filename - - Name of the file in the archive. - - -.. attribute:: ZipInfo.date_time - - The time and date of the last modification to the archive member. This is a - tuple of six values: - - +-------+--------------------------+ - | Index | Value | - +=======+==========================+ - | ``0`` | Year (>= 1980) | - +-------+--------------------------+ - | ``1`` | Month (one-based) | - +-------+--------------------------+ - | ``2`` | Day of month (one-based) | - +-------+--------------------------+ - | ``3`` | Hours (zero-based) | - +-------+--------------------------+ - | ``4`` | Minutes (zero-based) | - +-------+--------------------------+ - | ``5`` | Seconds (zero-based) | - +-------+--------------------------+ - - .. note:: - - The ZIP file format does not support timestamps before 1980. - - -.. attribute:: ZipInfo.compress_type - - Type of compression for the archive member. - - -.. attribute:: ZipInfo.comment - - Comment for the individual archive member as a :class:`bytes` object. - - -.. attribute:: ZipInfo.extra - - Expansion field data. The `PKZIP Application Note`_ contains - some comments on the internal structure of the data contained in this - :class:`bytes` object. - - -.. attribute:: ZipInfo.create_system - - System which created ZIP archive. - - -.. attribute:: ZipInfo.create_version - - PKZIP version which created ZIP archive. - - -.. attribute:: ZipInfo.extract_version - - PKZIP version needed to extract archive. - - -.. attribute:: ZipInfo.reserved - - Must be zero. - - -.. attribute:: ZipInfo.flag_bits - - ZIP flag bits. - - -.. attribute:: ZipInfo.volume - - Volume number of file header. - - -.. attribute:: ZipInfo.internal_attr - - Internal attributes. - - -.. attribute:: ZipInfo.external_attr - - External file attributes. - - -.. attribute:: ZipInfo.header_offset - - Byte offset to the file header. - - -.. attribute:: ZipInfo.CRC - - CRC-32 of the uncompressed file. - - -.. attribute:: ZipInfo.compress_size - - Size of the compressed data. - - -.. attribute:: ZipInfo.file_size - - Size of the uncompressed file. - - -.. _zipfile-commandline: -.. program:: zipfile - -Command-Line Interface ----------------------- - -The :mod:`zipfile` module provides a simple command-line interface to interact -with ZIP archives. - -If you want to create a new ZIP archive, specify its name after the :option:`-c` -option and then list the filename(s) that should be included: - -.. code-block:: shell-session - - $ python -m zipfile -c monty.zip spam.txt eggs.txt - -Passing a directory is also acceptable: - -.. code-block:: shell-session - - $ python -m zipfile -c monty.zip life-of-brian_1979/ - -If you want to extract a ZIP archive into the specified directory, use -the :option:`-e` option: - -.. code-block:: shell-session - - $ python -m zipfile -e monty.zip target-dir/ - -For a list of the files in a ZIP archive, use the :option:`-l` option: - -.. code-block:: shell-session - - $ python -m zipfile -l monty.zip - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -.. cmdoption:: -l - - List files in a zipfile. - -.. cmdoption:: -c ... - - Create zipfile from source files. - -.. cmdoption:: -e - - Extract zipfile into target directory. - -.. cmdoption:: -t - - Test whether the zipfile is valid or not. - - -.. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT diff --git a/third_party/python/Doc/library/zipimport.rst b/third_party/python/Doc/library/zipimport.rst deleted file mode 100644 index eaae2bb04..000000000 --- a/third_party/python/Doc/library/zipimport.rst +++ /dev/null @@ -1,167 +0,0 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== - -.. module:: zipimport - :synopsis: support for importing Python modules from ZIP archives. - -.. moduleauthor:: Just van Rossum - --------------- - -This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used -by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths -to ZIP archives. - -Typically, :data:`sys.path` is a list of directory names as strings. This module -also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. -The ZIP archive can contain a subdirectory structure to support package imports, -and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only -import from the :file:`lib/` subdirectory within the archive. - -Any files may be present in the ZIP archive, but only files :file:`.py` and -:file:`.pyc` are available for import. ZIP import of dynamic modules -(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains -:file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive -doesn't contain :file:`.pyc` files, importing may be rather slow. - -ZIP archives with an archive comment are currently not supported. - -.. seealso:: - - `PKZIP Application Note `_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - :pep:`273` - Import Modules from Zip Archives - Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 - follows the specification in PEP 273, but uses an implementation written by Just - van Rossum that uses the import hooks described in PEP 302. - - :pep:`302` - New Import Hooks - The PEP to add the import hooks that help this module work. - - -This module defines an exception: - -.. exception:: ZipImportError - - Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, - so it can be caught as :exc:`ImportError`, too. - - -.. _zipimporter-objects: - -zipimporter Objects -------------------- - -:class:`zipimporter` is the class for importing ZIP files. - -.. class:: zipimporter(archivepath) - - Create a new zipimporter instance. *archivepath* must be a path to a ZIP - file, or to a specific path within a ZIP file. For example, an *archivepath* - of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory - inside the ZIP file :file:`foo/bar.zip` (provided that it exists). - - :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP - archive. - - .. method:: find_module(fullname[, path]) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - - .. method:: get_code(fullname) - - Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: get_data(pathname) - - Return the data associated with *pathname*. Raise :exc:`OSError` if the - file wasn't found. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - - - .. method:: get_filename(fullname) - - Return the value ``__file__`` would be set to if the specified module - was imported. Raise :exc:`ZipImportError` if the module couldn't be - found. - - .. versionadded:: 3.1 - - - .. method:: get_source(fullname) - - Return the source code for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found, return - :const:`None` if the archive does contain the module, but has no source - for it. - - - .. method:: is_package(fullname) - - Return ``True`` if the module specified by *fullname* is a package. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the imported module, or raises - :exc:`ZipImportError` if it wasn't found. - - - .. attribute:: archive - - The file name of the importer's associated ZIP file, without a possible - subpath. - - - .. attribute:: prefix - - The subpath within the ZIP file where modules are searched. This is the - empty string for zipimporter objects which point to the root of the ZIP - file. - - The :attr:`archive` and :attr:`prefix` attributes, when combined with a - slash, equal the original *archivepath* argument given to the - :class:`zipimporter` constructor. - - -.. _zipimport-examples: - -Examples --------- - -Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. - -.. code-block:: shell-session - - $ unzip -l example.zip - Archive: example.zip - Length Date Time Name - -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py - -------- ------- - 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) - >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' diff --git a/third_party/python/Doc/library/zlib.rst b/third_party/python/Doc/library/zlib.rst deleted file mode 100644 index 8a531c92b..000000000 --- a/third_party/python/Doc/library/zlib.rst +++ /dev/null @@ -1,330 +0,0 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== - -.. module:: zlib - :synopsis: Low-level interface to compression and decompression routines - compatible with gzip. - --------------- - -For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at http://www.zlib.net. There are known -incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a security vulnerability, so we recommend using -1.1.4 or later. - -zlib's functions have many options and often need to be used in a particular -order. This documentation doesn't attempt to cover all of the permutations; -consult the zlib manual at http://www.zlib.net/manual.html for authoritative -information. - -For reading and writing ``.gz`` files see the :mod:`gzip` module. - -The available exception and functions in this module are: - - -.. exception:: error - - Exception raised on compression and decompression errors. - - -.. function:: adler32(data[, value]) - - Computes an Adler-32 checksum of *data*. (An Adler-32 checksum is almost as - reliable as a CRC32 but can be computed much more quickly.) The result - is an unsigned 32-bit integer. If *value* is present, it is used as - the starting value of the checksum; otherwise, a default value of 1 - is used. Passing in *value* allows computing a running checksum over the - concatenation of several inputs. The algorithm is not cryptographically - strong, and should not be used for authentication or digital signatures. Since - the algorithm is designed for use as a checksum algorithm, it is not suitable - for use as a general hash algorithm. - - .. versionchanged:: 3.0 - Always returns an unsigned value. - To generate the same numeric value across all Python versions and - platforms, use ``adler32(data) & 0xffffffff``. - - -.. function:: compress(data, level=-1) - - Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) - is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. - The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). - Raises the :exc:`error` exception if any error occurs. - - .. versionchanged:: 3.6 - *level* can now be used as a keyword parameter. - - -.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) - - Returns a compression object, to be used for compressing data streams that won't - fit into memory at once. - - *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, - while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. - ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). - Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression - (currently equivalent to level 6). - - *method* is the compression algorithm. Currently, the only supported value is - :const:`DEFLATED`. - - The *wbits* argument controls the size of the history buffer (or the - "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values, - defaulting to ``15`` (MAX_WBITS): - - * +9 to +15: The base-two logarithm of the window size, which - therefore ranges between 512 and 32768. Larger values produce - better compression at the expense of greater memory usage. The - resulting output will include a zlib-specific header and trailer. - - * −9 to −15: Uses the absolute value of *wbits* as the - window size logarithm, while producing a raw output stream with no - header or trailing checksum. - - * +25 to +31 = 16 + (9 to 15): Uses the low 4 bits of the value as the - window size logarithm, while including a basic :program:`gzip` header - and trailing checksum in the output. - - The *memLevel* argument controls the amount of memory used for the - internal compression state. Valid values range from ``1`` to ``9``. - Higher values use more memory, but are faster and produce smaller output. - - *strategy* is used to tune the compression algorithm. Possible values are - :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, - :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). - - *zdict* is a predefined compression dictionary. This is a sequence of bytes - (such as a :class:`bytes` object) containing subsequences that are expected - to occur frequently in the data that is to be compressed. Those subsequences - that are expected to be most common should come at the end of the dictionary. - - .. versionchanged:: 3.3 - Added the *zdict* parameter and keyword argument support. - - -.. function:: crc32(data[, value]) - - .. index:: - single: Cyclic Redundancy Check - single: checksum; Cyclic Redundancy Check - - Computes a CRC (Cyclic Redundancy Check) checksum of *data*. The - result is an unsigned 32-bit integer. If *value* is present, it is used - as the starting value of the checksum; otherwise, a default value of 0 - is used. Passing in *value* allows computing a running checksum over the - concatenation of several inputs. The algorithm is not cryptographically - strong, and should not be used for authentication or digital signatures. Since - the algorithm is designed for use as a checksum algorithm, it is not suitable - for use as a general hash algorithm. - - .. versionchanged:: 3.0 - Always returns an unsigned value. - To generate the same numeric value across all Python versions and - platforms, use ``crc32(data) & 0xffffffff``. - - -.. function:: decompress(data, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) - - Decompresses the bytes in *data*, returning a bytes object containing the - uncompressed data. The *wbits* parameter depends on - the format of *data*, and is discussed further below. - If *bufsize* is given, it is used as the initial size of the output - buffer. Raises the :exc:`error` exception if any error occurs. - - .. _decompress-wbits: - - The *wbits* parameter controls the size of the history buffer - (or "window size"), and what header and trailer format is expected. - It is similar to the parameter for :func:`compressobj`, but accepts - more ranges of values: - - * +8 to +15: The base-two logarithm of the window size. The input - must include a zlib header and trailer. - - * 0: Automatically determine the window size from the zlib header. - Only supported since zlib 1.2.3.5. - - * −8 to −15: Uses the absolute value of *wbits* as the window size - logarithm. The input must be a raw stream with no header or trailer. - - * +24 to +31 = 16 + (8 to 15): Uses the low 4 bits of the value as - the window size logarithm. The input must include a gzip header and - trailer. - - * +40 to +47 = 32 + (8 to 15): Uses the low 4 bits of the value as - the window size logarithm, and automatically accepts either - the zlib or gzip format. - - When decompressing a stream, the window size must not be smaller - than the size originally used to compress the stream; using a too-small - value may result in an :exc:`error` exception. The default *wbits* value - corresponds to the largest window size and requires a zlib header and - trailer to be included. - - *bufsize* is the initial size of the buffer used to hold decompressed data. If - more space is required, the buffer size will be increased as needed, so you - don't have to get this value exactly right; tuning it will only save a few calls - to :c:func:`malloc`. - - .. versionchanged:: 3.6 - *wbits* and *bufsize* can be used as keyword arguments. - -.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) - - Returns a decompression object, to be used for decompressing data streams that - won't fit into memory at once. - - The *wbits* parameter controls the size of the history buffer (or the - "window size"), and what header and trailer format is expected. It has - the same meaning as `described for decompress() <#decompress-wbits>`__. - - The *zdict* parameter specifies a predefined compression dictionary. If - provided, this must be the same dictionary as was used by the compressor that - produced the data that is to be decompressed. - - .. note:: - - If *zdict* is a mutable object (such as a :class:`bytearray`), you must not - modify its contents between the call to :func:`decompressobj` and the first - call to the decompressor's ``decompress()`` method. - - .. versionchanged:: 3.3 - Added the *zdict* parameter. - - -Compression objects support the following methods: - - -.. method:: Compress.compress(data) - - Compress *data*, returning a bytes object containing compressed data for at least - part of the data in *data*. This data should be concatenated to the output - produced by any preceding calls to the :meth:`compress` method. Some input may - be kept in internal buffers for later processing. - - -.. method:: Compress.flush([mode]) - - All pending input is processed, and a bytes object containing the remaining compressed - output is returned. *mode* can be selected from the constants - :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, - :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants - allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the - compressed stream and prevents compressing any more data. After calling :meth:`flush` - with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; - the only realistic action is to delete the object. - - -.. method:: Compress.copy() - - Returns a copy of the compression object. This can be used to efficiently - compress a set of data that share a common initial prefix. - - -Decompression objects support the following methods and attributes: - - -.. attribute:: Decompress.unused_data - - A bytes object which contains any bytes past the end of the compressed data. That is, - this remains ``b""`` until the last byte that contains compression data is - available. If the whole bytestring turned out to contain compressed data, this is - ``b""``, an empty bytes object. - - -.. attribute:: Decompress.unconsumed_tail - - A bytes object that contains any data that was not consumed by the last - :meth:`decompress` call because it exceeded the limit for the uncompressed data - buffer. This data has not yet been seen by the zlib machinery, so you must feed - it (possibly with further data concatenated to it) back to a subsequent - :meth:`decompress` method call in order to get correct output. - - -.. attribute:: Decompress.eof - - A boolean indicating whether the end of the compressed data stream has been - reached. - - This makes it possible to distinguish between a properly-formed compressed - stream, and an incomplete or truncated one. - - .. versionadded:: 3.3 - - -.. method:: Decompress.decompress(data, max_length=0) - - Decompress *data*, returning a bytes object containing the uncompressed data - corresponding to at least part of the data in *string*. This data should be - concatenated to the output produced by any preceding calls to the - :meth:`decompress` method. Some of the input data may be preserved in internal - buffers for later processing. - - If the optional parameter *max_length* is non-zero then the return value will be - no longer than *max_length*. This may mean that not all of the compressed input - can be processed; and unconsumed data will be stored in the attribute - :attr:`unconsumed_tail`. This bytestring must be passed to a subsequent call to - :meth:`decompress` if decompression is to continue. If *max_length* is zero - then the whole input is decompressed, and :attr:`unconsumed_tail` is empty. - - .. versionchanged:: 3.6 - *max_length* can be used as a keyword argument. - - -.. method:: Decompress.flush([length]) - - All pending input is processed, and a bytes object containing the remaining - uncompressed output is returned. After calling :meth:`flush`, the - :meth:`decompress` method cannot be called again; the only realistic action is - to delete the object. - - The optional parameter *length* sets the initial size of the output buffer. - - -.. method:: Decompress.copy() - - Returns a copy of the decompression object. This can be used to save the state - of the decompressor midway through the data stream in order to speed up random - seeks into the stream at a future point. - - -Information about the version of the zlib library in use is available through -the following constants: - - -.. data:: ZLIB_VERSION - - The version string of the zlib library that was used for building the module. - This may be different from the zlib library actually used at runtime, which - is available as :const:`ZLIB_RUNTIME_VERSION`. - - -.. data:: ZLIB_RUNTIME_VERSION - - The version string of the zlib library actually loaded by the interpreter. - - .. versionadded:: 3.3 - - -.. seealso:: - - Module :mod:`gzip` - Reading and writing :program:`gzip`\ -format files. - - http://www.zlib.net - The zlib library home page. - - http://www.zlib.net/manual.html - The zlib manual explains the semantics and usage of the library's many - functions. - diff --git a/third_party/python/Doc/license.rst b/third_party/python/Doc/license.rst deleted file mode 100644 index 2b5ac8ff5..000000000 --- a/third_party/python/Doc/license.rst +++ /dev/null @@ -1,957 +0,0 @@ -.. highlightlang:: none - -.. _history-and-license: - -******************* -History and License -******************* - - -History of the software -======================= - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see https://www.cwi.nl/) in the Netherlands as a -successor of a language called ABC. Guido remains Python's principal author, -although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for National -Research Initiatives (CNRI, see https://www.cnri.reston.va.us/) in Reston, -Virginia where he released several versions of the software. - -In May 2000, Guido and the Python core development team moved to BeOpen.com to -form the BeOpen PythonLabs team. In October of the same year, the PythonLabs -team moved to Digital Creations (now Zope Corporation; see -https://www.zope.org/). In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization created -specifically to own Python-related Intellectual Property. Zope Corporation is a -sponsoring member of the PSF. - -All Python releases are Open Source (see https://opensource.org/ for the Open -Source Definition). Historically, most, but not all, Python releases have also -been GPL-compatible; the table below summarizes the various releases. - -+----------------+--------------+------------+------------+-----------------+ -| Release | Derived from | Year | Owner | GPL compatible? | -+================+==============+============+============+=================+ -| 0.9.0 thru 1.2 | n/a | 1991-1995 | CWI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.3 thru 1.5.2 | 1.2 | 1995-1999 | CNRI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.6 | 1.5.2 | 2000 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0 | 1.6 | 2000 | BeOpen.com | no | -+----------------+--------------+------------+------------+-----------------+ -| 1.6.1 | 1.6 | 2001 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.1 | 2.0+1.6.1 | 2001 | PSF | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0.1 | 2.0+1.6.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.2 | 2.1.1 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.3 | 2.1.2 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ - -.. note:: - - GPL-compatible doesn't mean that we're distributing Python under the GPL. All - Python licenses, unlike the GPL, let you distribute a modified version without - making your changes open source. The GPL-compatible licenses make it possible to - combine Python with other software that is released under the GPL; the others - don't. - -Thanks to the many outside volunteers who have worked under Guido's direction to -make these releases possible. - - -Terms and conditions for accessing or otherwise using Python -============================================================ - - -PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------- - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and - the Individual or Organization ("Licensee") accessing and otherwise using Python - |release| software in source or binary form and its associated documentation. - - 2. Subject to the terms and conditions of this License Agreement, PSF hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python |release| alone or in any derivative - version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2021 Python Software Foundation; All Rights - Reserved" are retained in Python |release| alone or in any derivative version - prepared by Licensee. - - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python |release| or any part thereof, and wants to make the - derivative work available to others as provided herein, then Licensee hereby - agrees to include in any such work a brief summary of the changes made to Python - |release|. - - 4. PSF is making Python |release| available to Licensee on an "AS IS" basis. - PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF PYTHON |release| WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON |release| - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON |release|, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 7. Nothing in this License Agreement shall be deemed to create any relationship - of agency, partnership, or joint venture between PSF and Licensee. This License - Agreement does not grant permission to use PSF trademarks or trade name in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. - - 8. By copying, installing or otherwise using Python |release|, Licensee agrees - to be bound by the terms and conditions of this License Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at - 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization - ("Licensee") accessing and otherwise using this software in source or binary - form and its associated documentation ("the Software"). - - 2. Subject to the terms and conditions of this BeOpen Python License Agreement, - BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license - to reproduce, analyze, test, perform and/or display publicly, prepare derivative - works, distribute, and otherwise use the Software alone or in any derivative - version, provided, however, that the BeOpen Python License is retained in the - Software, alone or in any derivative version prepared by Licensee. - - 3. BeOpen is making the Software available to Licensee on an "AS IS" basis. - BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, - MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF - ADVISED OF THE POSSIBILITY THEREOF. - - 5. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 6. This License Agreement shall be governed by and interpreted in all respects - by the law of the State of California, excluding conflict of law provisions. - Nothing in this License Agreement shall be deemed to create any relationship of - agency, partnership, or joint venture between BeOpen and Licensee. This License - Agreement does not grant permission to use BeOpen trademarks or trade names in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. As an exception, the "BeOpen Python" logos available at - http://www.pythonlabs.com/logos.html may be used according to the permissions - granted on that web page. - - 7. By copying, installing or otherwise using the software, Licensee agrees to be - bound by the terms and conditions of this License Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between the Corporation for National Research - Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 - ("CNRI"), and the Individual or Organization ("Licensee") accessing and - otherwise using Python 1.6.1 software in source or binary form and its - associated documentation. - - 2. Subject to the terms and conditions of this License Agreement, CNRI hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python 1.6.1 alone or in any derivative version, - provided, however, that CNRI's License Agreement and CNRI's notice of copyright, - i.e., "Copyright © 1995-2001 Corporation for National Research Initiatives; All - Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version - prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, - Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 - is made available subject to the terms and conditions in CNRI's License - Agreement. This Agreement together with Python 1.6.1 may be located on the - Internet using the following unique, persistent identifier (known as a handle): - 1895.22/1013. This Agreement may also be obtained from a proxy server on the - Internet using the following URL: http://hdl.handle.net/1895.22/1013." - - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python 1.6.1 or any part thereof, and wants to make the derivative - work available to others as provided herein, then Licensee hereby agrees to - include in any such work a brief summary of the changes made to Python 1.6.1. - - 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI - MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, - BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY - OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF - PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 7. This License Agreement shall be governed by the federal intellectual property - law of the United States, including without limitation the federal copyright - law, and, to the extent such U.S. federal law does not apply, by the law of the - Commonwealth of Virginia, excluding Virginia's conflict of law provisions. - Notwithstanding the foregoing, with regard to derivative works based on Python - 1.6.1 that incorporate non-separable material that was previously distributed - under the GNU General Public License (GPL), the law of the Commonwealth of - Virginia shall govern this License Agreement only as to issues arising under or - with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in - this License Agreement shall be deemed to create any relationship of agency, - partnership, or joint venture between CNRI and Licensee. This License Agreement - does not grant permission to use CNRI trademarks or trade name in a trademark - sense to endorse or promote products or services of Licensee, or any third - party. - - 8. By clicking on the "ACCEPT" button where indicated, or by copying, installing - or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and - conditions of this License Agreement. - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -.. parsed-literal:: - - Copyright © 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The - Netherlands. All rights reserved. - - Permission to use, copy, modify, and distribute this software and its - documentation for any purpose and without fee is hereby granted, provided that - the above copyright notice appear in all copies and that both that copyright - notice and this permission notice appear in supporting documentation, and that - the name of Stichting Mathematisch Centrum or CWI not be used in advertising or - publicity pertaining to distribution of the software without specific, written - prior permission. - - STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS - SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO - EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT - OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, - DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS - SOFTWARE. - - -Licenses and Acknowledgements for Incorporated Software -======================================================= - -This section is an incomplete, but growing list of licenses and acknowledgements -for third-party software incorporated in the Python distribution. - - -Mersenne Twister ----------------- - -The :mod:`_random` module includes code based on a download from -http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html. The following are -the verbatim comments from the original code:: - - A C-program for MT19937, with initialization improved 2002/1/26. - Coded by Takuji Nishimura and Makoto Matsumoto. - - Before using, initialize the state by using init_genrand(seed) - or init_by_array(init_key, key_length). - - Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. The names of its contributors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - Any feedback is very welcome. - http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html - email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) - - -Sockets -------- - -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE -Project, http://www.wide.ad.jp/. :: - - Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the project nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - -Floating point exception control --------------------------------- - -The source for the :mod:`fpectl` module includes the following notice:: - - --------------------------------------------------------------------- - / Copyright (c) 1996. \ - | The Regents of the University of California. | - | All rights reserved. | - | | - | Permission to use, copy, modify, and distribute this software for | - | any purpose without fee is hereby granted, provided that this en- | - | tire notice is included in all copies of any software which is or | - | includes a copy or modification of this software and in all | - | copies of the supporting documentation for such software. | - | | - | This work was produced at the University of California, Lawrence | - | Livermore National Laboratory under contract no. W-7405-ENG-48 | - | between the U.S. Department of Energy and The Regents of the | - | University of California for the operation of UC LLNL. | - | | - | DISCLAIMER | - | | - | This software was prepared as an account of work sponsored by an | - | agency of the United States Government. Neither the United States | - | Government nor the University of California nor any of their em- | - | ployees, makes any warranty, express or implied, or assumes any | - | liability or responsibility for the accuracy, completeness, or | - | usefulness of any information, apparatus, product, or process | - | disclosed, or represents that its use would not infringe | - | privately-owned rights. Reference herein to any specific commer- | - | cial products, process, or service by trade name, trademark, | - | manufacturer, or otherwise, does not necessarily constitute or | - | imply its endorsement, recommendation, or favoring by the United | - | States Government or the University of California. The views and | - | opinions of authors expressed herein do not necessarily state or | - | reflect those of the United States Government or the University | - | of California, and shall not be used for advertising or product | - \ endorsement purposes. / - --------------------------------------------------------------------- - - -Asynchronous socket services ----------------------------- - -The :mod:`asynchat` and :mod:`asyncore` modules contain the following notice:: - - Copyright 1996 by Sam Rushing - - All Rights Reserved - - Permission to use, copy, modify, and distribute this software and - its documentation for any purpose and without fee is hereby - granted, provided that the above copyright notice appear in all - copies and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of Sam - Rushing not be used in advertising or publicity pertaining to - distribution of the software without specific, written prior - permission. - - SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, - INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN - NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR - CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -Cookie management ------------------ - -The :mod:`http.cookies` module contains the following notice:: - - Copyright 2000 by Timothy O'Malley - - All Rights Reserved - - Permission to use, copy, modify, and distribute this software - and its documentation for any purpose and without fee is hereby - granted, provided that the above copyright notice appear in all - copies and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of - Timothy O'Malley not be used in advertising or publicity - pertaining to distribution of the software without specific, written - prior permission. - - Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS - SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR - ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. - - -Execution tracing ------------------ - -The :mod:`trace` module contains the following notice:: - - portions copyright 2001, Autonomous Zones Industries, Inc., all rights... - err... reserved and offered to the public under the terms of the - Python 2.2 license. - Author: Zooko O'Whielacronx - http://zooko.com/ - mailto:zooko@zooko.com - - Copyright 2000, Mojam Media, Inc., all rights reserved. - Author: Skip Montanaro - - Copyright 1999, Bioreason, Inc., all rights reserved. - Author: Andrew Dalke - - Copyright 1995-1997, Automatrix, Inc., all rights reserved. - Author: Skip Montanaro - - Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved. - - - Permission to use, copy, modify, and distribute this Python software and - its associated documentation for any purpose without fee is hereby - granted, provided that the above copyright notice appears in all copies, - and that both that copyright notice and this permission notice appear in - supporting documentation, and that the name of neither Automatrix, - Bioreason or Mojam Media be used in advertising or publicity pertaining to - distribution of the software without specific, written prior permission. - - -UUencode and UUdecode functions -------------------------------- - -The :mod:`uu` module contains the following notice:: - - Copyright 1994 by Lance Ellinghouse - Cathedral City, California Republic, United States of America. - All Rights Reserved - Permission to use, copy, modify, and distribute this software and its - documentation for any purpose and without fee is hereby granted, - provided that the above copyright notice appear in all copies and that - both that copyright notice and this permission notice appear in - supporting documentation, and that the name of Lance Ellinghouse - not be used in advertising or publicity pertaining to distribution - of the software without specific, written prior permission. - LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO - THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND - FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE - FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - Modified by Jack Jansen, CWI, July 1995: - - Use binascii module to do the actual line-by-line conversion - between ascii and binary. This results in a 1000-fold speedup. The C - version is still 5 times faster, though. - - Arguments more compliant with Python standard - - -XML Remote Procedure Calls --------------------------- - -The :mod:`xmlrpc.client` module contains the following notice:: - - The XML-RPC client interface is - - Copyright (c) 1999-2002 by Secret Labs AB - Copyright (c) 1999-2002 by Fredrik Lundh - - By obtaining, using, and/or copying this software and/or its - associated documentation, you agree that you have read, understood, - and will comply with the following terms and conditions: - - Permission to use, copy, modify, and distribute this software and - its associated documentation for any purpose and without fee is - hereby granted, provided that the above copyright notice appears in - all copies, and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of - Secret Labs AB or the author not be used in advertising or publicity - pertaining to distribution of the software without specific, written - prior permission. - - SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD - TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- - ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR - BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY - DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE - OF THIS SOFTWARE. - - -test_epoll ----------- - -The :mod:`test_epoll` module contains the following notice:: - - Copyright (c) 2001-2006 Twisted Matrix Laboratories. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Select kqueue -------------- - -The :mod:`select` module contains the following notice for the kqueue -interface:: - - Copyright (c) 2000 Doug White, 2006 James Knight, 2007 Christian Heimes - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - -SipHash24 ---------- - -The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. The contains the following note:: - - - Copyright (c) 2013 Marek Majkowski - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - - Original location: - https://github.com/majek/csiphash/ - - Solution inspired by code from: - Samuel Neves (supercop/crypto_auth/siphash24/little) - djb (supercop/crypto_auth/siphash24/little2) - Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) - - -strtod and dtoa ---------------- - -The file :file:`Python/dtoa.c`, which supplies C functions dtoa and -strtod for conversion of C doubles to and from strings, is derived -from the file of the same name by David M. Gay, currently available -from http://www.netlib.org/fp/. The original file, as retrieved on -March 16, 2009, contains the following copyright and licensing -notice:: - - /**************************************************************** - * - * The author of this software is David M. Gay. - * - * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose without fee is hereby granted, provided that this entire notice - * is included in all copies of any software which is or includes a copy - * or modification of this software and in all copies of the supporting - * documentation for such software. - * - * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY - * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY - * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. - * - ***************************************************************/ - - -OpenSSL -------- - -The modules :mod:`hashlib`, :mod:`posix`, :mod:`ssl`, :mod:`crypt` use -the OpenSSL library for added performance if made available by the -operating system. Additionally, the Windows and Mac OS X installers for -Python may include a copy of the OpenSSL libraries, so we include a copy -of the OpenSSL license here:: - - - LICENSE ISSUES - ============== - - The OpenSSL toolkit stays under a dual license, i.e. both the conditions of - the OpenSSL License and the original SSLeay license apply to the toolkit. - See below for the actual license texts. Actually both licenses are BSD-style - Open Source licenses. In case of any license issues related to OpenSSL - please contact openssl-core@openssl.org. - - OpenSSL License - --------------- - - /* ==================================================================== - * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - - Original SSLeay License - ----------------------- - - /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) - * All rights reserved. - * - * This package is an SSL implementation written - * by Eric Young (eay@cryptsoft.com). - * The implementation was written so as to conform with Netscapes SSL. - * - * This library is free for commercial and non-commercial use as long as - * the following conditions are aheared to. The following conditions - * apply to all code found in this distribution, be it the RC4, RSA, - * lhash, DES, etc., code; not just the SSL code. The SSL documentation - * included with this distribution is covered by the same copyright terms - * except that the holder is Tim Hudson (tjh@cryptsoft.com). - * - * Copyright remains Eric Young's, and as such any Copyright notices in - * the code are not to be removed. - * If this package is used in a product, Eric Young should be given attribution - * as the author of the parts of the library used. - * This can be in the form of a textual message at program startup or - * in documentation (online or textual) provided with the package. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * "This product includes cryptographic software written by - * Eric Young (eay@cryptsoft.com)" - * The word 'cryptographic' can be left out if the rouines from the library - * being used are not cryptographic related :-). - * 4. If you include any Windows specific code (or a derivative thereof) from - * the apps directory (application code) you must include an acknowledgement: - * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" - * - * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * The licence and distribution terms for any publically available version or - * derivative of this code cannot be changed. i.e. this code cannot simply be - * copied and put under another distribution licence - * [including the GNU Public Licence.] - */ - - -expat ------ - -The :mod:`pyexpat` extension is built using an included copy of the expat -sources unless the build is configured ``--with-system-expat``:: - - Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -libffi ------- - -The :mod:`_ctypes` extension is built using an included copy of the libffi -sources unless the build is configured ``--with-system-libffi``:: - - Copyright (c) 1996-2008 Red Hat, Inc and others. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - - -zlib ----- - -The :mod:`zlib` extension is built using an included copy of the zlib -sources if the zlib version found on the system is too old to be -used for the build:: - - Copyright (C) 1995-2011 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - - -cfuhash -------- - -The implementation of the hash table used by the :mod:`tracemalloc` is based -on the cfuhash project:: - - Copyright (c) 2005 Don Owens - All rights reserved. - - This code is released under the BSD license: - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the author nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - OF THE POSSIBILITY OF SUCH DAMAGE. - - -libmpdec --------- - -The :mod:`_decimal` module is built using an included copy of the libmpdec -library unless the build is configured ``--with-system-libmpdec``:: - - Copyright (c) 2008-2016 Stefan Krah. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. diff --git a/third_party/python/Doc/make.bat b/third_party/python/Doc/make.bat deleted file mode 100644 index 562beb35a..000000000 --- a/third_party/python/Doc/make.bat +++ /dev/null @@ -1,178 +0,0 @@ -@echo off -setlocal - -pushd %~dp0 - -set this=%~n0 - -call ..\PCbuild\find_python.bat %PYTHON% - -if not defined PYTHON set PYTHON=py - -if not defined SPHINXBUILD ( - %PYTHON% -c "import sphinx" > nul 2> nul - if errorlevel 1 ( - echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install sphinx - if errorlevel 1 exit /B - ) - set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" -) - -%PYTHON% -c "import python_docs_theme" > nul 2> nul -if errorlevel 1 ( - echo Installing python-docs-theme with %PYTHON% - %PYTHON% -m pip install python-docs-theme - if errorlevel 1 exit /B -) - -if not defined BLURB ( - %PYTHON% -c "import blurb" > nul 2> nul - if errorlevel 1 ( - echo Installing blurb with %PYTHON% - %PYTHON% -m pip install blurb - if errorlevel 1 exit /B - ) - set BLURB=%PYTHON% -m blurb -) - -if "%1" NEQ "htmlhelp" goto :skiphhcsearch -if exist "%HTMLHELP%" goto :skiphhcsearch - -rem Search for HHC in likely places -set HTMLHELP= -where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch -where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" ( - echo. - echo.The HTML Help Workshop was not found. Set the HTMLHELP variable - echo.to the path to hhc.exe or download and install it from - echo.http://msdn.microsoft.com/en-us/library/ms669985 - exit /B 1 -) -:skiphhcsearch - -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v - -if "%BUILDDIR%" EQU "" set BUILDDIR=build - -rem Targets that don't require sphinx-build -if "%1" EQU "" goto help -if "%1" EQU "help" goto help -if "%1" EQU "check" goto check -if "%1" EQU "serve" goto serve -if "%1" == "clean" ( - rmdir /q /s "%BUILDDIR%" - goto end -) - -%SPHINXBUILD% >nul 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - popd - exit /B 1 -) - -rem Targets that do require sphinx-build and have their own label -if "%1" EQU "htmlview" goto htmlview - -rem Everything else -goto build - -:help -echo.usage: %this% BUILDER [filename ...] -echo. -echo.Call %this% with the desired Sphinx builder as the first argument, e.g. -echo.``%this% html`` or ``%this% doctest``. Interesting targets that are -echo.always available include: -echo. -echo. Provided by Sphinx: -echo. html, htmlhelp, latex, text -echo. suspicious, linkcheck, changes, doctest -echo. Provided by this script: -echo. clean, check, serve, htmlview -echo. -echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.rst in this directory or -echo.the documentation for your version of Sphinx for more exhaustive lists -echo.of available targets and descriptions of each. -echo. -echo.This script assumes that the SPHINXBUILD environment variable contains -echo.a legitimate command for calling sphinx-build, or that sphinx-build is -echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can -echo.be passed by setting the SPHINXOPTS environment variable. -goto end - -:build -if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" - -rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py -if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 -if exist ..\Misc\NEWS ( - echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS - copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul -) else if exist ..\Misc\NEWS.D ( - if defined BLURB ( - echo.Merging Misc/NEWS with %BLURB% - %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" - ) else ( - echo.No Misc/NEWS file and Blurb is not available. - exit /B 1 - ) -) - -if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% -) -if "%1" EQU "htmlhelp" ( - set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS% -) -cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" - -if "%1" EQU "htmlhelp" ( - "%HTMLHELP%" "%BUILDDIR%\htmlhelp\python%DISTVERSION:.=%.hhp" - rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 - if not errorlevel 2 cmd /C exit /b 0 -) - -echo. -if errorlevel 1 ( - echo.Build failed (exit code %ERRORLEVEL%^), check for error messages - echo.above. Any output will be found in %BUILDDIR%\%1 -) else ( - echo.Build succeeded. All output should be in %BUILDDIR%\%1 -) -goto end - -:htmlview -if NOT "%2" EQU "" ( - echo.Can't specify filenames to build with htmlview target, ignoring. -) -cmd /C %this% html - -if EXIST "%BUILDDIR%\html\index.html" ( - echo.Opening "%BUILDDIR%\html\index.html" in the default web browser... - start "" "%BUILDDIR%\html\index.html" -) - -goto end - -:check -cmd /S /C "%PYTHON% tools\rstlint.py -i tools" -goto end - -:serve -cmd /S /C "%PYTHON% ..\Tools\scripts\serve.py "%BUILDDIR%\html"" -goto end - -:end -popd diff --git a/third_party/python/Doc/reference/compound_stmts.rst b/third_party/python/Doc/reference/compound_stmts.rst deleted file mode 100644 index 8d050a69a..000000000 --- a/third_party/python/Doc/reference/compound_stmts.rst +++ /dev/null @@ -1,842 +0,0 @@ -.. _compound: - -******************* -Compound statements -******************* - -.. index:: pair: compound; statement - -Compound statements contain (groups of) other statements; they affect or control -the execution of those other statements in some way. In general, compound -statements span multiple lines, although in simple incarnations a whole compound -statement may be contained in one line. - -The :keyword:`if`, :keyword:`while` and :keyword:`for` statements implement -traditional control flow constructs. :keyword:`try` specifies exception -handlers and/or cleanup code for a group of statements, while the -:keyword:`with` statement allows the execution of initialization and -finalization code around a block of code. Function and class definitions are -also syntactically compound statements. - -.. index:: - single: clause - single: suite - single: ; (semicolon) - -A compound statement consists of one or more 'clauses.' A clause consists of a -header and a 'suite.' The clause headers of a particular compound statement are -all at the same indentation level. Each clause header begins with a uniquely -identifying keyword and ends with a colon. A suite is a group of statements -controlled by a clause. A suite can be one or more semicolon-separated simple -statements on the same line as the header, following the header's colon, or it -can be one or more indented statements on subsequent lines. Only the latter -form of a suite can contain nested compound statements; the following is illegal, -mostly because it wouldn't be clear to which :keyword:`if` clause a following -:keyword:`else` clause would belong:: - - if test1: if test2: print(x) - -Also note that the semicolon binds tighter than the colon in this context, so -that in the following example, either all or none of the :func:`print` calls are -executed:: - - if x < y < z: print(x); print(y); print(z) - -Summarizing: - -.. productionlist:: - compound_stmt: `if_stmt` - : | `while_stmt` - : | `for_stmt` - : | `try_stmt` - : | `with_stmt` - : | `funcdef` - : | `classdef` - : | `async_with_stmt` - : | `async_for_stmt` - : | `async_funcdef` - suite: `stmt_list` NEWLINE | NEWLINE INDENT `statement`+ DEDENT - statement: `stmt_list` NEWLINE | `compound_stmt` - stmt_list: `simple_stmt` (";" `simple_stmt`)* [";"] - -.. index:: - single: NEWLINE token - single: DEDENT token - pair: dangling; else - -Note that statements always end in a ``NEWLINE`` possibly followed by a -``DEDENT``. Also note that optional continuation clauses always begin with a -keyword that cannot start a statement, thus there are no ambiguities (the -'dangling :keyword:`else`' problem is solved in Python by requiring nested -:keyword:`if` statements to be indented). - -The formatting of the grammar rules in the following sections places each clause -on a separate line for clarity. - - -.. _if: -.. _elif: -.. _else: - -The :keyword:`if` statement -=========================== - -.. index:: - statement: if - keyword: elif - keyword: else - single: : (colon); compound statement - -The :keyword:`if` statement is used for conditional execution: - -.. productionlist:: - if_stmt: "if" `expression` ":" `suite` - : ("elif" `expression` ":" `suite`)* - : ["else" ":" `suite`] - -It selects exactly one of the suites by evaluating the expressions one by one -until one is found to be true (see section :ref:`booleans` for the definition of -true and false); then that suite is executed (and no other part of the -:keyword:`if` statement is executed or evaluated). If all expressions are -false, the suite of the :keyword:`else` clause, if present, is executed. - - -.. _while: - -The :keyword:`while` statement -============================== - -.. index:: - statement: while - keyword: else - pair: loop; statement - keyword: else - single: : (colon); compound statement - -The :keyword:`while` statement is used for repeated execution as long as an -expression is true: - -.. productionlist:: - while_stmt: "while" `expression` ":" `suite` - : ["else" ":" `suite`] - -This repeatedly tests the expression and, if it is true, executes the first -suite; if the expression is false (which may be the first time it is tested) the -suite of the :keyword:`else` clause, if present, is executed and the loop -terminates. - -.. index:: - statement: break - statement: continue - -A :keyword:`break` statement executed in the first suite terminates the loop -without executing the :keyword:`else` clause's suite. A :keyword:`continue` -statement executed in the first suite skips the rest of the suite and goes back -to testing the expression. - - -.. _for: - -The :keyword:`for` statement -============================ - -.. index:: - statement: for - keyword: in - keyword: else - pair: target; list - pair: loop; statement - keyword: in - keyword: else - pair: target; list - object: sequence - single: : (colon); compound statement - -The :keyword:`for` statement is used to iterate over the elements of a sequence -(such as a string, tuple or list) or other iterable object: - -.. productionlist:: - for_stmt: "for" `target_list` "in" `expression_list` ":" `suite` - : ["else" ":" `suite`] - -The expression list is evaluated once; it should yield an iterable object. An -iterator is created for the result of the ``expression_list``. The suite is -then executed once for each item provided by the iterator, in the order returned -by the iterator. Each item in turn is assigned to the target list using the -standard rules for assignments (see :ref:`assignment`), and then the suite is -executed. When the items are exhausted (which is immediately when the sequence -is empty or an iterator raises a :exc:`StopIteration` exception), the suite in -the :keyword:`else` clause, if present, is executed, and the loop terminates. - -.. index:: - statement: break - statement: continue - -A :keyword:`break` statement executed in the first suite terminates the loop -without executing the :keyword:`else` clause's suite. A :keyword:`continue` -statement executed in the first suite skips the rest of the suite and continues -with the next item, or with the :keyword:`else` clause if there is no next -item. - -The for-loop makes assignments to the variables(s) in the target list. -This overwrites all previous assignments to those variables including -those made in the suite of the for-loop:: - - for i in range(10): - print(i) - i = 5 # this will not affect the for-loop - # because i will be overwritten with the next - # index in the range - - -.. index:: - builtin: range - -Names in the target list are not deleted when the loop is finished, but if the -sequence is empty, they will not have been assigned to at all by the loop. Hint: -the built-in function :func:`range` returns an iterator of integers suitable to -emulate the effect of Pascal's ``for i := a to b do``; e.g., ``list(range(3))`` -returns the list ``[0, 1, 2]``. - -.. note:: - - .. index:: - single: loop; over mutable sequence - single: mutable sequence; loop over - - There is a subtlety when the sequence is being modified by the loop (this can - only occur for mutable sequences, e.g. lists). An internal counter is used - to keep track of which item is used next, and this is incremented on each - iteration. When this counter has reached the length of the sequence the loop - terminates. This means that if the suite deletes the current (or a previous) - item from the sequence, the next item will be skipped (since it gets the - index of the current item which has already been treated). Likewise, if the - suite inserts an item in the sequence before the current item, the current - item will be treated again the next time through the loop. This can lead to - nasty bugs that can be avoided by making a temporary copy using a slice of - the whole sequence, e.g., :: - - for x in a[:]: - if x < 0: a.remove(x) - - -.. _try: -.. _except: -.. _finally: - -The :keyword:`try` statement -============================ - -.. index:: - statement: try - keyword: except - keyword: finally - keyword: else - keyword: as - single: : (colon); compound statement - -The :keyword:`try` statement specifies exception handlers and/or cleanup code -for a group of statements: - -.. productionlist:: - try_stmt: `try1_stmt` | `try2_stmt` - try1_stmt: "try" ":" `suite` - : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ - : ["else" ":" `suite`] - : ["finally" ":" `suite`] - try2_stmt: "try" ":" `suite` - : "finally" ":" `suite` - - -The :keyword:`except` clause(s) specify one or more exception handlers. When no -exception occurs in the :keyword:`try` clause, no exception handler is executed. -When an exception occurs in the :keyword:`try` suite, a search for an exception -handler is started. This search inspects the except clauses in turn until one -is found that matches the exception. An expression-less except clause, if -present, must be last; it matches any exception. For an except clause with an -expression, that expression is evaluated, and the clause matches the exception -if the resulting object is "compatible" with the exception. An object is -compatible with an exception if it is the class or a base class of the exception -object or a tuple containing an item compatible with the exception. - -If no except clause matches the exception, the search for an exception handler -continues in the surrounding code and on the invocation stack. [#]_ - -If the evaluation of an expression in the header of an except clause raises an -exception, the original search for a handler is canceled and a search starts for -the new exception in the surrounding code and on the call stack (it is treated -as if the entire :keyword:`try` statement raised the exception). - -.. index:: single: as; except clause - -When a matching except clause is found, the exception is assigned to the target -specified after the :keyword:`as` keyword in that except clause, if present, and -the except clause's suite is executed. All except clauses must have an -executable block. When the end of this block is reached, execution continues -normally after the entire try statement. (This means that if two nested -handlers exist for the same exception, and the exception occurs in the try -clause of the inner handler, the outer handler will not handle the exception.) - -When an exception has been assigned using ``as target``, it is cleared at the -end of the except clause. This is as if :: - - except E as N: - foo - -was translated to :: - - except E as N: - try: - foo - finally: - del N - -This means the exception must be assigned to a different name to be able to -refer to it after the except clause. Exceptions are cleared because with the -traceback attached to them, they form a reference cycle with the stack frame, -keeping all locals in that frame alive until the next garbage collection occurs. - -.. index:: - module: sys - object: traceback - -Before an except clause's suite is executed, details about the exception are -stored in the :mod:`sys` module and can be accessed via :func:`sys.exc_info`. -:func:`sys.exc_info` returns a 3-tuple consisting of the exception class, the -exception instance and a traceback object (see section :ref:`types`) identifying -the point in the program where the exception occurred. :func:`sys.exc_info` -values are restored to their previous values (before the call) when returning -from a function that handled an exception. - -.. index:: - keyword: else - statement: return - statement: break - statement: continue - -The optional :keyword:`else` clause is executed if the control flow leaves the -:keyword:`try` suite, no exception was raised, and no :keyword:`return`, -:keyword:`continue`, or :keyword:`break` statement was executed. Exceptions in -the :keyword:`else` clause are not handled by the preceding :keyword:`except` -clauses. - -.. index:: keyword: finally - -If :keyword:`finally` is present, it specifies a 'cleanup' handler. The -:keyword:`try` clause is executed, including any :keyword:`except` and -:keyword:`else` clauses. If an exception occurs in any of the clauses and is -not handled, the exception is temporarily saved. The :keyword:`finally` clause -is executed. If there is a saved exception it is re-raised at the end of the -:keyword:`finally` clause. If the :keyword:`finally` clause raises another -exception, the saved exception is set as the context of the new exception. -If the :keyword:`finally` clause executes a :keyword:`return` or :keyword:`break` -statement, the saved exception is discarded:: - - >>> def f(): - ... try: - ... 1/0 - ... finally: - ... return 42 - ... - >>> f() - 42 - -The exception information is not available to the program during execution of -the :keyword:`finally` clause. - -.. index:: - statement: return - statement: break - statement: continue - -When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is -executed in the :keyword:`try` suite of a :keyword:`try`...\ :keyword:`finally` -statement, the :keyword:`finally` clause is also executed 'on the way out.' A -:keyword:`continue` statement is illegal in the :keyword:`finally` clause. (The -reason is a problem with the current implementation --- this restriction may be -lifted in the future). - -The return value of a function is determined by the last :keyword:`return` -statement executed. Since the :keyword:`finally` clause always executes, a -:keyword:`return` statement executed in the :keyword:`finally` clause will -always be the last one executed:: - - >>> def foo(): - ... try: - ... return 'try' - ... finally: - ... return 'finally' - ... - >>> foo() - 'finally' - -Additional information on exceptions can be found in section :ref:`exceptions`, -and information on using the :keyword:`raise` statement to generate exceptions -may be found in section :ref:`raise`. - - -.. _with: -.. _as: - -The :keyword:`with` statement -============================= - -.. index:: - statement: with - keyword: as - single: as; with statement - single: , (comma); with statement - single: : (colon); compound statement - -The :keyword:`with` statement is used to wrap the execution of a block with -methods defined by a context manager (see section :ref:`context-managers`). -This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` -usage patterns to be encapsulated for convenient reuse. - -.. productionlist:: - with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` - with_item: `expression` ["as" `target`] - -The execution of the :keyword:`with` statement with one "item" proceeds as follows: - -#. The context expression (the expression given in the :token:`with_item`) is - evaluated to obtain a context manager. - -#. The context manager's :meth:`__exit__` is loaded for later use. - -#. The context manager's :meth:`__enter__` method is invoked. - -#. If a target was included in the :keyword:`with` statement, the return value - from :meth:`__enter__` is assigned to it. - - .. note:: - - The :keyword:`with` statement guarantees that if the :meth:`__enter__` - method returns without an error, then :meth:`__exit__` will always be - called. Thus, if an error occurs during the assignment to the target list, - it will be treated the same as an error occurring within the suite would - be. See step 6 below. - -#. The suite is executed. - -#. The context manager's :meth:`__exit__` method is invoked. If an exception - caused the suite to be exited, its type, value, and traceback are passed as - arguments to :meth:`__exit__`. Otherwise, three :const:`None` arguments are - supplied. - - If the suite was exited due to an exception, and the return value from the - :meth:`__exit__` method was false, the exception is reraised. If the return - value was true, the exception is suppressed, and execution continues with the - statement following the :keyword:`with` statement. - - If the suite was exited for any reason other than an exception, the return - value from :meth:`__exit__` is ignored, and execution proceeds at the normal - location for the kind of exit that was taken. - -With more than one item, the context managers are processed as if multiple -:keyword:`with` statements were nested:: - - with A() as a, B() as b: - suite - -is equivalent to :: - - with A() as a: - with B() as b: - suite - -.. versionchanged:: 3.1 - Support for multiple context expressions. - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - - -.. index:: - single: parameter; function definition - -.. _function: -.. _def: - -Function definitions -==================== - -.. index:: - statement: def - pair: function; definition - pair: function; name - pair: name; binding - object: user-defined function - object: function - pair: function; name - pair: name; binding - single: () (parentheses); function definition - single: , (comma); parameter list - single: : (colon); compound statement - -A function definition defines a user-defined function object (see section -:ref:`types`): - -.. productionlist:: - funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" - : ["->" `expression`] ":" `suite` - decorators: `decorator`+ - decorator: "@" `dotted_name` ["(" [`argument_list` [","]] ")"] NEWLINE - dotted_name: `identifier` ("." `identifier`)* - parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] - : | "**" `parameter` [","] - parameter: `identifier` [":" `expression`] - defparameter: `parameter` ["=" `expression`] - funcname: `identifier` - - -A function definition is an executable statement. Its execution binds the -function name in the current local namespace to a function object (a wrapper -around the executable code for the function). This function object contains a -reference to the current global namespace as the global namespace to be used -when the function is called. - -The function definition does not execute the function body; this gets executed -only when the function is called. [#]_ - -.. index:: - single: @ (at); function definition - -A function definition may be wrapped by one or more :term:`decorator` expressions. -Decorator expressions are evaluated when the function is defined, in the scope -that contains the function definition. The result must be a callable, which is -invoked with the function object as the only argument. The returned value is -bound to the function name instead of the function object. Multiple decorators -are applied in nested fashion. For example, the following code :: - - @f1(arg) - @f2 - def func(): pass - -is roughly equivalent to :: - - def func(): pass - func = f1(arg)(f2(func)) - -except that the original function is not temporarily bound to the name ``func``. - -.. index:: - triple: default; parameter; value - single: argument; function definition - single: = (equals); function definition - -When one or more :term:`parameters ` have the form *parameter* ``=`` -*expression*, the function is said to have "default parameter values." For a -parameter with a default value, the corresponding :term:`argument` may be -omitted from a call, in which -case the parameter's default value is substituted. If a parameter has a default -value, all following parameters up until the "``*``" must also have a default -value --- this is a syntactic restriction that is not expressed by the grammar. - -**Default parameter values are evaluated from left to right when the function -definition is executed.** This means that the expression is evaluated once, when -the function is defined, and that the same "pre-computed" value is used for each -call. This is especially important to understand when a default parameter is a -mutable object, such as a list or a dictionary: if the function modifies the -object (e.g. by appending an item to a list), the default value is in effect -modified. This is generally not what was intended. A way around this is to use -``None`` as the default, and explicitly test for it in the body of the function, -e.g.:: - - def whats_on_the_telly(penguin=None): - if penguin is None: - penguin = [] - penguin.append("property of the zoo") - return penguin - -.. index:: - single: * (asterisk); function definition - single: **; function definition - -Function call semantics are described in more detail in section :ref:`calls`. A -function call always assigns values to all parameters mentioned in the parameter -list, either from position arguments, from keyword arguments, or from default -values. If the form "``*identifier``" is present, it is initialized to a tuple -receiving any excess positional parameters, defaulting to the empty tuple. -If the form "``**identifier``" is present, it is initialized to a new -ordered mapping receiving any excess keyword arguments, defaulting to a -new empty mapping of the same type. Parameters after "``*``" or -"``*identifier``" are keyword-only parameters and may only be passed -used keyword arguments. - -.. index:: - pair: function; annotations - single: ->; function annotations - single: : (colon); function annotations - -Parameters may have annotations of the form "``: expression``" following the -parameter name. Any parameter may have an annotation even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of -the form "``-> expression``" after the parameter list. These annotations can be -any valid Python expression and are evaluated when the function definition is -executed. Annotations may be evaluated in a different order than they appear in -the source code. The presence of annotations does not change the semantics of a -function. The annotation values are available as values of a dictionary keyed -by the parameters' names in the :attr:`__annotations__` attribute of the -function object. - -.. index:: pair: lambda; expression - -It is also possible to create anonymous functions (functions not bound to a -name), for immediate use in expressions. This uses lambda expressions, described in -section :ref:`lambda`. Note that the lambda expression is merely a shorthand for a -simplified function definition; a function defined in a ":keyword:`def`" -statement can be passed around or assigned to another name just like a function -defined by a lambda expression. The ":keyword:`def`" form is actually more powerful -since it allows the execution of multiple statements and annotations. - -**Programmer's note:** Functions are first-class objects. A "``def``" statement -executed inside a function definition defines a local function that can be -returned or passed around. Free variables used in the nested function can -access the local variables of the function containing the def. See section -:ref:`naming` for details. - -.. seealso:: - - :pep:`3107` - Function Annotations - The original specification for function annotations. - - -.. _class: - -Class definitions -================= - -.. index:: - object: class - statement: class - pair: class; definition - pair: class; name - pair: name; binding - pair: execution; frame - single: inheritance - single: docstring - single: () (parentheses); class definition - single: , (comma); expression list - single: : (colon); compound statement - -A class definition defines a class object (see section :ref:`types`): - -.. productionlist:: - classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` - inheritance: "(" [`argument_list`] ")" - classname: `identifier` - -A class definition is an executable statement. The inheritance list usually -gives a list of base classes (see :ref:`metaclasses` for more advanced uses), so -each item in the list should evaluate to a class object which allows -subclassing. Classes without an inheritance list inherit, by default, from the -base class :class:`object`; hence, :: - - class Foo: - pass - -is equivalent to :: - - class Foo(object): - pass - -The class's suite is then executed in a new execution frame (see :ref:`naming`), -using a newly created local namespace and the original global namespace. -(Usually, the suite contains mostly function definitions.) When the class's -suite finishes execution, its execution frame is discarded but its local -namespace is saved. [#]_ A class object is then created using the inheritance -list for the base classes and the saved local namespace for the attribute -dictionary. The class name is bound to this class object in the original local -namespace. - -The order in which attributes are defined in the class body is preserved -in the new class's ``__dict__``. Note that this is reliable only right -after the class is created and only for classes that were defined using -the definition syntax. - -Class creation can be customized heavily using :ref:`metaclasses `. - -.. index:: - single: @ (at); class definition - -Classes can also be decorated: just like when decorating functions, :: - - @f1(arg) - @f2 - class Foo: pass - -is roughly equivalent to :: - - class Foo: pass - Foo = f1(arg)(f2(Foo)) - -The evaluation rules for the decorator expressions are the same as for function -decorators. The result is then bound to the class name. - -**Programmer's note:** Variables defined in the class definition are class -attributes; they are shared by instances. Instance attributes can be set in a -method with ``self.name = value``. Both class and instance attributes are -accessible through the notation "``self.name``", and an instance attribute hides -a class attribute with the same name when accessed in this way. Class -attributes can be used as defaults for instance attributes, but using mutable -values there can lead to unexpected results. :ref:`Descriptors ` -can be used to create instance variables with different implementation details. - - -.. seealso:: - - :pep:`3115` - Metaclasses in Python 3000 - The proposal that changed the declaration of metaclasses to the current - syntax, and the semantics for how classes with metaclasses are - constructed. - - :pep:`3129` - Class Decorators - The proposal that added class decorators. Function and method decorators - were introduced in :pep:`318`. - - -Coroutines -========== - -.. versionadded:: 3.5 - -.. index:: statement: async def -.. _`async def`: - -Coroutine function definition ------------------------------ - -.. productionlist:: - async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" - : ["->" `expression`] ":" `suite` - -.. index:: - keyword: async - keyword: await - -Execution of Python coroutines can be suspended and resumed at many points -(see :term:`coroutine`). In the body of a coroutine, any ``await`` and -``async`` identifiers become reserved keywords; :keyword:`await` expressions, -:keyword:`async for` and :keyword:`async with` can only be used in -coroutine bodies. - -Functions defined with ``async def`` syntax are always coroutine functions, -even if they do not contain ``await`` or ``async`` keywords. - -It is a :exc:`SyntaxError` to use ``yield from`` expressions in -``async def`` coroutines. - -An example of a coroutine function:: - - async def func(param1, param2): - do_stuff() - await some_coroutine() - - -.. index:: statement: async for -.. _`async for`: - -The :keyword:`async for` statement ----------------------------------- - -.. productionlist:: - async_for_stmt: "async" `for_stmt` - -An :term:`asynchronous iterable` is able to call asynchronous code in its -*iter* implementation, and :term:`asynchronous iterator` can call asynchronous -code in its *next* method. - -The ``async for`` statement allows convenient iteration over asynchronous -iterators. - -The following code:: - - async for TARGET in ITER: - BLOCK - else: - BLOCK2 - -Is semantically equivalent to:: - - iter = (ITER) - iter = type(iter).__aiter__(iter) - running = True - while running: - try: - TARGET = await type(iter).__anext__(iter) - except StopAsyncIteration: - running = False - else: - BLOCK - else: - BLOCK2 - -See also :meth:`__aiter__` and :meth:`__anext__` for details. - -It is a :exc:`SyntaxError` to use ``async for`` statement outside of an -:keyword:`async def` function. - - -.. index:: statement: async with -.. _`async with`: - -The :keyword:`async with` statement ------------------------------------ - -.. productionlist:: - async_with_stmt: "async" `with_stmt` - -An :term:`asynchronous context manager` is a :term:`context manager` that is -able to suspend execution in its *enter* and *exit* methods. - -The following code:: - - async with EXPR as VAR: - BLOCK - -Is semantically equivalent to:: - - mgr = (EXPR) - aexit = type(mgr).__aexit__ - aenter = type(mgr).__aenter__(mgr) - - VAR = await aenter - try: - BLOCK - except: - if not await aexit(mgr, *sys.exc_info()): - raise - else: - await aexit(mgr, None, None, None) - -See also :meth:`__aenter__` and :meth:`__aexit__` for details. - -It is a :exc:`SyntaxError` to use ``async with`` statement outside of an -:keyword:`async def` function. - -.. seealso:: - - :pep:`492` - Coroutines with async and await syntax - The proposal that made coroutines a proper standalone concept in Python, - and added supporting syntax. - - -.. rubric:: Footnotes - -.. [#] The exception is propagated to the invocation stack unless - there is a :keyword:`finally` clause which happens to raise another - exception. That new exception causes the old one to be lost. - -.. [#] A string literal appearing as the first statement in the function body is - transformed into the function's ``__doc__`` attribute and therefore the - function's :term:`docstring`. - -.. [#] A string literal appearing as the first statement in the class body is - transformed into the namespace's ``__doc__`` item and therefore the class's - :term:`docstring`. diff --git a/third_party/python/Doc/reference/datamodel.rst b/third_party/python/Doc/reference/datamodel.rst deleted file mode 100644 index 143376bc5..000000000 --- a/third_party/python/Doc/reference/datamodel.rst +++ /dev/null @@ -1,2657 +0,0 @@ - -.. _datamodel: - -********** -Data model -********** - - -.. _objects: - -Objects, values and types -========================= - -.. index:: - single: object - single: data - -:dfn:`Objects` are Python's abstraction for data. All data in a Python program -is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer," code is also -represented by objects.) - -.. index:: - builtin: id - builtin: type - single: identity of an object - single: value of an object - single: type of an object - single: mutable object - single: immutable object - -.. XXX it *is* now possible in some cases to change an object's - type, under certain controlled conditions - -Every object has an identity, a type and a value. An object's *identity* never -changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the -:func:`id` function returns an integer representing its identity. - -.. impl-detail:: - - For CPython, ``id(x)`` is the memory address where ``x`` is stored. - -An object's type determines the operations that the object supports (e.g., "does -it have a length?") and also defines the possible values for objects of that -type. The :func:`type` function returns an object's type (which is an object -itself). Like its identity, an object's :dfn:`type` is also unchangeable. -[#]_ - -The *value* of some objects can change. Objects whose value can -change are said to be *mutable*; objects whose value is unchangeable once they -are created are called *immutable*. (The value of an immutable container object -that contains a reference to a mutable object can change when the latter's value -is changed; however the container is still considered immutable, because the -collection of objects it contains cannot be changed. So, immutability is not -strictly the same as having an unchangeable value, it is more subtle.) An -object's mutability is determined by its type; for instance, numbers, strings -and tuples are immutable, while dictionaries and lists are mutable. - -.. index:: - single: garbage collection - single: reference counting - single: unreachable object - -Objects are never explicitly destroyed; however, when they become unreachable -they may be garbage-collected. An implementation is allowed to postpone garbage -collection or omit it altogether --- it is a matter of implementation quality -how garbage collection is implemented, as long as no objects are collected that -are still reachable. - -.. impl-detail:: - - CPython currently uses a reference-counting scheme with (optional) delayed - detection of cyclically linked garbage, which collects most objects as soon - as they become unreachable, but is not guaranteed to collect garbage - containing circular references. See the documentation of the :mod:`gc` - module for information on controlling the collection of cyclic garbage. - Other implementations act differently and CPython may change. - Do not depend on immediate finalization of objects when they become - unreachable (so you should always close files explicitly). - -Note that the use of the implementation's tracing or debugging facilities may -keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep -objects alive. - -Some objects contain references to "external" resources such as open files or -windows. It is understood that these resources are freed when the object is -garbage-collected, but since garbage collection is not guaranteed to happen, -such objects also provide an explicit way to release the external resource, -usually a :meth:`close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. - -.. index:: single: container - -Some objects contain references to other objects; these are called *containers*. -Examples of containers are tuples, lists and dictionaries. The references are -part of a container's value. In most cases, when we talk about the value of a -container, we imply the values, not the identities of the contained objects; -however, when we talk about the mutability of a container, only the identities -of the immediately contained objects are implied. So, if an immutable container -(like a tuple) contains a reference to a mutable object, its value changes if -that mutable object is changed. - -Types affect almost all aspects of object behavior. Even the importance of -object identity is affected in some sense: for immutable types, operations that -compute new values may actually return a reference to any existing object with -the same type and value, while for mutable objects this is not allowed. E.g., -after ``a = 1; b = 1``, ``a`` and ``b`` may or may not refer to the same object -with the value one, depending on the implementation, but after ``c = []; d = -[]``, ``c`` and ``d`` are guaranteed to refer to two different, unique, newly -created empty lists. (Note that ``c = d = []`` assigns the same object to both -``c`` and ``d``.) - - -.. _types: - -The standard type hierarchy -=========================== - -.. index:: - single: type - pair: data; type - pair: type; hierarchy - pair: extension; module - pair: C; language - -Below is a list of the types that are built into Python. Extension modules -(written in C, Java, or other languages, depending on the implementation) can -define additional types. Future versions of Python may add types to the type -hierarchy (e.g., rational numbers, efficiently stored arrays of integers, etc.), -although such additions will often be provided via the standard library instead. - -.. index:: - single: attribute - pair: special; attribute - triple: generic; special; attribute - -Some of the type descriptions below contain a paragraph listing 'special -attributes.' These are attributes that provide access to the implementation and -are not intended for general use. Their definition may change in the future. - -None - .. index:: object: None - - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``None``. It is used to signify the - absence of a value in many situations, e.g., it is returned from functions that - don't explicitly return anything. Its truth value is false. - -NotImplemented - .. index:: object: NotImplemented - - This type has a single value. There is a single object with this value. This - object is accessed through the built-in name ``NotImplemented``. Numeric methods - and rich comparison methods should return this value if they do not implement the - operation for the operands provided. (The interpreter will then try the - reflected operation, or some other fallback, depending on the operator.) Its - truth value is true. - - See - :ref:`implementing-the-arithmetic-operations` - for more details. - - -Ellipsis - .. index:: - object: Ellipsis - single: ...; ellipsis literal - - This type has a single value. There is a single object with this value. This - object is accessed through the literal ``...`` or the built-in name - ``Ellipsis``. Its truth value is true. - -:class:`numbers.Number` - .. index:: object: numeric - - These are created by numeric literals and returned as results by arithmetic - operators and arithmetic built-in functions. Numeric objects are immutable; - once created their value never changes. Python numbers are of course strongly - related to mathematical numbers, but subject to the limitations of numerical - representation in computers. - - Python distinguishes between integers, floating point numbers, and complex - numbers: - - :class:`numbers.Integral` - .. index:: object: integer - - These represent elements from the mathematical set of integers (positive and - negative). - - There are two types of integers: - - Integers (:class:`int`) - - These represent numbers in an unlimited range, subject to available (virtual) - memory only. For the purpose of shift and mask operations, a binary - representation is assumed, and negative numbers are represented in a variant of - 2's complement which gives the illusion of an infinite string of sign bits - extending to the left. - - Booleans (:class:`bool`) - .. index:: - object: Boolean - single: False - single: True - - These represent the truth values False and True. The two objects representing - the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a - subtype of the integer type, and Boolean values behave like the values 0 and 1, - respectively, in almost all contexts, the exception being that when converted to - a string, the strings ``"False"`` or ``"True"`` are returned, respectively. - - .. index:: pair: integer; representation - - The rules for integer representation are intended to give the most meaningful - interpretation of shift and mask operations involving negative integers. - - :class:`numbers.Real` (:class:`float`) - .. index:: - object: floating point - pair: floating point; number - pair: C; language - pair: Java; language - - These represent machine-level double precision floating point numbers. You are - at the mercy of the underlying machine architecture (and C or Java - implementation) for the accepted range and handling of overflow. Python does not - support single-precision floating point numbers; the savings in processor and - memory usage that are usually the reason for using these are dwarfed by the - overhead of using objects in Python, so there is no reason to complicate the - language with two kinds of floating point numbers. - - :class:`numbers.Complex` (:class:`complex`) - .. index:: - object: complex - pair: complex; number - - These represent complex numbers as a pair of machine-level double precision - floating point numbers. The same caveats apply as for floating point numbers. - The real and imaginary parts of a complex number ``z`` can be retrieved through - the read-only attributes ``z.real`` and ``z.imag``. - -Sequences - .. index:: - builtin: len - object: sequence - single: index operation - single: item selection - single: subscription - - These represent finite ordered sets indexed by non-negative numbers. The - built-in function :func:`len` returns the number of items of a sequence. When - the length of a sequence is *n*, the index set contains the numbers 0, 1, - ..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. - - .. index:: single: slicing - - Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such - that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a - sequence of the same type. This implies that the index set is renumbered so - that it starts at 0. - - Some sequences also support "extended slicing" with a third "step" parameter: - ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* - ``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. - - Sequences are distinguished according to their mutability: - - Immutable sequences - .. index:: - object: immutable sequence - object: immutable - - An object of an immutable sequence type cannot change once it is created. (If - the object contains references to other objects, these other objects may be - mutable and may be changed; however, the collection of objects directly - referenced by an immutable object cannot change.) - - The following types are immutable sequences: - - .. index:: - single: string; immutable sequences - - Strings - .. index:: - builtin: chr - builtin: ord - single: character - single: integer - single: Unicode - - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:type:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` - converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. - :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given text encoding, and - :meth:`bytes.decode` can be used to achieve the opposite. - - Tuples - .. index:: - object: tuple - pair: singleton; tuple - pair: empty; tuple - - The items of a tuple are arbitrary Python objects. Tuples of two or - more items are formed by comma-separated lists of expressions. A tuple - of one item (a 'singleton') can be formed by affixing a comma to an - expression (an expression by itself does not create a tuple, since - parentheses must be usable for grouping of expressions). An empty - tuple can be formed by an empty pair of parentheses. - - Bytes - .. index:: bytes, byte - - A bytes object is an immutable array. The items are 8-bit bytes, - represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor - can be used to create bytes objects. Also, bytes objects can be - decoded to strings via the :meth:`~bytes.decode` method. - - Mutable sequences - .. index:: - object: mutable sequence - object: mutable - pair: assignment; statement - single: subscription - single: slicing - - Mutable sequences can be changed after they are created. The subscription and - slicing notations can be used as the target of assignment and :keyword:`del` - (delete) statements. - - There are currently two intrinsic mutable sequence types: - - Lists - .. index:: object: list - - The items of a list are arbitrary Python objects. Lists are formed by - placing a comma-separated list of expressions in square brackets. (Note - that there are no special cases needed to form lists of length 0 or 1.) - - Byte Arrays - .. index:: bytearray - - A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable - (and hence unhashable), byte arrays otherwise provide the same interface - and functionality as immutable :class:`bytes` objects. - - .. index:: module: array - - The extension module :mod:`array` provides an additional example of a - mutable sequence type, as does the :mod:`collections` module. - -Set types - .. index:: - builtin: len - object: set type - - These represent unordered, finite sets of unique, immutable objects. As such, - they cannot be indexed by any subscript. However, they can be iterated over, and - the built-in function :func:`len` returns the number of items in a set. Common - uses for sets are fast membership testing, removing duplicates from a sequence, - and computing mathematical operations such as intersection, union, difference, - and symmetric difference. - - For set elements, the same immutability rules apply as for dictionary keys. Note - that numeric types obey the normal rules for numeric comparison: if two numbers - compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a - set. - - There are currently two intrinsic set types: - - Sets - .. index:: object: set - - These represent a mutable set. They are created by the built-in :func:`set` - constructor and can be modified afterwards by several methods, such as - :meth:`~set.add`. - - Frozen sets - .. index:: object: frozenset - - These represent an immutable set. They are created by the built-in - :func:`frozenset` constructor. As a frozenset is immutable and - :term:`hashable`, it can be used again as an element of another set, or as - a dictionary key. - -Mappings - .. index:: - builtin: len - single: subscription - object: mapping - - These represent finite sets of objects indexed by arbitrary index sets. The - subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping - ``a``; this can be used in expressions and as the target of assignments or - :keyword:`del` statements. The built-in function :func:`len` returns the number - of items in a mapping. - - There is currently a single intrinsic mapping type: - - Dictionaries - .. index:: object: dictionary - - These represent finite sets of objects indexed by nearly arbitrary values. The - only types of values not acceptable as keys are values containing lists or - dictionaries or other mutable types that are compared by value rather than by - object identity, the reason being that the efficient implementation of - dictionaries requires a key's hash value to remain constant. Numeric types used - for keys obey the normal rules for numeric comparison: if two numbers compare - equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index - the same dictionary entry. - - Dictionaries are mutable; they can be created by the ``{...}`` notation (see - section :ref:`dict`). - - .. index:: - module: dbm.ndbm - module: dbm.gnu - - The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide - additional examples of mapping types, as does the :mod:`collections` - module. - -Callable types - .. index:: - object: callable - pair: function; call - single: invocation - pair: function; argument - - These are the types to which the function call operation (see section - :ref:`calls`) can be applied: - - User-defined functions - .. index:: - pair: user-defined; function - object: function - object: user-defined function - - A user-defined function object is created by a function definition (see - section :ref:`function`). It should be called with an argument list - containing the same number of items as the function's formal parameter - list. - - Special attributes: - - .. tabularcolumns:: |l|L|l| - - .. index:: - single: __doc__ (function attribute) - single: __name__ (function attribute) - single: __module__ (function attribute) - single: __dict__ (function attribute) - single: __defaults__ (function attribute) - single: __closure__ (function attribute) - single: __code__ (function attribute) - single: __globals__ (function attribute) - single: __annotations__ (function attribute) - single: __kwdefaults__ (function attribute) - pair: global; namespace - - +-------------------------+-------------------------------+-----------+ - | Attribute | Meaning | | - +=========================+===============================+===========+ - | :attr:`__doc__` | The function's documentation | Writable | - | | string, or ``None`` if | | - | | unavailable; not inherited by | | - | | subclasses | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's name | Writable | - | __name__` | | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~definition.\ | The function's | Writable | - | __qualname__` | :term:`qualified name` | | - | | | | - | | .. versionadded:: 3.3 | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__module__` | The name of the module the | Writable | - | | function was defined in, or | | - | | ``None`` if unavailable. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__defaults__` | A tuple containing default | Writable | - | | argument values for those | | - | | arguments that have defaults, | | - | | or ``None`` if no arguments | | - | | have a default value | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__code__` | The code object representing | Writable | - | | the compiled function body. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__globals__` | A reference to the dictionary | Read-only | - | | that holds the function's | | - | | global variables --- the | | - | | global namespace of the | | - | | module in which the function | | - | | was defined. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`~object.__dict__`| The namespace supporting | Writable | - | | arbitrary function | | - | | attributes. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | - | | that contain bindings for the | | - | | function's free variables. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__annotations__` | A dict containing annotations | Writable | - | | of parameters. The keys of | | - | | the dict are the parameter | | - | | names, and ``'return'`` for | | - | | the return annotation, if | | - | | provided. | | - +-------------------------+-------------------------------+-----------+ - | :attr:`__kwdefaults__` | A dict containing defaults | Writable | - | | for keyword-only parameters. | | - +-------------------------+-------------------------------+-----------+ - - Most of the attributes labelled "Writable" check the type of the assigned value. - - Function objects also support getting and setting arbitrary attributes, which - can be used, for example, to attach metadata to functions. Regular attribute - dot-notation is used to get and set such attributes. *Note that the current - implementation only supports function attributes on user-defined functions. - Function attributes on built-in functions may be supported in the future.* - - Additional information about a function's definition can be retrieved from its - code object; see the description of internal types below. - - Instance methods - .. index:: - object: method - object: user-defined method - pair: user-defined; method - - An instance method object combines a class, a class instance and any - callable object (normally a user-defined function). - - .. index:: - single: __func__ (method attribute) - single: __self__ (method attribute) - single: __doc__ (method attribute) - single: __name__ (method attribute) - single: __module__ (method attribute) - - Special read-only attributes: :attr:`__self__` is the class instance object, - :attr:`__func__` is the function object; :attr:`__doc__` is the method's - documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the - method name (same as ``__func__.__name__``); :attr:`__module__` is the - name of the module the method was defined in, or ``None`` if unavailable. - - Methods also support accessing (but not setting) the arbitrary function - attributes on the underlying function object. - - User-defined method objects may be created when getting an attribute of a - class (perhaps via an instance of that class), if that attribute is a - user-defined function object or a class method object. - - When an instance method object is created by retrieving a user-defined - function object from a class via one of its instances, its - :attr:`__self__` attribute is the instance, and the method object is said - to be bound. The new method's :attr:`__func__` attribute is the original - function object. - - When a user-defined method object is created by retrieving another method - object from a class or instance, the behaviour is the same as for a - function object, except that the :attr:`__func__` attribute of the new - instance is not the original method object but its :attr:`__func__` - attribute. - - When an instance method object is created by retrieving a class method - object from a class or instance, its :attr:`__self__` attribute is the - class itself, and its :attr:`__func__` attribute is the function object - underlying the class method. - - When an instance method object is called, the underlying function - (:attr:`__func__`) is called, inserting the class instance - (:attr:`__self__`) in front of the argument list. For instance, when - :class:`C` is a class which contains a definition for a function - :meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is - equivalent to calling ``C.f(x, 1)``. - - When an instance method object is derived from a class method object, the - "class instance" stored in :attr:`__self__` will actually be the class - itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to - calling ``f(C,1)`` where ``f`` is the underlying function. - - Note that the transformation from function object to instance method - object happens each time the attribute is retrieved from the instance. In - some cases, a fruitful optimization is to assign the attribute to a local - variable and call that local variable. Also notice that this - transformation only happens for user-defined functions; other callable - objects (and all non-callable objects) are retrieved without - transformation. It is also important to note that user-defined functions - which are attributes of a class instance are not converted to bound - methods; this *only* happens when the function is an attribute of the - class. - - Generator functions - .. index:: - single: generator; function - single: generator; iterator - - A function or method which uses the :keyword:`yield` statement (see section - :ref:`yield`) is called a :dfn:`generator function`. Such a function, when - called, always returns an iterator object which can be used to execute the - body of the function: calling the iterator's :meth:`iterator.__next__` - method will cause the function to execute until it provides a value - using the :keyword:`yield` statement. When the function executes a - :keyword:`return` statement or falls off the end, a :exc:`StopIteration` - exception is raised and the iterator will have reached the end of the set of - values to be returned. - - Coroutine functions - .. index:: - single: coroutine; function - - A function or method which is defined using :keyword:`async def` is called - a :dfn:`coroutine function`. Such a function, when called, returns a - :term:`coroutine` object. It may contain :keyword:`await` expressions, - as well as :keyword:`async with` and :keyword:`async for` statements. See - also the :ref:`coroutine-objects` section. - - Asynchronous generator functions - .. index:: - single: asynchronous generator; function - single: asynchronous generator; asynchronous iterator - - A function or method which is defined using :keyword:`async def` and - which uses the :keyword:`yield` statement is called a - :dfn:`asynchronous generator function`. Such a function, when called, - returns an asynchronous iterator object which can be used in an - :keyword:`async for` statement to execute the body of the function. - - Calling the asynchronous iterator's :meth:`aiterator.__anext__` method - will return an :term:`awaitable` which when awaited - will execute until it provides a value using the :keyword:`yield` - expression. When the function executes an empty :keyword:`return` - statement or falls off the end, a :exc:`StopAsyncIteration` exception - is raised and the asynchronous iterator will have reached the end of - the set of values to be yielded. - - Built-in functions - .. index:: - object: built-in function - object: function - pair: C; language - - A built-in function object is a wrapper around a C function. Examples of - built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a - standard built-in module). The number and type of the arguments are - determined by the C function. Special read-only attributes: - :attr:`__doc__` is the function's documentation string, or ``None`` if - unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is - set to ``None`` (but see the next item); :attr:`__module__` is the name of - the module the function was defined in or ``None`` if unavailable. - - Built-in methods - .. index:: - object: built-in method - object: method - pair: built-in; method - - This is really a different disguise of a built-in function, this time containing - an object passed to the C function as an implicit extra argument. An example of - a built-in method is ``alist.append()``, assuming *alist* is a list object. In - this case, the special read-only attribute :attr:`__self__` is set to the object - denoted by *alist*. - - Classes - Classes are callable. These objects normally act as factories for new - instances of themselves, but variations are possible for class types that - override :meth:`__new__`. The arguments of the call are passed to - :meth:`__new__` and, in the typical case, to :meth:`__init__` to - initialize the new instance. - - Class Instances - Instances of arbitrary classes can be made callable by defining a - :meth:`__call__` method in their class. - - -Modules - .. index:: - statement: import - object: module - - Modules are a basic organizational unit of Python code, and are created by - the :ref:`import system ` as invoked either by the - :keyword:`import` statement (see :keyword:`import`), or by calling - functions such as :func:`importlib.import_module` and built-in - :func:`__import__`. A module object has a namespace implemented by a - dictionary object (this is the dictionary referenced by the ``__globals__`` - attribute of functions defined in the module). Attribute references are - translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to - ``m.__dict__["x"]``. A module object does not contain the code object used - to initialize the module (since it isn't needed once the initialization is - done). - - Attribute assignment updates the module's namespace dictionary, e.g., - ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. - - .. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __annotations__ (module attribute) - pair: module; namespace - - Predefined (writable) attributes: :attr:`__name__` is the module's name; - :attr:`__doc__` is the module's documentation string, or ``None`` if - unavailable; :attr:`__annotations__` (optional) is a dictionary containing - :term:`variable annotations ` collected during module - body execution; :attr:`__file__` is the pathname of the file from which the - module was loaded, if it was loaded from a file. The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter; for extension modules - loaded dynamically from a shared library, it is the pathname of the shared - library file. - - .. index:: single: __dict__ (module attribute) - - Special read-only attribute: :attr:`~object.__dict__` is the module's - namespace as a dictionary object. - - .. impl-detail:: - - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. - -Custom classes - Custom class types are typically created by class definitions (see section - :ref:`class`). A class has a namespace implemented by a dictionary object. - Class attribute references are translated to lookups in this dictionary, e.g., - ``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of - hooks which allow for other means of locating attributes). When the attribute - name is not found there, the attribute search continues in the base classes. - This search of the base classes uses the C3 method resolution order which - behaves correctly even in the presence of 'diamond' inheritance structures - where there are multiple inheritance paths leading back to a common ancestor. - Additional details on the C3 MRO used by Python can be found in the - documentation accompanying the 2.3 release at - https://www.python.org/download/releases/2.3/mro/. - - .. XXX: Could we add that MRO doc as an appendix to the language ref? - - .. index:: - object: class - object: class instance - object: instance - pair: class object; call - single: container - object: dictionary - pair: class; attribute - - When a class attribute reference (for class :class:`C`, say) would yield a - class method object, it is transformed into an instance method object whose - :attr:`__self__` attribute is :class:`C`. When it would yield a static - method object, it is transformed into the object wrapped by the static method - object. See section :ref:`descriptors` for another way in which attributes - retrieved from a class may differ from those actually contained in its - :attr:`~object.__dict__`. - - .. index:: triple: class; attribute; assignment - - Class attribute assignments update the class's dictionary, never the dictionary - of a base class. - - .. index:: pair: class object; call - - A class object can be called (see above) to yield a class instance (see below). - - .. index:: - single: __name__ (class attribute) - single: __module__ (class attribute) - single: __dict__ (class attribute) - single: __bases__ (class attribute) - single: __doc__ (class attribute) - single: __annotations__ (class attribute) - - Special attributes: :attr:`~definition.__name__` is the class name; :attr:`__module__` is - the module name in which the class was defined; :attr:`~object.__dict__` is the - dictionary containing the class's namespace; :attr:`~class.__bases__` is a - tuple containing the base classes, in the order of their occurrence in the - base class list; :attr:`__doc__` is the class's documentation string, - or ``None`` if undefined; :attr:`__annotations__` (optional) is a dictionary - containing :term:`variable annotations ` collected during - class body execution. - -Class instances - .. index:: - object: class instance - object: instance - pair: class; instance - pair: class instance; attribute - - A class instance is created by calling a class object (see above). A class - instance has a namespace implemented as a dictionary which is the first place - in which attribute references are searched. When an attribute is not found - there, and the instance's class has an attribute by that name, the search - continues with the class attributes. If a class attribute is found that is a - user-defined function object, it is transformed into an instance method - object whose :attr:`__self__` attribute is the instance. Static method and - class method objects are also transformed; see above under "Classes". See - section :ref:`descriptors` for another way in which attributes of a class - retrieved via its instances may differ from the objects actually stored in - the class's :attr:`~object.__dict__`. If no class attribute is found, and the - object's class has a :meth:`__getattr__` method, that is called to satisfy - the lookup. - - .. index:: triple: class instance; attribute; assignment - - Attribute assignments and deletions update the instance's dictionary, never a - class's dictionary. If the class has a :meth:`__setattr__` or - :meth:`__delattr__` method, this is called instead of updating the instance - dictionary directly. - - .. index:: - object: numeric - object: sequence - object: mapping - - Class instances can pretend to be numbers, sequences, or mappings if they have - methods with certain special names. See section :ref:`specialnames`. - - .. index:: - single: __dict__ (instance attribute) - single: __class__ (instance attribute) - - Special attributes: :attr:`~object.__dict__` is the attribute dictionary; - :attr:`~instance.__class__` is the instance's class. - -I/O objects (also known as file objects) - .. index:: - builtin: open - module: io - single: popen() (in module os) - single: makefile() (socket method) - single: sys.stdin - single: sys.stdout - single: sys.stderr - single: stdio - single: stdin (in module sys) - single: stdout (in module sys) - single: stderr (in module sys) - - A :term:`file object` represents an open file. Various shortcuts are - available to create file objects: the :func:`open` built-in function, and - also :func:`os.popen`, :func:`os.fdopen`, and the - :meth:`~socket.socket.makefile` method of socket objects (and perhaps by - other functions or methods provided by extension modules). - - The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are - initialized to file objects corresponding to the interpreter's standard - input, output and error streams; they are all open in text mode and - therefore follow the interface defined by the :class:`io.TextIOBase` - abstract class. - -Internal types - .. index:: - single: internal type - single: types, internal - - A few types used internally by the interpreter are exposed to the user. Their - definitions may change with future versions of the interpreter, but they are - mentioned here for completeness. - - .. index:: bytecode, object; code, code object - - Code objects - Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. - The difference between a code object and a function object is that the function - object contains an explicit reference to the function's globals (the module in - which it was defined), while a code object contains no context; also the default - argument values are stored in the function object, not in the code object - (because they represent values calculated at run-time). Unlike function - objects, code objects are immutable and contain no references (directly or - indirectly) to mutable objects. - - .. index:: - single: co_argcount (code object attribute) - single: co_code (code object attribute) - single: co_consts (code object attribute) - single: co_filename (code object attribute) - single: co_firstlineno (code object attribute) - single: co_flags (code object attribute) - single: co_lnotab (code object attribute) - single: co_name (code object attribute) - single: co_names (code object attribute) - single: co_nlocals (code object attribute) - single: co_stacksize (code object attribute) - single: co_varnames (code object attribute) - single: co_cellvars (code object attribute) - single: co_freevars (code object attribute) - - Special read-only attributes: :attr:`co_name` gives the function name; - :attr:`co_argcount` is the number of positional arguments (including arguments - with default values); :attr:`co_nlocals` is the number of local variables used - by the function (including arguments); :attr:`co_varnames` is a tuple containing - the names of the local variables (starting with the argument names); - :attr:`co_cellvars` is a tuple containing the names of local variables that are - referenced by nested functions; :attr:`co_freevars` is a tuple containing the - names of free variables; :attr:`co_code` is a string representing the sequence - of bytecode instructions; :attr:`co_consts` is a tuple containing the literals - used by the bytecode; :attr:`co_names` is a tuple containing the names used by - the bytecode; :attr:`co_filename` is the filename from which the code was - compiled; :attr:`co_firstlineno` is the first line number of the function; - :attr:`co_lnotab` is a string encoding the mapping from bytecode offsets to - line numbers (for details see the source code of the interpreter); - :attr:`co_stacksize` is the required stack size (including local variables); - :attr:`co_flags` is an integer encoding a number of flags for the interpreter. - - .. index:: object: generator - - The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if - the function uses the ``*arguments`` syntax to accept an arbitrary number of - positional arguments; bit ``0x08`` is set if the function uses the - ``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set - if the function is a generator. - - Future feature declarations (``from __future__ import division``) also use bits - in :attr:`co_flags` to indicate whether a code object was compiled with a - particular feature enabled: bit ``0x2000`` is set if the function was compiled - with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier - versions of Python. - - Other bits in :attr:`co_flags` are reserved for internal use. - - .. index:: single: documentation string - - If a code object represents a function, the first item in :attr:`co_consts` is - the documentation string of the function, or ``None`` if undefined. - - .. _frame-objects: - - Frame objects - .. index:: object: frame - - Frame objects represent execution frames. They may occur in traceback objects - (see below). - - .. index:: - single: f_back (frame attribute) - single: f_code (frame attribute) - single: f_globals (frame attribute) - single: f_locals (frame attribute) - single: f_lasti (frame attribute) - single: f_builtins (frame attribute) - - Special read-only attributes: :attr:`f_back` is to the previous stack frame - (towards the caller), or ``None`` if this is the bottom stack frame; - :attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` - is the dictionary used to look up local variables; :attr:`f_globals` is used for - global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; - :attr:`f_lasti` gives the precise instruction (this is an index into the - bytecode string of the code object). - - .. index:: - single: f_trace (frame attribute) - single: f_lineno (frame attribute) - - Special writable attributes: :attr:`f_trace`, if not ``None``, is a function - called at the start of each source code line (this is used by the debugger); - :attr:`f_lineno` is the current line number of the frame --- writing to this - from within a trace function jumps to the given line (only for the bottom-most - frame). A debugger can implement a Jump command (aka Set Next Statement) - by writing to f_lineno. - - Frame objects support one method: - - .. method:: frame.clear() - - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator - is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). - - :exc:`RuntimeError` is raised if the frame is currently executing. - - .. versionadded:: 3.4 - - Traceback objects - .. index:: - object: traceback - pair: stack; trace - pair: exception; handler - pair: execution; stack - single: exc_info (in module sys) - single: last_traceback (in module sys) - single: sys.exc_info - single: sys.last_traceback - - Traceback objects represent a stack trace of an exception. A traceback object - is created when an exception occurs. When the search for an exception handler - unwinds the execution stack, at each unwound level a traceback object is - inserted in front of the current traceback. When an exception handler is - entered, the stack trace is made available to the program. (See section - :ref:`try`.) It is accessible as the third item of the - tuple returned by ``sys.exc_info()``. When the program contains no suitable - handler, the stack trace is written (nicely formatted) to the standard error - stream; if the interpreter is interactive, it is also made available to the user - as ``sys.last_traceback``. - - .. index:: - single: tb_next (traceback attribute) - single: tb_frame (traceback attribute) - single: tb_lineno (traceback attribute) - single: tb_lasti (traceback attribute) - statement: try - - Special read-only attributes: :attr:`tb_next` is the next level in the stack - trace (towards the frame where the exception occurred), or ``None`` if there is - no next level; :attr:`tb_frame` points to the execution frame of the current - level; :attr:`tb_lineno` gives the line number where the exception occurred; - :attr:`tb_lasti` indicates the precise instruction. The line number and last - instruction in the traceback may differ from the line number of its frame object - if the exception occurred in a :keyword:`try` statement with no matching except - clause or with a finally clause. - - Slice objects - .. index:: builtin: slice - - Slice objects are used to represent slices for :meth:`__getitem__` - methods. They are also created by the built-in :func:`slice` function. - - .. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - - Special read-only attributes: :attr:`~slice.start` is the lower bound; - :attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step - value; each is ``None`` if omitted. These attributes can have any type. - - Slice objects support one method: - - .. method:: slice.indices(self, length) - - This method takes a single integer argument *length* and computes - information about the slice that the slice object would describe if - applied to a sequence of *length* items. It returns a tuple of three - integers; respectively these are the *start* and *stop* indices and the - *step* or stride length of the slice. Missing or out-of-bounds indices - are handled in a manner consistent with regular slices. - - Static method objects - Static method objects provide a way of defeating the transformation of function - objects to method objects described above. A static method object is a wrapper - around any other object, usually a user-defined method object. When a static - method object is retrieved from a class or a class instance, the object actually - returned is the wrapped object, which is not subject to any further - transformation. Static method objects are not themselves callable, although the - objects they wrap usually are. Static method objects are created by the built-in - :func:`staticmethod` constructor. - - Class method objects - A class method object, like a static method object, is a wrapper around another - object that alters the way in which that object is retrieved from classes and - class instances. The behaviour of class method objects upon such retrieval is - described above, under "User-defined methods". Class method objects are created - by the built-in :func:`classmethod` constructor. - - -.. _specialnames: - -Special method names -==================== - -.. index:: - pair: operator; overloading - single: __getitem__() (mapping object method) - -A class can implement certain operations that are invoked by special syntax -(such as arithmetic operations or subscripting and slicing) by defining methods -with special names. This is Python's approach to :dfn:`operator overloading`, -allowing classes to define their own behavior with respect to language -operators. For instance, if a class defines a method named :meth:`__getitem__`, -and ``x`` is an instance of this class, then ``x[i]`` is roughly equivalent -to ``type(x).__getitem__(x, i)``. Except where mentioned, attempts to execute an -operation raise an exception when no appropriate method is defined (typically -:exc:`AttributeError` or :exc:`TypeError`). - -Setting a special method to ``None`` indicates that the corresponding -operation is not available. For example, if a class sets -:meth:`__iter__` to ``None``, the class is not iterable, so calling -:func:`iter` on its instances will raise a :exc:`TypeError` (without -falling back to :meth:`__getitem__`). [#]_ - -When implementing a class that emulates any built-in type, it is important that -the emulation only be implemented to the degree that it makes sense for the -object being modelled. For example, some sequences may work well with retrieval -of individual elements, but extracting a slice may not make sense. (One example -of this is the :class:`~xml.dom.NodeList` interface in the W3C's Document -Object Model.) - - -.. _customization: - -Basic customization -------------------- - -.. method:: object.__new__(cls[, ...]) - - .. index:: pair: subclassing; immutable types - - Called to create a new instance of class *cls*. :meth:`__new__` is a static - method (special-cased so you need not declare it as such) that takes the class - of which an instance was requested as its first argument. The remaining - arguments are those passed to the object constructor expression (the call to the - class). The return value of :meth:`__new__` should be the new object instance - (usually an instance of *cls*). - - Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` - with appropriate arguments and then modifying the newly-created instance - as necessary before returning it. - - If :meth:`__new__` returns an instance of *cls*, then the new instance's - :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where - *self* is the new instance and the remaining arguments are the same as were - passed to :meth:`__new__`. - - If :meth:`__new__` does not return an instance of *cls*, then the new instance's - :meth:`__init__` method will not be invoked. - - :meth:`__new__` is intended mainly to allow subclasses of immutable types (like - int, str, or tuple) to customize instance creation. It is also commonly - overridden in custom metaclasses in order to customize class creation. - - -.. method:: object.__init__(self[, ...]) - - .. index:: pair: class; constructor - - Called after the instance has been created (by :meth:`__new__`), but before - it is returned to the caller. The arguments are those passed to the - class constructor expression. If a base class has an :meth:`__init__` - method, the derived class's :meth:`__init__` method, if any, must explicitly - call it to ensure proper initialization of the base class part of the - instance; for example: ``super().__init__([args...])``. - - Because :meth:`__new__` and :meth:`__init__` work together in constructing - objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), - no non-``None`` value may be returned by :meth:`__init__`; doing so will - cause a :exc:`TypeError` to be raised at runtime. - - -.. method:: object.__del__(self) - - .. index:: - single: destructor - single: finalizer - statement: del - - Called when the instance is about to be destroyed. This is also called a - finalizer or (improperly) a destructor. If a base class has a - :meth:`__del__` method, the derived class's :meth:`__del__` method, - if any, must explicitly call it to ensure proper deletion of the base - class part of the instance. - - It is possible (though not recommended!) for the :meth:`__del__` method - to postpone destruction of the instance by creating a new reference to - it. This is called object *resurrection*. It is implementation-dependent - whether :meth:`__del__` is called a second time when a resurrected object - is about to be destroyed; the current :term:`CPython` implementation - only calls it once. - - It is not guaranteed that :meth:`__del__` methods are called for objects - that still exist when the interpreter exits. - - .. note:: - - ``del x`` doesn't directly call ``x.__del__()`` --- the former decrements - the reference count for ``x`` by one, and the latter is only called when - ``x``'s reference count reaches zero. - - .. impl-detail:: - It is possible for a reference cycle to prevent the reference count - of an object from going to zero. In this case, the cycle will be - later detected and deleted by the :term:`cyclic garbage collector - `. A common cause of reference cycles is when - an exception has been caught in a local variable. The frame's - locals then reference the exception, which references its own - traceback, which references the locals of all frames caught in the - traceback. - - .. seealso:: - Documentation for the :mod:`gc` module. - - .. warning:: - - Due to the precarious circumstances under which :meth:`__del__` methods are - invoked, exceptions that occur during their execution are ignored, and a warning - is printed to ``sys.stderr`` instead. In particular: - - * :meth:`__del__` can be invoked when arbitrary code is being executed, - including from any arbitrary thread. If :meth:`__del__` needs to take - a lock or invoke any other blocking resource, it may deadlock as - the resource may already be taken by the code that gets interrupted - to execute :meth:`__del__`. - - * :meth:`__del__` can be executed during interpreter shutdown. As a - consequence, the global variables it needs to access (including other - modules) may already have been deleted or set to ``None``. Python - guarantees that globals whose name begins with a single underscore - are deleted from their module before other globals are deleted; if - no other references to such globals exist, this may help in assuring - that imported modules are still available at the time when the - :meth:`__del__` method is called. - - - .. index:: - single: repr() (built-in function); __repr__() (object method) - -.. method:: object.__repr__(self) - - Called by the :func:`repr` built-in function to compute the "official" string - representation of an object. If at all possible, this should look like a - valid Python expression that could be used to recreate an object with the - same value (given an appropriate environment). If this is not possible, a - string of the form ``<...some useful description...>`` should be returned. - The return value must be a string object. If a class defines :meth:`__repr__` - but not :meth:`__str__`, then :meth:`__repr__` is also used when an - "informal" string representation of instances of that class is required. - - This is typically used for debugging, so it is important that the representation - is information-rich and unambiguous. - - .. index:: - single: string; __str__() (object method) - single: format() (built-in function); __str__() (object method) - single: print() (built-in function); __str__() (object method) - - -.. method:: object.__str__(self) - - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely - printable string representation of an object. The return value must be a - :ref:`string ` object. - - This method differs from :meth:`object.__repr__` in that there is no - expectation that :meth:`__str__` return a valid Python expression: a more - convenient or concise representation can be used. - - The default implementation defined by the built-in type :class:`object` - calls :meth:`object.__repr__`. - - .. XXX what about subclasses of string? - - -.. method:: object.__bytes__(self) - - .. index:: builtin: bytes - - Called by :ref:`bytes ` to compute a byte-string representation - of an object. This should return a :class:`bytes` object. - - .. index:: - single: string; __format__() (object method) - pair: string; conversion - builtin: print - - -.. method:: object.__format__(self, format_spec) - - Called by the :func:`format` built-in function, - and by extension, evaluation of :ref:`formatted string literals - ` and the :meth:`str.format` method, to produce a "formatted" - string representation of an object. The ``format_spec`` argument is - a string that contains a description of the formatting options desired. - The interpretation of the ``format_spec`` argument is up to the type - implementing :meth:`__format__`, however most classes will either - delegate formatting to one of the built-in types, or use a similar - formatting option syntax. - - See :ref:`formatspec` for a description of the standard formatting syntax. - - The return value must be a string object. - - .. versionchanged:: 3.4 - The __format__ method of ``object`` itself raises a :exc:`TypeError` - if passed any non-empty string. - - -.. _richcmpfuncs: -.. method:: object.__lt__(self, other) - object.__le__(self, other) - object.__eq__(self, other) - object.__ne__(self, other) - object.__gt__(self, other) - object.__ge__(self, other) - - .. index:: - single: comparisons - - These are the so-called "rich comparison" methods. The correspondence between - operator symbols and method names is as follows: ``xy`` calls ``x.__gt__(y)``, and ``x>=y`` calls - ``x.__ge__(y)``. - - A rich comparison method may return the singleton ``NotImplemented`` if it does - not implement the operation for a given pair of arguments. By convention, - ``False`` and ``True`` are returned for a successful comparison. However, these - methods can return any value, so if the comparison operator is used in a Boolean - context (e.g., in the condition of an ``if`` statement), Python will call - :func:`bool` on the value to determine if the result is true or false. - - By default, :meth:`__ne__` delegates to :meth:`__eq__` and - inverts the result unless it is ``NotImplemented``. There are no other - implied relationships among the comparison operators, for example, - the truth of ``(x.__hash__``. - - If a class that does not override :meth:`__eq__` wishes to suppress hash - support, it should include ``__hash__ = None`` in the class definition. - A class which defines its own :meth:`__hash__` that explicitly raises - a :exc:`TypeError` would be incorrectly identified as hashable by - an ``isinstance(obj, collections.Hashable)`` call. - - - .. note:: - - By default, the :meth:`__hash__` values of str, bytes and datetime - objects are "salted" with an unpredictable random value. Although they - remain constant within an individual Python process, they are not - predictable between repeated invocations of Python. - - This is intended to provide protection against a denial-of-service caused - by carefully-chosen inputs that exploit the worst case performance of a - dict insertion, O(n^2) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. - - Changing hash values affects the iteration order of dicts, sets and - other mappings. Python has never made guarantees about this ordering - (and it typically varies between 32-bit and 64-bit builds). - - See also :envvar:`PYTHONHASHSEED`. - - .. versionchanged:: 3.3 - Hash randomization is enabled by default. - - -.. method:: object.__bool__(self) - - .. index:: single: __len__() (mapping object method) - - Called to implement truth value testing and the built-in operation - ``bool()``; should return ``False`` or ``True``. When this method is not - defined, :meth:`__len__` is called, if it is defined, and the object is - considered true if its result is nonzero. If a class defines neither - :meth:`__len__` nor :meth:`__bool__`, all its instances are considered - true. - - -.. _attribute-access: - -Customizing attribute access ----------------------------- - -The following methods can be defined to customize the meaning of attribute -access (use of, assignment to, or deletion of ``x.name``) for class instances. - -.. XXX explain how descriptors interfere here! - - -.. method:: object.__getattr__(self, name) - - Called when the default attribute access fails with an :exc:`AttributeError` - (either :meth:`__getattribute__` raises an :exc:`AttributeError` because - *name* is not an instance attribute or an attribute in the class tree - for ``self``; or :meth:`__get__` of a *name* property raises - :exc:`AttributeError`). This method should either return the (computed) - attribute value or raise an :exc:`AttributeError` exception. - - Note that if the attribute is found through the normal mechanism, - :meth:`__getattr__` is not called. (This is an intentional asymmetry between - :meth:`__getattr__` and :meth:`__setattr__`.) This is done both for efficiency - reasons and because otherwise :meth:`__getattr__` would have no way to access - other attributes of the instance. Note that at least for instance variables, - you can fake total control by not inserting any values in the instance attribute - dictionary (but instead inserting them in another object). See the - :meth:`__getattribute__` method below for a way to actually get total control - over attribute access. - - -.. method:: object.__getattribute__(self, name) - - Called unconditionally to implement attribute accesses for instances of the - class. If the class also defines :meth:`__getattr__`, the latter will not be - called unless :meth:`__getattribute__` either calls it explicitly or raises an - :exc:`AttributeError`. This method should return the (computed) attribute value - or raise an :exc:`AttributeError` exception. In order to avoid infinite - recursion in this method, its implementation should always call the base class - method with the same name to access any attributes it needs, for example, - ``object.__getattribute__(self, name)``. - - .. note:: - - This method may still be bypassed when looking up special methods as the - result of implicit invocation via language syntax or built-in functions. - See :ref:`special-lookup`. - - -.. method:: object.__setattr__(self, name, value) - - Called when an attribute assignment is attempted. This is called instead of - the normal mechanism (i.e. store the value in the instance dictionary). - *name* is the attribute name, *value* is the value to be assigned to it. - - If :meth:`__setattr__` wants to assign to an instance attribute, it should - call the base class method with the same name, for example, - ``object.__setattr__(self, name, value)``. - - -.. method:: object.__delattr__(self, name) - - Like :meth:`__setattr__` but for attribute deletion instead of assignment. This - should only be implemented if ``del obj.name`` is meaningful for the object. - - -.. method:: object.__dir__(self) - - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. - - -Customizing module attribute access -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __class__ (module attribute) - -For a more fine grained customization of the module behavior (setting -attributes, properties, etc.), one can set the ``__class__`` attribute of -a module object to a subclass of :class:`types.ModuleType`. For example:: - - import sys - from types import ModuleType - - class VerboseModule(ModuleType): - def __repr__(self): - return f'Verbose {self.__name__}' - - def __setattr__(self, attr, value): - print(f'Setting {attr}...') - setattr(self, attr, value) - - sys.modules[__name__].__class__ = VerboseModule - -.. note:: - Setting module ``__class__`` only affects lookups made using the attribute - access syntax -- directly accessing the module globals (whether by code - within the module, or via a reference to the module's globals dictionary) - is unaffected. - -.. versionchanged:: 3.5 - ``__class__`` module attribute is now writable. - - -.. _descriptors: - -Implementing Descriptors -^^^^^^^^^^^^^^^^^^^^^^^^ - -The following methods only apply when an instance of the class containing the -method (a so-called *descriptor* class) appears in an *owner* class (the -descriptor must be in either the owner's class dictionary or in the class -dictionary for one of its parents). In the examples below, "the attribute" -refers to the attribute whose name is the key of the property in the owner -class' :attr:`~object.__dict__`. - - -.. method:: object.__get__(self, instance, owner) - - Called to get the attribute of the owner class (class attribute access) or of an - instance of that class (instance attribute access). *owner* is always the owner - class, while *instance* is the instance that the attribute was accessed through, - or ``None`` when the attribute is accessed through the *owner*. This method - should return the (computed) attribute value or raise an :exc:`AttributeError` - exception. - - -.. method:: object.__set__(self, instance, value) - - Called to set the attribute on an instance *instance* of the owner class to a - new value, *value*. - - -.. method:: object.__delete__(self, instance) - - Called to delete the attribute on an instance *instance* of the owner class. - - -.. method:: object.__set_name__(self, owner, name) - - Called at the time the owning class *owner* is created. The - descriptor has been assigned to *name*. - - .. versionadded:: 3.6 - - -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). - - -.. _descriptor-invocation: - -Invoking Descriptors -^^^^^^^^^^^^^^^^^^^^ - -In general, a descriptor is an object attribute with "binding behavior", one -whose attribute access has been overridden by methods in the descriptor -protocol: :meth:`__get__`, :meth:`__set__`, and :meth:`__delete__`. If any of -those methods are defined for an object, it is said to be a descriptor. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the base classes of ``type(a)`` excluding metaclasses. - -However, if the looked-up value is an object defining one of the descriptor -methods, then Python may override the default behavior and invoke the descriptor -method instead. Where this occurs in the precedence chain depends on which -descriptor methods were defined and how they were called. - -The starting point for descriptor invocation is a binding, ``a.x``. How the -arguments are assembled depends on ``a``: - -Direct Call - The simplest and least common call is when user code directly invokes a - descriptor method: ``x.__get__(a)``. - -Instance Binding - If binding to an object instance, ``a.x`` is transformed into the call: - ``type(a).__dict__['x'].__get__(a, type(a))``. - -Class Binding - If binding to a class, ``A.x`` is transformed into the call: - ``A.__dict__['x'].__get__(None, A)``. - -Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` - searches ``obj.__class__.__mro__`` for the base class ``A`` - immediately preceding ``B`` and then invokes the descriptor with the call: - ``A.__dict__['m'].__get__(obj, obj.__class__)``. - -For instance bindings, the precedence of descriptor invocation depends on the -which descriptor methods are defined. A descriptor can define any combination -of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`. If it does not -define :meth:`__get__`, then accessing the attribute will return the descriptor -object itself unless there is a value in the object's instance dictionary. If -the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data -descriptor; if it defines neither, it is a non-data descriptor. Normally, data -descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data -descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__set__` and :meth:`__get__` defined always override a redefinition in an -instance dictionary. In contrast, non-data descriptors can be overridden by -instances. - -Python methods (including :func:`staticmethod` and :func:`classmethod`) are -implemented as non-data descriptors. Accordingly, instances can redefine and -override methods. This allows individual instances to acquire behaviors that -differ from other instances of the same class. - -The :func:`property` function is implemented as a data descriptor. Accordingly, -instances cannot override the behavior of a property. - - -.. _slots: - -__slots__ -^^^^^^^^^ - -*__slots__* allow us to explicitly declare data members (like -properties) and deny the creation of *__dict__* and *__weakref__* -(unless explicitly declared in *__slots__* or available in a parent.) - -The space saved over using *__dict__* can be significant. - -.. data:: object.__slots__ - - This class variable can be assigned a string, iterable, or sequence of - strings with variable names used by instances. *__slots__* reserves space - for the declared variables and prevents the automatic creation of *__dict__* - and *__weakref__* for each instance. - - -Notes on using *__slots__* -"""""""""""""""""""""""""" - -* When inheriting from a class without *__slots__*, the *__dict__* and - *__weakref__* attribute of the instances will always be accessible. - -* Without a *__dict__* variable, instances cannot be assigned new variables not - listed in the *__slots__* definition. Attempts to assign to an unlisted - variable name raises :exc:`AttributeError`. If dynamic assignment of new - variables is desired, then add ``'__dict__'`` to the sequence of strings in - the *__slots__* declaration. - -* Without a *__weakref__* variable for each instance, classes defining - *__slots__* do not support weak references to its instances. If weak reference - support is needed, then add ``'__weakref__'`` to the sequence of strings in the - *__slots__* declaration. - -* *__slots__* are implemented at the class level by creating descriptors - (:ref:`descriptors`) for each variable name. As a result, class attributes - cannot be used to set default values for instance variables defined by - *__slots__*; otherwise, the class attribute would overwrite the descriptor - assignment. - -* The action of a *__slots__* declaration is not limited to the class - where it is defined. *__slots__* declared in parents are available in - child classes. However, child subclasses will get a *__dict__* and - *__weakref__* unless they also define *__slots__* (which should only - contain names of any *additional* slots). - -* If a class defines a slot also defined in a base class, the instance variable - defined by the base class slot is inaccessible (except by retrieving its - descriptor directly from the base class). This renders the meaning of the - program undefined. In the future, a check may be added to prevent this. - -* Nonempty *__slots__* does not work for classes derived from "variable-length" - built-in types such as :class:`int`, :class:`bytes` and :class:`tuple`. - -* Any non-string iterable may be assigned to *__slots__*. Mappings may also be - used; however, in the future, special meaning may be assigned to the values - corresponding to each key. - -* *__class__* assignment works only if both classes have the same *__slots__*. - -* Multiple inheritance with multiple slotted parent classes can be used, - but only one parent is allowed to have attributes created by slots - (the other bases must have empty slot layouts) - violations raise - :exc:`TypeError`. - -.. _class-customization: - -Customizing class creation --------------------------- - -Whenever a class inherits from another class, *__init_subclass__* is -called on that class. This way, it is possible to write classes which -change the behavior of subclasses. This is closely related to class -decorators, but where class decorators only affect the specific class they're -applied to, ``__init_subclass__`` solely applies to future subclasses of the -class defining the method. - -.. classmethod:: object.__init_subclass__(cls) - - This method is called whenever the containing class is subclassed. - *cls* is then the new subclass. If defined as a normal instance method, - this method is implicitly converted to a class method. - - Keyword arguments which are given to a new class are passed to - the parent's class ``__init_subclass__``. For compatibility with - other classes using ``__init_subclass__``, one should take out the - needed keyword arguments and pass the others over to the base - class, as in:: - - class Philosopher: - def __init_subclass__(cls, default_name, **kwargs): - super().__init_subclass__(**kwargs) - cls.default_name = default_name - - class AustralianPhilosopher(Philosopher, default_name="Bruce"): - pass - - The default implementation ``object.__init_subclass__`` does - nothing, but raises an error if it is called with any arguments. - - .. note:: - - The metaclass hint ``metaclass`` is consumed by the rest of the type - machinery, and is never passed to ``__init_subclass__`` implementations. - The actual metaclass (rather than the explicit hint) can be accessed as - ``type(cls)``. - - .. versionadded:: 3.6 - - -.. _metaclasses: - -Metaclasses -^^^^^^^^^^^ - -.. index:: - single: metaclass - builtin: type - single: = (equals); class definition - -By default, classes are constructed using :func:`type`. The class body is -executed in a new namespace and the class name is bound locally to the -result of ``type(name, bases, namespace)``. - -The class creation process can be customized by passing the ``metaclass`` -keyword argument in the class definition line, or by inheriting from an -existing class that included such an argument. In the following example, -both ``MyClass`` and ``MySubclass`` are instances of ``Meta``:: - - class Meta(type): - pass - - class MyClass(metaclass=Meta): - pass - - class MySubclass(MyClass): - pass - -Any other keyword arguments that are specified in the class definition are -passed through to all metaclass operations described below. - -When a class definition is executed, the following steps occur: - -* the appropriate metaclass is determined -* the class namespace is prepared -* the class body is executed -* the class object is created - -Determining the appropriate metaclass -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. index:: - single: metaclass hint - -The appropriate metaclass for a class definition is determined as follows: - -* if no bases and no explicit metaclass are given, then :func:`type` is used -* if an explicit metaclass is given and it is *not* an instance of - :func:`type`, then it is used directly as the metaclass -* if an instance of :func:`type` is given as the explicit metaclass, or - bases are defined, then the most derived metaclass is used - -The most derived metaclass is selected from the explicitly specified -metaclass (if any) and the metaclasses (i.e. ``type(cls)``) of all specified -base classes. The most derived metaclass is one which is a subtype of *all* -of these candidate metaclasses. If none of the candidate metaclasses meets -that criterion, then the class definition will fail with ``TypeError``. - - -.. _prepare: - -Preparing the class namespace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __prepare__ (metaclass method) - -Once the appropriate metaclass has been identified, then the class namespace -is prepared. If the metaclass has a ``__prepare__`` attribute, it is called -as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the -additional keyword arguments, if any, come from the class definition). - -If the metaclass has no ``__prepare__`` attribute, then the class namespace -is initialised as an empty ordered mapping. - -.. seealso:: - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - - -Executing the class body -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: class; body - -The class body is executed (approximately) as -``exec(body, globals(), namespace)``. The key difference from a normal -call to :func:`exec` is that lexical scoping allows the class body (including -any methods) to reference names from the current and outer scopes when the -class definition occurs inside a function. - -However, even when the class definition occurs inside the function, methods -defined inside the class still cannot see names defined at the class scope. -Class variables must be accessed through the first parameter of instance or -class methods, or through the implicit lexically scoped ``__class__`` reference -described in the next section. - -.. _class-object-creation: - -Creating the class object -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __class__ (method cell) - single: __classcell__ (class namespace entry) - - -Once the class namespace has been populated by executing the class body, -the class object is created by calling -``metaclass(name, bases, namespace, **kwds)`` (the additional keywords -passed here are the same as those passed to ``__prepare__``). - -This class object is the one that will be referenced by the zero-argument -form of :func:`super`. ``__class__`` is an implicit closure reference -created by the compiler if any methods in a class body refer to either -``__class__`` or ``super``. This allows the zero argument form of -:func:`super` to correctly identify the class being defined based on -lexical scoping, while the class or instance that was used to make the -current call is identified based on the first argument passed to the method. - -.. impl-detail:: - - In CPython 3.6 and later, the ``__class__`` cell is passed to the metaclass - as a ``__classcell__`` entry in the class namespace. If present, this must - be propagated up to the ``type.__new__`` call in order for the class to be - initialised correctly. - Failing to do so will result in a :exc:`DeprecationWarning` in Python 3.6, - and a :exc:`RuntimeError` in Python 3.8. - -When using the default metaclass :class:`type`, or any metaclass that ultimately -calls ``type.__new__``, the following additional customisation steps are -invoked after creating the class object: - -* first, ``type.__new__`` collects all of the descriptors in the class - namespace that define a :meth:`~object.__set_name__` method; -* second, all of these ``__set_name__`` methods are called with the class - being defined and the assigned name of that particular descriptor; and -* finally, the :meth:`~object.__init_subclass__` hook is called on the - immediate parent of the new class in its method resolution order. - -After the class object is created, it is passed to the class decorators -included in the class definition (if any) and the resulting object is bound -in the local namespace as the defined class. - -When a new class is created by ``type.__new__``, the object provided as the -namespace parameter is copied to a new ordered mapping and the original -object is discarded. The new copy is wrapped in a read-only proxy, which -becomes the :attr:`~object.__dict__` attribute of the class object. - -.. seealso:: - - :pep:`3135` - New super - Describes the implicit ``__class__`` closure reference - - -Uses for metaclasses -^^^^^^^^^^^^^^^^^^^^ - -The potential uses for metaclasses are boundless. Some ideas that have been -explored include enum, logging, interface checking, automatic delegation, -automatic property creation, proxies, frameworks, and automatic resource -locking/synchronization. - - -Customizing instance and subclass checks ----------------------------------------- - -The following methods are used to override the default behavior of the -:func:`isinstance` and :func:`issubclass` built-in functions. - -In particular, the metaclass :class:`abc.ABCMeta` implements these methods in -order to allow the addition of Abstract Base Classes (ABCs) as "virtual base -classes" to any class or type (including built-in types), including other -ABCs. - -.. method:: class.__instancecheck__(self, instance) - - Return true if *instance* should be considered a (direct or indirect) - instance of *class*. If defined, called to implement ``isinstance(instance, - class)``. - - -.. method:: class.__subclasscheck__(self, subclass) - - Return true if *subclass* should be considered a (direct or indirect) - subclass of *class*. If defined, called to implement ``issubclass(subclass, - class)``. - - -Note that these methods are looked up on the type (metaclass) of a class. They -cannot be defined as class methods in the actual class. This is consistent with -the lookup of special methods that are called on instances, only in this -case the instance is itself a class. - -.. seealso:: - - :pep:`3119` - Introducing Abstract Base Classes - Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality - in the context of adding Abstract Base Classes (see the :mod:`abc` - module) to the language. - - -.. _callable-types: - -Emulating callable objects --------------------------- - - -.. method:: object.__call__(self[, args...]) - - .. index:: pair: call; instance - - Called when the instance is "called" as a function; if this method is defined, - ``x(arg1, arg2, ...)`` is a shorthand for ``x.__call__(arg1, arg2, ...)``. - - -.. _sequence-types: - -Emulating container types -------------------------- - -The following methods can be defined to implement container objects. Containers -usually are sequences (such as lists or tuples) or mappings (like dictionaries), -but can represent other containers as well. The first set of methods is used -either to emulate a sequence or to emulate a mapping; the difference is that for -a sequence, the allowable keys should be the integers *k* for which ``0 <= k < -N`` where *N* is the length of the sequence, or slice objects, which define a -range of items. It is also recommended that mappings provide the methods -:meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard dictionary -objects. The :mod:`collections` module provides a -:class:`~collections.abc.MutableMapping` -abstract base class to help create those methods from a base set of -:meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`. -Mutable sequences should provide methods :meth:`append`, :meth:`count`, -:meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard list objects. Finally, -sequence types should implement addition (meaning concatenation) and -multiplication (meaning repetition) by defining the methods :meth:`__add__`, -:meth:`__radd__`, :meth:`__iadd__`, :meth:`__mul__`, :meth:`__rmul__` and -:meth:`__imul__` described below; they should not define other numerical -operators. It is recommended that both mappings and sequences implement the -:meth:`__contains__` method to allow efficient use of the ``in`` operator; for -mappings, ``in`` should search the mapping's keys; for sequences, it should -search through the values. It is further recommended that both mappings and -sequences implement the :meth:`__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should be the same as -:meth:`keys`; for sequences, it should iterate through the values. - -.. method:: object.__len__(self) - - .. index:: - builtin: len - single: __bool__() (object method) - - Called to implement the built-in function :func:`len`. Should return the length - of the object, an integer ``>=`` 0. Also, an object that doesn't define a - :meth:`__bool__` method and whose :meth:`__len__` method returns zero is - considered to be false in a Boolean context. - - .. impl-detail:: - - In CPython, the length is required to be at most :attr:`sys.maxsize`. - If the length is larger than :attr:`!sys.maxsize` some features (such as - :func:`len`) may raise :exc:`OverflowError`. To prevent raising - :exc:`!OverflowError` by truth value testing, an object must define a - :meth:`__bool__` method. - - -.. method:: object.__length_hint__(self) - - Called to implement :func:`operator.length_hint`. Should return an estimated - length for the object (which may be greater or less than the actual length). - The length must be an integer ``>=`` 0. This method is purely an - optimization and is never required for correctness. - - .. versionadded:: 3.4 - - -.. index:: object: slice - -.. note:: - - Slicing is done exclusively with the following three methods. A call like :: - - a[1:2] = b - - is translated to :: - - a[slice(1, 2, None)] = b - - and so forth. Missing slice items are always filled in with ``None``. - - -.. method:: object.__getitem__(self, key) - - Called to implement evaluation of ``self[key]``. For sequence types, the - accepted keys should be integers and slice objects. Note that the special - interpretation of negative indexes (if the class wishes to emulate a sequence - type) is up to the :meth:`__getitem__` method. If *key* is of an inappropriate - type, :exc:`TypeError` may be raised; if of a value outside the set of indexes - for the sequence (after any special interpretation of negative values), - :exc:`IndexError` should be raised. For mapping types, if *key* is missing (not - in the container), :exc:`KeyError` should be raised. - - .. note:: - - :keyword:`for` loops expect that an :exc:`IndexError` will be raised for illegal - indexes to allow proper detection of the end of the sequence. - - -.. method:: object.__setitem__(self, key, value) - - Called to implement assignment to ``self[key]``. Same note as for - :meth:`__getitem__`. This should only be implemented for mappings if the - objects support changes to the values for keys, or if new keys can be added, or - for sequences if elements can be replaced. The same exceptions should be raised - for improper *key* values as for the :meth:`__getitem__` method. - - -.. method:: object.__delitem__(self, key) - - Called to implement deletion of ``self[key]``. Same note as for - :meth:`__getitem__`. This should only be implemented for mappings if the - objects support removal of keys, or for sequences if elements can be removed - from the sequence. The same exceptions should be raised for improper *key* - values as for the :meth:`__getitem__` method. - - -.. method:: object.__missing__(self, key) - - Called by :class:`dict`\ .\ :meth:`__getitem__` to implement ``self[key]`` for dict subclasses - when key is not in the dictionary. - - -.. method:: object.__iter__(self) - - This method is called when an iterator is required for a container. This method - should return a new iterator object that can iterate over all the objects in the - container. For mappings, it should iterate over the keys of the container. - - Iterator objects also need to implement this method; they are required to return - themselves. For more information on iterator objects, see :ref:`typeiter`. - - -.. method:: object.__reversed__(self) - - Called (if present) by the :func:`reversed` built-in to implement - reverse iteration. It should return a new iterator object that iterates - over all the objects in the container in reverse order. - - If the :meth:`__reversed__` method is not provided, the :func:`reversed` - built-in will fall back to using the sequence protocol (:meth:`__len__` and - :meth:`__getitem__`). Objects that support the sequence protocol should - only provide :meth:`__reversed__` if they can provide an implementation - that is more efficient than the one provided by :func:`reversed`. - - -The membership test operators (:keyword:`in` and :keyword:`not in`) are normally -implemented as an iteration through a sequence. However, container objects can -supply the following special method with a more efficient implementation, which -also does not require the object be a sequence. - -.. method:: object.__contains__(self, item) - - Called to implement membership test operators. Should return true if *item* - is in *self*, false otherwise. For mapping objects, this should consider the - keys of the mapping rather than the values or the key-item pairs. - - For objects that don't define :meth:`__contains__`, the membership test first - tries iteration via :meth:`__iter__`, then the old sequence iteration - protocol via :meth:`__getitem__`, see :ref:`this section in the language - reference `. - - -.. _numeric-types: - -Emulating numeric types ------------------------ - -The following methods can be defined to emulate numeric objects. Methods -corresponding to operations that are not supported by the particular kind of -number implemented (e.g., bitwise operations for non-integral numbers) should be -left undefined. - - -.. method:: object.__add__(self, other) - object.__sub__(self, other) - object.__mul__(self, other) - object.__matmul__(self, other) - object.__truediv__(self, other) - object.__floordiv__(self, other) - object.__mod__(self, other) - object.__divmod__(self, other) - object.__pow__(self, other[, modulo]) - object.__lshift__(self, other) - object.__rshift__(self, other) - object.__and__(self, other) - object.__xor__(self, other) - object.__or__(self, other) - - .. index:: - builtin: divmod - builtin: pow - builtin: pow - - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For instance, to - evaluate the expression ``x + y``, where *x* is an instance of a class that - has an :meth:`__add__` method, ``x.__add__(y)`` is called. The - :meth:`__divmod__` method should be the equivalent to using - :meth:`__floordiv__` and :meth:`__mod__`; it should not be related to - :meth:`__truediv__`. Note that :meth:`__pow__` should be defined to accept - an optional third argument if the ternary version of the built-in :func:`pow` - function is to be supported. - - If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. - - -.. method:: object.__radd__(self, other) - object.__rsub__(self, other) - object.__rmul__(self, other) - object.__rmatmul__(self, other) - object.__rtruediv__(self, other) - object.__rfloordiv__(self, other) - object.__rmod__(self, other) - object.__rdivmod__(self, other) - object.__rpow__(self, other) - object.__rlshift__(self, other) - object.__rrshift__(self, other) - object.__rand__(self, other) - object.__rxor__(self, other) - object.__ror__(self, other) - - .. index:: - builtin: divmod - builtin: pow - - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected - (swapped) operands. These functions are only called if the left operand does - not support the corresponding operation [#]_ and the operands are of different - types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is - an instance of a class that has an :meth:`__rsub__` method, ``y.__rsub__(x)`` - is called if ``x.__sub__(y)`` returns *NotImplemented*. - - .. index:: builtin: pow - - Note that ternary :func:`pow` will not try calling :meth:`__rpow__` (the - coercion rules would become too complicated). - - .. note:: - - If the right operand's type is a subclass of the left operand's type and that - subclass provides the reflected method for the operation, this method will be - called before the left operand's non-reflected method. This behavior allows - subclasses to override their ancestors' operations. - - -.. method:: object.__iadd__(self, other) - object.__isub__(self, other) - object.__imul__(self, other) - object.__imatmul__(self, other) - object.__itruediv__(self, other) - object.__ifloordiv__(self, other) - object.__imod__(self, other) - object.__ipow__(self, other[, modulo]) - object.__ilshift__(self, other) - object.__irshift__(self, other) - object.__iand__(self, other) - object.__ixor__(self, other) - object.__ior__(self, other) - - These methods are called to implement the augmented arithmetic assignments - (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, - ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the - operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the - augmented assignment falls back to the normal methods. For instance, if *x* - is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and - ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In - certain situations, augmented assignment can result in unexpected errors (see - :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact - part of the data model. - - -.. method:: object.__neg__(self) - object.__pos__(self) - object.__abs__(self) - object.__invert__(self) - - .. index:: builtin: abs - - Called to implement the unary arithmetic operations (``-``, ``+``, :func:`abs` - and ``~``). - - -.. method:: object.__complex__(self) - object.__int__(self) - object.__float__(self) - - .. index:: - builtin: complex - builtin: int - builtin: float - - Called to implement the built-in functions :func:`complex`, - :func:`int` and :func:`float`. Should return a value - of the appropriate type. - - -.. method:: object.__index__(self) - - Called to implement :func:`operator.index`, and whenever Python needs to - losslessly convert the numeric object to an integer object (such as in - slicing, or in the built-in :func:`bin`, :func:`hex` and :func:`oct` - functions). Presence of this method indicates that the numeric object is - an integer type. Must return an integer. - - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. - - -.. method:: object.__round__(self, [,ndigits]) - object.__trunc__(self) - object.__floor__(self) - object.__ceil__(self) - - .. index:: builtin: round - - Called to implement the built-in function :func:`round` and :mod:`math` - functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. - Unless *ndigits* is passed to :meth:`!__round__` all these methods should - return the value of the object truncated to an :class:`~numbers.Integral` - (typically an :class:`int`). - - If :meth:`__int__` is not defined then the built-in function :func:`int` - falls back to :meth:`__trunc__`. - - -.. _context-managers: - -With Statement Context Managers -------------------------------- - -A :dfn:`context manager` is an object that defines the runtime context to be -established when executing a :keyword:`with` statement. The context manager -handles the entry into, and the exit from, the desired runtime context for the -execution of the block of code. Context managers are normally invoked using the -:keyword:`with` statement (described in section :ref:`with`), but can also be -used by directly invoking their methods. - -.. index:: - statement: with - single: context manager - -Typical uses of context managers include saving and restoring various kinds of -global state, locking and unlocking resources, closing opened files, etc. - -For more information on context managers, see :ref:`typecontextmanager`. - - -.. method:: object.__enter__(self) - - Enter the runtime context related to this object. The :keyword:`with` statement - will bind this method's return value to the target(s) specified in the - :keyword:`as` clause of the statement, if any. - - -.. method:: object.__exit__(self, exc_type, exc_value, traceback) - - Exit the runtime context related to this object. The parameters describe the - exception that caused the context to be exited. If the context was exited - without an exception, all three arguments will be :const:`None`. - - If an exception is supplied, and the method wishes to suppress the exception - (i.e., prevent it from being propagated), it should return a true value. - Otherwise, the exception will be processed normally upon exit from this method. - - Note that :meth:`__exit__` methods should not reraise the passed-in exception; - this is the caller's responsibility. - - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - - -.. _special-lookup: - -Special method lookup ---------------------- - -For custom classes, implicit invocations of special methods are only guaranteed -to work correctly if defined on an object's type, not in the object's instance -dictionary. That behaviour is the reason why the following code raises an -exception:: - - >>> class C: - ... pass - ... - >>> c = C() - >>> c.__len__ = lambda: 5 - >>> len(c) - Traceback (most recent call last): - File "", line 1, in - TypeError: object of type 'C' has no len() - -The rationale behind this behaviour lies with a number of special methods such -as :meth:`__hash__` and :meth:`__repr__` that are implemented by all objects, -including type objects. If the implicit lookup of these methods used the -conventional lookup process, they would fail when invoked on the type object -itself:: - - >>> 1 .__hash__() == hash(1) - True - >>> int.__hash__() == hash(int) - Traceback (most recent call last): - File "", line 1, in - TypeError: descriptor '__hash__' of 'int' object needs an argument - -Incorrectly attempting to invoke an unbound method of a class in this way is -sometimes referred to as 'metaclass confusion', and is avoided by bypassing -the instance when looking up special methods:: - - >>> type(1).__hash__(1) == hash(1) - True - >>> type(int).__hash__(int) == hash(int) - True - -In addition to bypassing any instance attributes in the interest of -correctness, implicit special method lookup generally also bypasses the -:meth:`__getattribute__` method even of the object's metaclass:: - - >>> class Meta(type): - ... def __getattribute__(*args): - ... print("Metaclass getattribute invoked") - ... return type.__getattribute__(*args) - ... - >>> class C(object, metaclass=Meta): - ... def __len__(self): - ... return 10 - ... def __getattribute__(*args): - ... print("Class getattribute invoked") - ... return object.__getattribute__(*args) - ... - >>> c = C() - >>> c.__len__() # Explicit lookup via instance - Class getattribute invoked - 10 - >>> type(c).__len__(c) # Explicit lookup via type - Metaclass getattribute invoked - 10 - >>> len(c) # Implicit lookup - 10 - -Bypassing the :meth:`__getattribute__` machinery in this fashion -provides significant scope for speed optimisations within the -interpreter, at the cost of some flexibility in the handling of -special methods (the special method *must* be set on the class -object itself in order to be consistently invoked by the interpreter). - - -.. index:: - single: coroutine - -Coroutines -========== - - -Awaitable Objects ------------------ - -An :term:`awaitable` object generally implements an :meth:`__await__` method. -:term:`Coroutine` objects returned from :keyword:`async def` functions -are awaitable. - -.. note:: - - The :term:`generator iterator` objects returned from generators - decorated with :func:`types.coroutine` or :func:`asyncio.coroutine` - are also awaitable, but they do not implement :meth:`__await__`. - -.. method:: object.__await__(self) - - Must return an :term:`iterator`. Should be used to implement - :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements - this method to be compatible with the :keyword:`await` expression. - -.. versionadded:: 3.5 - -.. seealso:: :pep:`492` for additional information about awaitable objects. - - -.. _coroutine-objects: - -Coroutine Objects ------------------ - -:term:`Coroutine` objects are :term:`awaitable` objects. -A coroutine's execution can be controlled by calling :meth:`__await__` and -iterating over the result. When the coroutine has finished executing and -returns, the iterator raises :exc:`StopIteration`, and the exception's -:attr:`~StopIteration.value` attribute holds the return value. If the -coroutine raises an exception, it is propagated by the iterator. Coroutines -should not directly raise unhandled :exc:`StopIteration` exceptions. - -Coroutines also have the methods listed below, which are analogous to -those of generators (see :ref:`generator-methods`). However, unlike -generators, coroutines do not directly support iteration. - -.. versionchanged:: 3.5.2 - It is a :exc:`RuntimeError` to await on a coroutine more than once. - - -.. method:: coroutine.send(value) - - Starts or resumes execution of the coroutine. If *value* is ``None``, - this is equivalent to advancing the iterator returned by - :meth:`__await__`. If *value* is not ``None``, this method delegates - to the :meth:`~generator.send` method of the iterator that caused - the coroutine to suspend. The result (return value, - :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. - -.. method:: coroutine.throw(type[, value[, traceback]]) - - Raises the specified exception in the coroutine. This method delegates - to the :meth:`~generator.throw` method of the iterator that caused - the coroutine to suspend, if it has such a method. Otherwise, - the exception is raised at the suspension point. The result - (return value, :exc:`StopIteration`, or other exception) is the same as - when iterating over the :meth:`__await__` return value, described - above. If the exception is not caught in the coroutine, it propagates - back to the caller. - -.. method:: coroutine.close() - - Causes the coroutine to clean itself up and exit. If the coroutine - is suspended, this method first delegates to the :meth:`~generator.close` - method of the iterator that caused the coroutine to suspend, if it - has such a method. Then it raises :exc:`GeneratorExit` at the - suspension point, causing the coroutine to immediately clean itself up. - Finally, the coroutine is marked as having finished executing, even if - it was never started. - - Coroutine objects are automatically closed using the above process when - they are about to be destroyed. - -.. _async-iterators: - -Asynchronous Iterators ----------------------- - -An *asynchronous iterable* is able to call asynchronous code in its -``__aiter__`` implementation, and an *asynchronous iterator* can call -asynchronous code in its ``__anext__`` method. - -Asynchronous iterators can be used in an :keyword:`async for` statement. - -.. method:: object.__aiter__(self) - - Must return an *asynchronous iterator* object. - -.. method:: object.__anext__(self) - - Must return an *awaitable* resulting in a next value of the iterator. Should - raise a :exc:`StopAsyncIteration` error when the iteration is over. - -An example of an asynchronous iterable object:: - - class Reader: - async def readline(self): - ... - - def __aiter__(self): - return self - - async def __anext__(self): - val = await self.readline() - if val == b'': - raise StopAsyncIteration - return val - -.. versionadded:: 3.5 - -.. note:: - - .. versionchanged:: 3.5.2 - Starting with CPython 3.5.2, ``__aiter__`` can directly return - :term:`asynchronous iterators `. Returning - an :term:`awaitable` object will result in a - :exc:`PendingDeprecationWarning`. - - The recommended way of writing backwards compatible code in - CPython 3.5.x is to continue returning awaitables from - ``__aiter__``. If you want to avoid the PendingDeprecationWarning - and keep the code backwards compatible, the following decorator - can be used:: - - import functools - import sys - - if sys.version_info < (3, 5, 2): - def aiter_compat(func): - @functools.wraps(func) - async def wrapper(self): - return func(self) - return wrapper - else: - def aiter_compat(func): - return func - - Example:: - - class AsyncIterator: - - @aiter_compat - def __aiter__(self): - return self - - async def __anext__(self): - ... - - Starting with CPython 3.6, the :exc:`PendingDeprecationWarning` - will be replaced with the :exc:`DeprecationWarning`. - In CPython 3.7, returning an awaitable from ``__aiter__`` will - result in a :exc:`RuntimeError`. - - -Asynchronous Context Managers ------------------------------ - -An *asynchronous context manager* is a *context manager* that is able to -suspend execution in its ``__aenter__`` and ``__aexit__`` methods. - -Asynchronous context managers can be used in an :keyword:`async with` statement. - -.. method:: object.__aenter__(self) - - This method is semantically similar to the :meth:`__enter__`, with only - difference that it must return an *awaitable*. - -.. method:: object.__aexit__(self, exc_type, exc_value, traceback) - - This method is semantically similar to the :meth:`__exit__`, with only - difference that it must return an *awaitable*. - -An example of an asynchronous context manager class:: - - class AsyncContextManager: - async def __aenter__(self): - await log('entering context') - - async def __aexit__(self, exc_type, exc, tb): - await log('exiting context') - -.. versionadded:: 3.5 - - -.. rubric:: Footnotes - -.. [#] It *is* possible in some cases to change an object's type, under certain - controlled conditions. It generally isn't a good idea though, since it can - lead to some very strange behaviour if it is handled incorrectly. - -.. [#] The :meth:`__hash__`, :meth:`__iter__`, :meth:`__reversed__`, and - :meth:`__contains__` methods have special handling for this; others - will still raise a :exc:`TypeError`, but may do so by relying on - the behavior that ``None`` is not callable. - -.. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to - ``None`` if you want to force fallback to the right operand's reflected - method—that will instead have the opposite effect of explicitly - *blocking* such fallback. - -.. [#] For operands of the same type, it is assumed that if the non-reflected method - (such as :meth:`__add__`) fails the operation is not supported, which is why the - reflected method is not called. diff --git a/third_party/python/Doc/reference/executionmodel.rst b/third_party/python/Doc/reference/executionmodel.rst deleted file mode 100644 index 1a69e972f..000000000 --- a/third_party/python/Doc/reference/executionmodel.rst +++ /dev/null @@ -1,266 +0,0 @@ - -.. _execmodel: - -*************** -Execution model -*************** - -.. index:: - single: execution model - pair: code; block - -.. _prog_structure: - -Structure of a program -====================== - -.. index:: block - -A Python program is constructed from code blocks. -A :dfn:`block` is a piece of Python program text that is executed as a unit. -The following are blocks: a module, a function body, and a class definition. -Each command typed interactively is a block. A script file (a file given as -standard input to the interpreter or specified as a command line argument to the -interpreter) is a code block. A script command (a command specified on the -interpreter command line with the :option:`-c` option) is a code block. The string -argument passed to the built-in functions :func:`eval` and :func:`exec` is a -code block. - -.. index:: pair: execution; frame - -A code block is executed in an :dfn:`execution frame`. A frame contains some -administrative information (used for debugging) and determines where and how -execution continues after the code block's execution has completed. - -.. _naming: - -Naming and binding -================== - -.. index:: - single: namespace - single: scope - -.. _bind_names: - -Binding of names ----------------- - -.. index:: - single: name - pair: binding; name - -:dfn:`Names` refer to objects. Names are introduced by name binding operations. - -.. index:: single: from; import statement - -The following constructs bind names: formal parameters to functions, -:keyword:`import` statements, class and function definitions (these bind the -class or function name in the defining block), and targets that are identifiers -if occurring in an assignment, :keyword:`for` loop header, or after -:keyword:`as` in a :keyword:`with` statement or :keyword:`except` clause. -The :keyword:`import` statement -of the form ``from ... import *`` binds all names defined in the imported -module, except those beginning with an underscore. This form may only be used -at the module level. - -A target occurring in a :keyword:`del` statement is also considered bound for -this purpose (though the actual semantics are to unbind the name). - -Each assignment or import statement occurs within a block defined by a class or -function definition or at the module level (the top-level code block). - -.. index:: pair: free; variable - -If a name is bound in a block, it is a local variable of that block, unless -declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at -the module level, it is a global variable. (The variables of the module code -block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. - -Each occurrence of a name in the program text refers to the :dfn:`binding` of -that name established by the following name resolution rules. - -.. _resolve_names: - -Resolution of names -------------------- - -.. index:: scope - -A :dfn:`scope` defines the visibility of a name within a block. If a local -variable is defined in a block, its scope includes that block. If the -definition occurs in a function block, the scope extends to any blocks contained -within the defining one, unless a contained block introduces a different binding -for the name. - -.. index:: single: environment - -When a name is used in a code block, it is resolved using the nearest enclosing -scope. The set of all such scopes visible to a code block is called the block's -:dfn:`environment`. - -.. index:: - single: NameError (built-in exception) - single: UnboundLocalError - -When a name is not found at all, a :exc:`NameError` exception is raised. -If the current scope is a function scope, and the name refers to a local -variable that has not yet been bound to a value at the point where the name is -used, an :exc:`UnboundLocalError` exception is raised. -:exc:`UnboundLocalError` is a subclass of :exc:`NameError`. - -If a name binding operation occurs anywhere within a code block, all uses of the -name within the block are treated as references to the current block. This can -lead to errors when a name is used within a block before it is bound. This rule -is subtle. Python lacks declarations and allows name binding operations to -occur anywhere within a code block. The local variables of a code block can be -determined by scanning the entire text of the block for name binding operations. - -If the :keyword:`global` statement occurs within a block, all uses of the name -specified in the statement refer to the binding of that name in the top-level -namespace. Names are resolved in the top-level namespace by searching the -global namespace, i.e. the namespace of the module containing the code block, -and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the name is not found there, the -builtins namespace is searched. The :keyword:`global` statement must precede -all uses of the name. - -The :keyword:`global` statement has the same scope as a name binding operation -in the same block. If the nearest enclosing scope for a free variable contains -a global statement, the free variable is treated as a global. - -.. XXX say more about "nonlocal" semantics here - -The :keyword:`nonlocal` statement causes corresponding names to refer -to previously bound variables in the nearest enclosing function scope. -:exc:`SyntaxError` is raised at compile time if the given name does not -exist in any enclosing function scope. - -.. index:: module: __main__ - -The namespace for a module is automatically created the first time a module is -imported. The main module for a script is always called :mod:`__main__`. - -Class definition blocks and arguments to :func:`exec` and :func:`eval` are -special in the context of name resolution. -A class definition is an executable statement that may use and define names. -These references follow the normal rules for name resolution with an exception -that unbound local variables are looked up in the global namespace. -The namespace of the class definition becomes the attribute dictionary of -the class. The scope of names defined in a class block is limited to the -class block; it does not extend to the code blocks of methods -- this includes -comprehensions and generator expressions since they are implemented using a -function scope. This means that the following will fail:: - - class A: - a = 42 - b = list(a + i for i in range(10)) - -.. _restrict_exec: - -Builtins and restricted execution ---------------------------------- - -.. index:: pair: restricted; execution - -.. impl-detail:: - - Users should not touch ``__builtins__``; it is strictly an implementation - detail. Users wanting to override values in the builtins namespace should - :keyword:`import` the :mod:`builtins` module and modify its - attributes appropriately. - -The builtins namespace associated with the execution of a code block -is actually found by looking up the name ``__builtins__`` in its -global namespace; this should be a dictionary or a module (in the -latter case the module's dictionary is used). By default, when in the -:mod:`__main__` module, ``__builtins__`` is the built-in module -:mod:`builtins`; when in any other module, ``__builtins__`` is an -alias for the dictionary of the :mod:`builtins` module itself. - - -.. _dynamic-features: - -Interaction with dynamic features ---------------------------------- - -Name resolution of free variables occurs at runtime, not at compile time. -This means that the following code will print 42:: - - i = 10 - def f(): - print(i) - i = 42 - f() - -.. XXX from * also invalid with relative imports (at least currently) - -The :func:`eval` and :func:`exec` functions do not have access to the full -environment for resolving names. Names may be resolved in the local and global -namespaces of the caller. Free variables are not resolved in the nearest -enclosing namespace, but in the global namespace. [#]_ The :func:`exec` and -:func:`eval` functions have optional arguments to override the global and local -namespace. If only one namespace is specified, it is used for both. - - -.. _exceptions: - -Exceptions -========== - -.. index:: single: exception - -.. index:: - single: raise an exception - single: handle an exception - single: exception handler - single: errors - single: error handling - -Exceptions are a means of breaking out of the normal flow of control of a code -block in order to handle errors or other exceptional conditions. An exception -is *raised* at the point where the error is detected; it may be *handled* by the -surrounding code block or by any code block that directly or indirectly invoked -the code block where the error occurred. - -The Python interpreter raises an exception when it detects a run-time error -(such as division by zero). A Python program can also explicitly raise an -exception with the :keyword:`raise` statement. Exception handlers are specified -with the :keyword:`try` ... :keyword:`except` statement. The :keyword:`finally` -clause of such a statement can be used to specify cleanup code which does not -handle the exception, but is executed whether an exception occurred or not in -the preceding code. - -.. index:: single: termination model - -Python uses the "termination" model of error handling: an exception handler can -find out what happened and continue execution at an outer level, but it cannot -repair the cause of the error and retry the failing operation (except by -re-entering the offending piece of code from the top). - -.. index:: single: SystemExit (built-in exception) - -When an exception is not handled at all, the interpreter terminates execution of -the program, or returns to its interactive main loop. In either case, it prints -a stack backtrace, except when the exception is :exc:`SystemExit`. - -Exceptions are identified by class instances. The :keyword:`except` clause is -selected depending on the class of the instance: it must reference the class of -the instance or a base class thereof. The instance can be received by the -handler and can carry additional information about the exceptional condition. - -.. note:: - - Exception messages are not part of the Python API. Their contents may change - from one version of Python to the next without warning and should not be - relied on by code which will run under multiple versions of the interpreter. - -See also the description of the :keyword:`try` statement in section :ref:`try` -and :keyword:`raise` statement in section :ref:`raise`. - - -.. rubric:: Footnotes - -.. [#] This limitation occurs because the code that is executed by these operations - is not available at the time the module is compiled. diff --git a/third_party/python/Doc/reference/expressions.rst b/third_party/python/Doc/reference/expressions.rst deleted file mode 100644 index e13072fee..000000000 --- a/third_party/python/Doc/reference/expressions.rst +++ /dev/null @@ -1,1843 +0,0 @@ - -.. _expressions: - -*********** -Expressions -*********** - -.. index:: expression, BNF - -This chapter explains the meaning of the elements of expressions in Python. - -**Syntax Notes:** In this and the following chapters, extended BNF notation will -be used to describe syntax, not lexical analysis. When (one alternative of) a -syntax rule has the form - -.. productionlist:: * - name: `othername` - -and no semantics are given, the semantics of this form of ``name`` are the same -as for ``othername``. - - -.. _conversions: - -Arithmetic conversions -====================== - -.. index:: pair: arithmetic; conversion - -When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type," this means that the operator -implementation for built-in types works as follows: - -* If either argument is a complex number, the other is converted to complex; - -* otherwise, if either argument is a floating point number, the other is - converted to floating point; - -* otherwise, both must be integers and no conversion is necessary. - -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. - - -.. _atoms: - -Atoms -===== - -.. index:: atom - -Atoms are the most basic elements of expressions. The simplest atoms are -identifiers or literals. Forms enclosed in parentheses, brackets or braces are -also categorized syntactically as atoms. The syntax for atoms is: - -.. productionlist:: - atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` - - -.. _atom-identifiers: - -Identifiers (Names) -------------------- - -.. index:: name, identifier - -An identifier occurring as an atom is a name. See section :ref:`identifiers` -for lexical definition and section :ref:`naming` for documentation of naming and -binding. - -.. index:: exception: NameError - -When the name is bound to an object, evaluation of the atom yields that object. -When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` -exception. - -.. index:: - pair: name; mangling - pair: private; names - -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. - - -.. _atom-literals: - -Literals --------- - -.. index:: single: literal - -Python supports string and bytes literals and various numeric literals: - -.. productionlist:: - literal: `stringliteral` | `bytesliteral` - : | `integer` | `floatnumber` | `imagnumber` - -Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating point number, complex number) with the given value. The value -may be approximated in the case of floating point and imaginary (complex) -literals. See section :ref:`literals` for details. - -.. index:: - triple: immutable; data; type - pair: immutable; object - -All literals correspond to immutable data types, and hence the object's identity -is less important than its value. Multiple evaluations of literals with the -same value (either the same occurrence in the program text or a different -occurrence) may obtain the same object or a different object with the same -value. - - -.. _parenthesized: - -Parenthesized forms -------------------- - -.. index:: - single: parenthesized form - single: () (parentheses); tuple display - -A parenthesized form is an optional expression list enclosed in parentheses: - -.. productionlist:: - parenth_form: "(" [`starred_expression`] ")" - -A parenthesized expression list yields whatever that expression list yields: if -the list contains at least one comma, it yields a tuple; otherwise, it yields -the single expression that makes up the expression list. - -.. index:: pair: empty; tuple - -An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the rules for literals apply (i.e., two occurrences of the empty -tuple may or may not yield the same object). - -.. index:: - single: comma; tuple display - pair: tuple; display - single: , (comma); tuple display - -Note that tuples are not formed by the parentheses, but rather by use of the -comma operator. The exception is the empty tuple, for which parentheses *are* -required --- allowing unparenthesized "nothing" in expressions would cause -ambiguities and allow common typos to pass uncaught. - - -.. _comprehensions: - -Displays for lists, sets and dictionaries ------------------------------------------ - -For constructing a list, a set or a dictionary Python provides special syntax -called "displays", each of them in two flavors: - -* either the container contents are listed explicitly, or - -* they are computed via a set of looping and filtering instructions, called a - :dfn:`comprehension`. - -.. index:: - single: for; in comprehensions - single: if; in comprehensions - single: async for; in comprehensions - -Common syntax elements for comprehensions are: - -.. productionlist:: - comprehension: `expression` `comp_for` - comp_for: [ASYNC] "for" `target_list` "in" `or_test` [`comp_iter`] - comp_iter: `comp_for` | `comp_if` - comp_if: "if" `expression_nocond` [`comp_iter`] - -The comprehension consists of a single expression followed by at least one -:keyword:`for` clause and zero or more :keyword:`for` or :keyword:`if` clauses. -In this case, the elements of the new container are those that would be produced -by considering each of the :keyword:`for` or :keyword:`if` clauses a block, -nesting from left to right, and evaluating the expression to produce an element -each time the innermost block is reached. - -Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" into the enclosing scope. - -.. index:: - single: await; in comprehensions - -Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` -clause may be used to iterate over a :term:`asynchronous iterator`. -A comprehension in an :keyword:`async def` function may consist of either a -:keyword:`for` or :keyword:`async for` clause following the leading -expression, may contain additional :keyword:`for` or :keyword:`async for` -clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`async for` clauses -or :keyword:`await` expressions it is called an -:dfn:`asynchronous comprehension`. An asynchronous comprehension may -suspend the execution of the coroutine function in which it appears. -See also :pep:`530`. - -.. _lists: - -List displays -------------- - -.. index:: - pair: list; display - pair: list; comprehensions - pair: empty; list - object: list - single: [] (square brackets); list expression - single: , (comma); expression list - -A list display is a possibly empty series of expressions enclosed in square -brackets: - -.. productionlist:: - list_display: "[" [`starred_list` | `comprehension`] "]" - -A list display yields a new list object, the contents being specified by either -a list of expressions or a comprehension. When a comma-separated list of -expressions is supplied, its elements are evaluated from left to right and -placed into the list object in that order. When a comprehension is supplied, -the list is constructed from the elements resulting from the comprehension. - - -.. _set: - -Set displays ------------- - -.. index:: - pair: set; display - object: set - single: {} (curly brackets); set expression - single: , (comma); expression list - -A set display is denoted by curly braces and distinguishable from dictionary -displays by the lack of colons separating keys and values: - -.. productionlist:: - set_display: "{" (`starred_list` | `comprehension`) "}" - -A set display yields a new mutable set object, the contents being specified by -either a sequence of expressions or a comprehension. When a comma-separated -list of expressions is supplied, its elements are evaluated from left to right -and added to the set object. When a comprehension is supplied, the set is -constructed from the elements resulting from the comprehension. - -An empty set cannot be constructed with ``{}``; this literal constructs an empty -dictionary. - - -.. _dict: - -Dictionary displays -------------------- - -.. index:: - pair: dictionary; display - key, datum, key/datum pair - object: dictionary - single: {} (curly brackets); dictionary expression - single: : (colon); in dictionary expressions - single: , (comma); in dictionary displays - -A dictionary display is a possibly empty series of key/datum pairs enclosed in -curly braces: - -.. productionlist:: - dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" - key_datum_list: `key_datum` ("," `key_datum`)* [","] - key_datum: `expression` ":" `expression` | "**" `or_expr` - dict_comprehension: `expression` ":" `expression` `comp_for` - -A dictionary display yields a new dictionary object. - -If a comma-separated sequence of key/datum pairs is given, they are evaluated -from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding datum. This means -that you can specify the same key multiple times in the key/datum list, and the -final dictionary's value for that key will be the last one given. - -.. index:: - unpacking; dictionary - single: **; in dictionary displays - -A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. -Its operand must be a :term:`mapping`. Each mapping item is added -to the new dictionary. Later values replace values already set by -earlier key/datum pairs and earlier dictionary unpackings. - -.. versionadded:: 3.5 - Unpacking into dictionary displays, originally proposed by :pep:`448`. - -A dict comprehension, in contrast to list and set comprehensions, needs two -expressions separated with a colon followed by the usual "for" and "if" clauses. -When the comprehension is run, the resulting key and value elements are inserted -in the new dictionary in the order they are produced. - -.. index:: pair: immutable; object - hashable - -Restrictions on the types of the key values are listed earlier in section -:ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes -all mutable objects.) Clashes between duplicate keys are not detected; the last -datum (textually rightmost in the display) stored for a given key value -prevails. - - -.. _genexpr: - -Generator expressions ---------------------- - -.. index:: - pair: generator; expression - object: generator - single: () (parentheses); generator expression - -A generator expression is a compact generator notation in parentheses: - -.. productionlist:: - generator_expression: "(" `expression` `comp_for` ")" - -A generator expression yields a new generator object. Its syntax is the same as -for comprehensions, except that it is enclosed in parentheses instead of -brackets or curly braces. - -Variables used in the generator expression are evaluated lazily when the -:meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the leftmost :keyword:`for` clause is -immediately evaluated, so that an error produced by it can be seen before any -other possible error in the code that handles the generator expression. -Subsequent :keyword:`for` clauses cannot be evaluated immediately since they -may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in bar(x))``. - -The parentheses can be omitted on calls with only one argument. See section -:ref:`calls` for details. - -Since Python 3.6, if the generator appears in an :keyword:`async def` function, -then :keyword:`async for` clauses and :keyword:`await` expressions are permitted -as with an asynchronous comprehension. If a generator expression -contains either :keyword:`async for` clauses or :keyword:`await` expressions -it is called an :dfn:`asynchronous generator expression`. -An asynchronous generator expression yields a new asynchronous -generator object, which is an asynchronous iterator -(see :ref:`async-iterators`). - -.. _yieldexpr: - -Yield expressions ------------------ - -.. index:: - keyword: yield - keyword: from - pair: yield; expression - pair: generator; function - -.. productionlist:: - yield_atom: "(" `yield_expression` ")" - yield_expression: "yield" [`expression_list` | "from" `expression`] - -The yield expression is used when defining a :term:`generator` function -or an :term:`asynchronous generator` function and -thus can only be used in the body of a function definition. Using a yield -expression in a function's body causes that function to be a generator, -and using it in an :keyword:`async def` function's body causes that -coroutine function to be an asynchronous generator. For example:: - - def gen(): # defines a generator function - yield 123 - - async def agen(): # defines an asynchronous generator function - yield 123 - -Generator functions are described below, while asynchronous generator -functions are described separately in section -:ref:`asynchronous-generator-functions`. - -When a generator function is called, it returns an iterator known as a -generator. That generator then controls the execution of the generator function. -The execution starts when one of the generator's methods is called. At that -time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`expression_list` to the generator's -caller. By suspended, we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, the internal -evaluation stack, and the state of any exception handling. When the execution -is resumed by calling one of the -generator's methods, the function can proceed exactly as if the yield expression -were just another external call. The value of the yield expression after -resuming depends on the method which resumed the execution. If -:meth:`~generator.__next__` is used (typically via either a :keyword:`for` or -the :func:`next` builtin) then the result is :const:`None`. Otherwise, if -:meth:`~generator.send` is used, then the result will be the value passed in to -that method. - -.. index:: single: coroutine - -All of this makes generator functions quite similar to coroutines; they yield -multiple times, they have more than one entry point and their execution can be -suspended. The only difference is that a generator function cannot control -where the execution should continue after it yields; the control is always -transferred to the generator's caller. - -Yield expressions are allowed anywhere in a :keyword:`try` construct. If the -generator is not resumed before it is -finalized (by reaching a zero reference count or by being garbage collected), -the generator-iterator's :meth:`~generator.close` method will be called, -allowing any pending :keyword:`finally` clauses to execute. - -.. index:: - single: from; yield from expression - -When ``yield from `` is used, it treats the supplied expression as -a subiterator. All values produced by that subiterator are passed directly -to the caller of the current generator's methods. Any values passed in with -:meth:`~generator.send` and any exceptions passed in with -:meth:`~generator.throw` are passed to the underlying iterator if it has the -appropriate methods. If this is not the case, then :meth:`~generator.send` -will raise :exc:`AttributeError` or :exc:`TypeError`, while -:meth:`~generator.throw` will just raise the passed in exception immediately. - -When the underlying iterator is complete, the :attr:`~StopIteration.value` -attribute of the raised :exc:`StopIteration` instance becomes the value of -the yield expression. It can be either set explicitly when raising -:exc:`StopIteration`, or automatically when the sub-iterator is a generator -(by returning a value from the sub-generator). - - .. versionchanged:: 3.3 - Added ``yield from `` to delegate control flow to a subiterator. - -The parentheses may be omitted when the yield expression is the sole expression -on the right hand side of an assignment statement. - -.. seealso:: - - :pep:`255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. - - :pep:`342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. - - :pep:`380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. - - :pep:`525` - Asynchronous Generators - The proposal that expanded on :pep:`492` by adding generator capabilities to - coroutine functions. - -.. index:: object: generator -.. _generator-methods: - -Generator-iterator methods -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the methods of a generator iterator. They can -be used to control the execution of a generator function. - -Note that calling any of the generator methods below when the generator -is already executing raises a :exc:`ValueError` exception. - -.. index:: exception: StopIteration - - -.. method:: generator.__next__() - - Starts the execution of a generator function or resumes it at the last - executed yield expression. When a generator function is resumed with a - :meth:`~generator.__next__` method, the current yield expression always - evaluates to :const:`None`. The execution then continues to the next yield - expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`__next__`'s caller. If the - generator exits without yielding another value, a :exc:`StopIteration` - exception is raised. - - This method is normally called implicitly, e.g. by a :keyword:`for` loop, or - by the built-in :func:`next` function. - - -.. method:: generator.send(value) - - Resumes the execution and "sends" a value into the generator function. The - *value* argument becomes the result of the current yield expression. The - :meth:`send` method returns the next value yielded by the generator, or - raises :exc:`StopIteration` if the generator exits without yielding another - value. When :meth:`send` is called to start the generator, it must be called - with :const:`None` as the argument, because there is no yield expression that - could receive the value. - - -.. method:: generator.throw(type[, value[, traceback]]) - - Raises an exception of type ``type`` at the point where the generator was paused, - and returns the next value yielded by the generator function. If the generator - exits without yielding another value, a :exc:`StopIteration` exception is - raised. If the generator function does not catch the passed-in exception, or - raises a different exception, then that exception propagates to the caller. - -.. index:: exception: GeneratorExit - - -.. method:: generator.close() - - Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. - -.. index:: single: yield; examples - -Examples -^^^^^^^^ - -Here is a simple example that demonstrates the behavior of generators and -generator functions:: - - >>> def echo(value=None): - ... print("Execution starts when 'next()' is called for the first time.") - ... try: - ... while True: - ... try: - ... value = (yield value) - ... except Exception as e: - ... value = e - ... finally: - ... print("Don't forget to clean up when 'close()' is called.") - ... - >>> generator = echo(1) - >>> print(next(generator)) - Execution starts when 'next()' is called for the first time. - 1 - >>> print(next(generator)) - None - >>> print(generator.send(2)) - 2 - >>> generator.throw(TypeError, "spam") - TypeError('spam',) - >>> generator.close() - Don't forget to clean up when 'close()' is called. - -For examples using ``yield from``, see :ref:`pep-380` in "What's New in -Python." - -.. _asynchronous-generator-functions: - -Asynchronous generator functions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The presence of a yield expression in a function or method defined using -:keyword:`async def` further defines the function as a -:term:`asynchronous generator` function. - -When an asynchronous generator function is called, it returns an -asynchronous iterator known as an asynchronous generator object. -That object then controls the execution of the generator function. -An asynchronous generator object is typically used in an -:keyword:`async for` statement in a coroutine function analogously to -how a generator object would be used in a :keyword:`for` statement. - -Calling one of the asynchronous generator's methods returns an -:term:`awaitable` object, and the execution starts when this object -is awaited on. At that time, the execution proceeds to the first yield -expression, where it is suspended again, returning the value of -:token:`expression_list` to the awaiting coroutine. As with a generator, -suspension means that all local state is retained, including the -current bindings of local variables, the instruction pointer, the internal -evaluation stack, and the state of any exception handling. When the execution -is resumed by awaiting on the next object returned by the asynchronous -generator's methods, the function can proceed exactly as if the yield -expression were just another external call. The value of the yield expression -after resuming depends on the method which resumed the execution. If -:meth:`~agen.__anext__` is used then the result is :const:`None`. Otherwise, if -:meth:`~agen.asend` is used, then the result will be the value passed in to -that method. - -In an asynchronous generator function, yield expressions are allowed anywhere -in a :keyword:`try` construct. However, if an asynchronous generator is not -resumed before it is finalized (by reaching a zero reference count or by -being garbage collected), then a yield expression within a :keyword:`try` -construct could result in a failure to execute pending :keyword:`finally` -clauses. In this case, it is the responsibility of the event loop or -scheduler running the asynchronous generator to call the asynchronous -generator-iterator's :meth:`~agen.aclose` method and run the resulting -coroutine object, thus allowing any pending :keyword:`finally` clauses -to execute. - -To take care of finalization, an event loop should define -a *finalizer* function which takes an asynchronous generator-iterator -and presumably calls :meth:`~agen.aclose` and executes the coroutine. -This *finalizer* may be registered by calling :func:`sys.set_asyncgen_hooks`. -When first iterated over, an asynchronous generator-iterator will store the -registered *finalizer* to be called upon finalization. For a reference example -of a *finalizer* method see the implementation of -``asyncio.Loop.shutdown_asyncgens`` in :source:`Lib/asyncio/base_events.py`. - -The expression ``yield from `` is a syntax error when used in an -asynchronous generator function. - -.. index:: object: asynchronous-generator -.. _asynchronous-generator-methods: - -Asynchronous generator-iterator methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the methods of an asynchronous generator iterator, -which are used to control the execution of a generator function. - - -.. index:: exception: StopAsyncIteration - -.. coroutinemethod:: agen.__anext__() - - Returns an awaitable which when run starts to execute the asynchronous - generator or resumes it at the last executed yield expression. When an - asynchronous generator function is resumed with a :meth:`~agen.__anext__` - method, the current yield expression always evaluates to :const:`None` in - the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`expression_list` of the yield - expression is the value of the :exc:`StopIteration` exception raised by - the completing coroutine. If the asynchronous generator exits without - yielding another value, the awaitable instead raises an - :exc:`StopAsyncIteration` exception, signalling that the asynchronous - iteration has completed. - - This method is normally called implicitly by a :keyword:`async for` loop. - - -.. coroutinemethod:: agen.asend(value) - - Returns an awaitable which when run resumes the execution of the - asynchronous generator. As with the :meth:`~generator.send()` method for a - generator, this "sends" a value into the asynchronous generator function, - and the *value* argument becomes the result of the current yield expression. - The awaitable returned by the :meth:`asend` method will return the next - value yielded by the generator as the value of the raised - :exc:`StopIteration`, or raises :exc:`StopAsyncIteration` if the - asynchronous generator exits without yielding another value. When - :meth:`asend` is called to start the asynchronous - generator, it must be called with :const:`None` as the argument, - because there is no yield expression that could receive the value. - - -.. coroutinemethod:: agen.athrow(type[, value[, traceback]]) - - Returns an awaitable that raises an exception of type ``type`` at the point - where the asynchronous generator was paused, and returns the next value - yielded by the generator function as the value of the raised - :exc:`StopIteration` exception. If the asynchronous generator exits - without yielding another value, an :exc:`StopAsyncIteration` exception is - raised by the awaitable. - If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitable is run that exception - propagates to the caller of the awaitable. - -.. index:: exception: GeneratorExit - - -.. coroutinemethod:: agen.aclose() - - Returns an awaitable that when run will throw a :exc:`GeneratorExit` into - the asynchronous generator function at the point where it was paused. - If the asynchronous generator function then exits gracefully, is already - closed, or raises :exc:`GeneratorExit` (by not catching the exception), - then the returned awaitable will raise a :exc:`StopIteration` exception. - Any further awaitables returned by subsequent calls to the asynchronous - generator will raise a :exc:`StopAsyncIteration` exception. If the - asynchronous generator yields a value, a :exc:`RuntimeError` is raised - by the awaitable. If the asynchronous generator raises any other exception, - it is propagated to the caller of the awaitable. If the asynchronous - generator has already exited due to an exception or normal exit, then - further calls to :meth:`aclose` will return an awaitable that does nothing. - -.. _primaries: - -Primaries -========= - -.. index:: single: primary - -Primaries represent the most tightly bound operations of the language. Their -syntax is: - -.. productionlist:: - primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` - - -.. _attribute-references: - -Attribute references --------------------- - -.. index:: - pair: attribute; reference - single: . (dot); attribute reference - -An attribute reference is a primary followed by a period and a name: - -.. productionlist:: - attributeref: `primary` "." `identifier` - -.. index:: - exception: AttributeError - object: module - object: list - -The primary must evaluate to an object of a type that supports attribute -references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier. This production can be customized by -overriding the :meth:`__getattr__` method. If this attribute is not available, -the exception :exc:`AttributeError` is raised. Otherwise, the type and value of -the object produced is determined by the object. Multiple evaluations of the -same attribute reference may yield different objects. - - -.. _subscriptions: - -Subscriptions -------------- - -.. index:: - single: subscription - single: [] (square brackets); subscription - -.. index:: - object: sequence - object: mapping - object: string - object: tuple - object: list - object: dictionary - pair: sequence; item - -A subscription selects an item of a sequence (string, tuple or list) or mapping -(dictionary) object: - -.. productionlist:: - subscription: `primary` "[" `expression_list` "]" - -The primary must evaluate to an object that supports subscription (lists or -dictionaries for example). User-defined objects can support subscription by -defining a :meth:`__getitem__` method. - -For built-in objects, there are two types of objects that support subscription: - -If the primary is a mapping, the expression list must evaluate to an object -whose value is one of the keys of the mapping, and the subscription selects the -value in the mapping that corresponds to that key. (The expression list is a -tuple except if it has exactly one item.) - -If the primary is a sequence, the expression list must evaluate to an integer -or a slice (as discussed in the following section). - -The formal syntax makes no special provision for negative indices in -sequences; however, built-in sequences all provide a :meth:`__getitem__` -method that interprets negative indices by adding the length of the sequence -to the index (so that ``x[-1]`` selects the last item of ``x``). The -resulting value must be a nonnegative integer less than the number of items in -the sequence, and the subscription selects the item whose index is that value -(counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`__getitem__` method, subclasses overriding -this method will need to explicitly add that support. - -.. index:: - single: character - pair: string; item - -A string's items are characters. A character is not a separate data type but a -string of exactly one character. - - -.. _slicings: - -Slicings --------- - -.. index:: - single: slicing - single: slice - single: : (colon); slicing - single: , (comma); slicing - -.. index:: - object: sequence - object: string - object: tuple - object: list - -A slicing selects a range of items in a sequence object (e.g., a string, tuple -or list). Slicings may be used as expressions or as targets in assignment or -:keyword:`del` statements. The syntax for a slicing: - -.. productionlist:: - slicing: `primary` "[" `slice_list` "]" - slice_list: `slice_item` ("," `slice_item`)* [","] - slice_item: `expression` | `proper_slice` - proper_slice: [`lower_bound`] ":" [`upper_bound`] [ ":" [`stride`] ] - lower_bound: `expression` - upper_bound: `expression` - stride: `expression` - -There is ambiguity in the formal syntax here: anything that looks like an -expression list also looks like a slice list, so any subscription can be -interpreted as a slicing. Rather than further complicating the syntax, this is -disambiguated by defining that in this case the interpretation as a subscription -takes priority over the interpretation as a slicing (this is the case if the -slice list contains no proper slice). - -.. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - -The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`__getitem__` method as -normal subscription) with a key that is constructed from the slice list, as -follows. If the slice list contains at least one comma, the key is a tuple -containing the conversion of the slice items; otherwise, the conversion of the -lone slice item is the key. The conversion of a slice item that is an -expression is that expression. The conversion of a proper slice is a slice -object (see section :ref:`types`) whose :attr:`~slice.start`, -:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the -expressions given as lower bound, upper bound and stride, respectively, -substituting ``None`` for missing expressions. - - -.. index:: - object: callable - single: call - single: argument; call semantics - single: () (parentheses); call - single: , (comma); argument list - single: = (equals); in function calls - -.. _calls: - -Calls ------ - -A call calls a callable object (e.g., a :term:`function`) with a possibly empty -series of :term:`arguments `: - -.. productionlist:: - call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" - argument_list: `positional_arguments` ["," `starred_and_keywords`] - : ["," `keywords_arguments`] - : | `starred_and_keywords` ["," `keywords_arguments`] - : | `keywords_arguments` - positional_arguments: ["*"] `expression` ("," ["*"] `expression`)* - starred_and_keywords: ("*" `expression` | `keyword_item`) - : ("," "*" `expression` | "," `keyword_item`)* - keywords_arguments: (`keyword_item` | "**" `expression`) - : ("," `keyword_item` | "," "**" `expression`)* - keyword_item: `identifier` "=" `expression` - -An optional trailing comma may be present after the positional and keyword arguments -but does not affect the semantics. - -.. index:: - single: parameter; call semantics - -The primary must evaluate to a callable object (user-defined functions, built-in -functions, methods of built-in objects, class objects, methods of class -instances, and all objects having a :meth:`__call__` method are callable). All -argument expressions are evaluated before the call is attempted. Please refer -to section :ref:`function` for the syntax of formal :term:`parameter` lists. - -.. XXX update with kwonly args PEP - -If keyword arguments are present, they are first converted to positional -arguments, as follows. First, a list of unfilled slots is created for the -formal parameters. If there are N positional arguments, they are placed in the -first N slots. Next, for each keyword argument, the identifier is used to -determine the corresponding slot (if the identifier is the same as the first -formal parameter name, the first slot is used, and so on). If the slot is -already filled, a :exc:`TypeError` exception is raised. Otherwise, the value of -the argument is placed in the slot, filling it (even if the expression is -``None``, it fills the slot). When all arguments have been processed, the slots -that are still unfilled are filled with the corresponding default value from the -function definition. (Default values are calculated, once, when the function is -defined; thus, a mutable object such as a list or dictionary used as default -value will be shared by all calls that don't specify an argument value for the -corresponding slot; this should usually be avoided.) If there are any unfilled -slots for which no default value is specified, a :exc:`TypeError` exception is -raised. Otherwise, the list of filled slots is used as the argument list for -the call. - -.. impl-detail:: - - An implementation may provide built-in functions whose positional parameters - do not have names, even if they are 'named' for the purpose of documentation, - and which therefore cannot be supplied by keyword. In CPython, this is the - case for functions implemented in C that use :c:func:`PyArg_ParseTuple` to - parse their arguments. - -If there are more positional arguments than there are formal parameter slots, a -:exc:`TypeError` exception is raised, unless a formal parameter using the syntax -``*identifier`` is present; in this case, that formal parameter receives a tuple -containing the excess positional arguments (or an empty tuple if there were no -excess positional arguments). - -If any keyword argument does not correspond to a formal parameter name, a -:exc:`TypeError` exception is raised, unless a formal parameter using the syntax -``**identifier`` is present; in this case, that formal parameter receives a -dictionary containing the excess keyword arguments (using the keywords as keys -and the argument values as corresponding values), or a (new) empty dictionary if -there were no excess keyword arguments. - -.. index:: - single: * (asterisk); in function calls - single: unpacking; in function calls - -If the syntax ``*expression`` appears in the function call, ``expression`` must -evaluate to an :term:`iterable`. Elements from these iterables are -treated as if they were additional positional arguments. For the call -``f(x1, x2, *y, x3, x4)``, if *y* evaluates to a sequence *y1*, ..., *yM*, -this is equivalent to a call with M+4 positional arguments *x1*, *x2*, -*y1*, ..., *yM*, *x3*, *x4*. - -A consequence of this is that although the ``*expression`` syntax may appear -*after* explicit keyword arguments, it is processed *before* the -keyword arguments (and any ``**expression`` arguments -- see below). So:: - - >>> def f(a, b): - ... print(a, b) - ... - >>> f(b=1, *(2,)) - 2 1 - >>> f(a=1, *(2,)) - Traceback (most recent call last): - File "", line 1, in - TypeError: f() got multiple values for keyword argument 'a' - >>> f(1, *(2,)) - 1 2 - -It is unusual for both keyword arguments and the ``*expression`` syntax to be -used in the same call, so in practice this confusion does not arise. - -.. index:: - single: **; in function calls - -If the syntax ``**expression`` appears in the function call, ``expression`` must -evaluate to a :term:`mapping`, the contents of which are treated as -additional keyword arguments. If a keyword is already present -(as an explicit keyword argument, or from another unpacking), -a :exc:`TypeError` exception is raised. - -Formal parameters using the syntax ``*identifier`` or ``**identifier`` cannot be -used as positional argument slots or as keyword argument names. - -.. versionchanged:: 3.5 - Function calls accept any number of ``*`` and ``**`` unpackings, - positional arguments may follow iterable unpackings (``*``), - and keyword arguments may follow dictionary unpackings (``**``). - Originally proposed by :pep:`448`. - -A call always returns some value, possibly ``None``, unless it raises an -exception. How this value is computed depends on the type of the callable -object. - -If it is--- - -a user-defined function: - .. index:: - pair: function; call - triple: user-defined; function; call - object: user-defined function - object: function - - The code block for the function is executed, passing it the argument list. The - first thing the code block will do is bind the formal parameters to the - arguments; this is described in section :ref:`function`. When the code block - executes a :keyword:`return` statement, this specifies the return value of the - function call. - -a built-in function or method: - .. index:: - pair: function; call - pair: built-in function; call - pair: method; call - pair: built-in method; call - object: built-in method - object: built-in function - object: method - object: function - - The result is up to the interpreter; see :ref:`built-in-funcs` for the - descriptions of built-in functions and methods. - -a class object: - .. index:: - object: class - pair: class object; call - - A new instance of that class is returned. - -a class instance method: - .. index:: - object: class instance - object: instance - pair: class instance; call - - The corresponding user-defined function is called, with an argument list that is - one longer than the argument list of the call: the instance becomes the first - argument. - -a class instance: - .. index:: - pair: instance; call - single: __call__() (object method) - - The class must define a :meth:`__call__` method; the effect is then the same as - if that method was called. - - -.. index:: keyword: await -.. _await: - -Await expression -================ - -Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. -Can only be used inside a :term:`coroutine function`. - -.. productionlist:: - await_expr: "await" `primary` - -.. versionadded:: 3.5 - - -.. _power: - -The power operator -================== - -.. index:: - pair: power; operation - operator: ** - -The power operator binds more tightly than unary operators on its left; it binds -less tightly than unary operators on its right. The syntax is: - -.. productionlist:: - power: (`await_expr` | `primary`) ["**" `u_expr`] - -Thus, in an unparenthesized sequence of power and unary operators, the operators -are evaluated from right to left (this does not constrain the evaluation order -for the operands): ``-1**2`` results in ``-1``. - -The power operator has the same semantics as the built-in :func:`pow` function, -when called with two arguments: it yields its left argument raised to the power -of its right argument. The numeric arguments are first converted to a common -type, and the result is of that type. - -For int operands, the result has the same type as the operands unless the second -argument is negative; in that case, all arguments are converted to float and a -float result is delivered. For example, ``10**2`` returns ``100``, but -``10**-2`` returns ``0.01``. - -Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. -Raising a negative number to a fractional power results in a :class:`complex` -number. (In earlier versions it raised a :exc:`ValueError`.) - - -.. _unary: - -Unary arithmetic and bitwise operations -======================================= - -.. index:: - triple: unary; arithmetic; operation - triple: unary; bitwise; operation - -All unary arithmetic and bitwise operations have the same priority: - -.. productionlist:: - u_expr: `power` | "-" `u_expr` | "+" `u_expr` | "~" `u_expr` - -.. index:: - single: negation - single: minus - single: operator; - (minus) - single: - (minus); unary operator - -The unary ``-`` (minus) operator yields the negation of its numeric argument. - -.. index:: - single: plus - single: operator; + (plus) - single: + (plus); unary operator - -The unary ``+`` (plus) operator yields its numeric argument unchanged. - -.. index:: - single: inversion - operator: ~ (tilde) - -The unary ``~`` (invert) operator yields the bitwise inversion of its integer -argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only -applies to integral numbers. - -.. index:: exception: TypeError - -In all three cases, if the argument does not have the proper type, a -:exc:`TypeError` exception is raised. - - -.. _binary: - -Binary arithmetic operations -============================ - -.. index:: triple: binary; arithmetic; operation - -The binary arithmetic operations have the conventional priority levels. Note -that some of these operations also apply to certain non-numeric types. Apart -from the power operator, there are only two levels, one for multiplicative -operators and one for additive operators: - -.. productionlist:: - m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | - : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | - : `m_expr` "%" `u_expr` - a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr` - -.. index:: - single: multiplication - operator: * (asterisk) - -The ``*`` (multiplication) operator yields the product of its arguments. The -arguments must either both be numbers, or one argument must be an integer and -the other must be a sequence. In the former case, the numbers are converted to a -common type and then multiplied together. In the latter case, sequence -repetition is performed; a negative repetition factor yields an empty sequence. - -.. index:: - single: matrix multiplication - operator: @ (at) - -The ``@`` (at) operator is intended to be used for matrix multiplication. No -builtin Python types implement this operator. - -.. versionadded:: 3.5 - -.. index:: - exception: ZeroDivisionError - single: division - operator: / (slash) - operator: // - -The ``/`` (division) and ``//`` (floor division) operators yield the quotient of -their arguments. The numeric arguments are first converted to a common type. -Division of integers yields a float, while floor division of integers results in an -integer; the result is that of mathematical division with the 'floor' function -applied to the result. Division by zero raises the :exc:`ZeroDivisionError` -exception. - -.. index:: - single: modulo - operator: % (percent) - -The ``%`` (modulo) operator yields the remainder from the division of the first -argument by the second. The numeric arguments are first converted to a common -type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The -arguments may be floating point numbers, e.g., ``3.14%0.7`` equals ``0.34`` -(since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a -result with the same sign as its second operand (or zero); the absolute value of -the result is strictly smaller than the absolute value of the second operand -[#]_. - -The floor division and modulo operators are connected by the following -identity: ``x == (x//y)*y + (x%y)``. Floor division and modulo are also -connected with the built-in function :func:`divmod`: ``divmod(x, y) == (x//y, -x%y)``. [#]_. - -In addition to performing the modulo operation on numbers, the ``%`` operator is -also overloaded by string objects to perform old-style string formatting (also -known as interpolation). The syntax for string formatting is described in the -Python Library Reference, section :ref:`old-string-formatting`. - -The floor division operator, the modulo operator, and the :func:`divmod` -function are not defined for complex numbers. Instead, convert to a floating -point number using the :func:`abs` function if appropriate. - -.. index:: - single: addition - single: operator; + (plus) - single: + (plus); binary operator - -The ``+`` (addition) operator yields the sum of its arguments. The arguments -must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common type and then added together. -In the latter case, the sequences are concatenated. - -.. index:: - single: subtraction - single: operator; - (minus) - single: - (minus); binary operator - -The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common type. - - -.. _shifting: - -Shifting operations -=================== - -.. index:: - pair: shifting; operation - operator: << - operator: >> - -The shifting operations have lower priority than the arithmetic operations: - -.. productionlist:: - shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` - -These operators accept integers as arguments. They shift the first argument to -the left or right by the number of bits given by the second argument. - -.. index:: exception: ValueError - -A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left -shift by *n* bits is defined as multiplication with ``pow(2,n)``. - -.. note:: - - In the current implementation, the right-hand operand is required - to be at most :attr:`sys.maxsize`. If the right-hand operand is larger than - :attr:`sys.maxsize` an :exc:`OverflowError` exception is raised. - -.. _bitwise: - -Binary bitwise operations -========================= - -.. index:: triple: binary; bitwise; operation - -Each of the three bitwise operations has a different priority level: - -.. productionlist:: - and_expr: `shift_expr` | `and_expr` "&" `shift_expr` - xor_expr: `and_expr` | `xor_expr` "^" `and_expr` - or_expr: `xor_expr` | `or_expr` "|" `xor_expr` - -.. index:: - pair: bitwise; and - operator: & (ampersand) - -The ``&`` operator yields the bitwise AND of its arguments, which must be -integers. - -.. index:: - pair: bitwise; xor - pair: exclusive; or - operator: ^ (caret) - -The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which -must be integers. - -.. index:: - pair: bitwise; or - pair: inclusive; or - operator: | (vertical bar) - -The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which -must be integers. - - -.. _comparisons: - -Comparisons -=========== - -.. index:: - single: comparison - pair: C; language - operator: < (less) - operator: > (greater) - operator: <= - operator: >= - operator: == - operator: != - -Unlike C, all comparison operations in Python have the same priority, which is -lower than that of any arithmetic, shifting or bitwise operation. Also unlike -C, expressions like ``a < b < c`` have the interpretation that is conventional -in mathematics: - -.. productionlist:: - comparison: `or_expr` (`comp_operator` `or_expr`)* - comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" - : | "is" ["not"] | ["not"] "in" - -Comparisons yield boolean values: ``True`` or ``False``. - -.. index:: pair: chaining; comparisons - -Comparisons can be chained arbitrarily, e.g., ``x < y <= z`` is equivalent to -``x < y and y <= z``, except that ``y`` is evaluated only once (but in both -cases ``z`` is not evaluated at all when ``x < y`` is found to be false). - -Formally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*, *op2*, ..., -*opN* are comparison operators, then ``a op1 b op2 c ... y opN z`` is equivalent -to ``a op1 b and b op2 c and ... y opN z``, except that each expression is -evaluated at most once. - -Note that ``a op1 b op2 c`` doesn't imply any kind of comparison between *a* and -*c*, so that, e.g., ``x < y > z`` is perfectly legal (though perhaps not -pretty). - -Value comparisons ------------------ - -The operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare the -values of two objects. The objects do not need to have the same type. - -Chapter :ref:`objects` states that objects have a value (in addition to type -and identity). The value of an object is a rather abstract notion in Python: -For example, there is no canonical access method for an object's value. Also, -there is no requirement that the value of an object should be constructed in a -particular way, e.g. comprised of all its data attributes. Comparison operators -implement a particular notion of what the value of an object is. One can think -of them as defining the value of an object indirectly, by means of their -comparison implementation. - -Because all types are (direct or indirect) subtypes of :class:`object`, they -inherit the default comparison behavior from :class:`object`. Types can -customize their comparison behavior by implementing -:dfn:`rich comparison methods` like :meth:`__lt__`, described in -:ref:`customization`. - -The default behavior for equality comparison (``==`` and ``!=``) is based on -the identity of the objects. Hence, equality comparison of instances with the -same identity results in equality, and equality comparison of instances with -different identities results in inequality. A motivation for this default -behavior is the desire that all objects should be reflexive (i.e. ``x is y`` -implies ``x == y``). - -A default order comparison (``<``, ``>``, ``<=``, and ``>=``) is not provided; -an attempt raises :exc:`TypeError`. A motivation for this default behavior is -the lack of a similar invariant as for equality. - -The behavior of the default equality comparison, that instances with different -identities are always unequal, may be in contrast to what types will need that -have a sensible definition of object value and value-based equality. Such -types will need to customize their comparison behavior, and in fact, a number -of built-in types have done that. - -The following list describes the comparison behavior of the most important -built-in types. - -* Numbers of built-in numeric types (:ref:`typesnumeric`) and of the standard - library types :class:`fractions.Fraction` and :class:`decimal.Decimal` can be - compared within and across their types, with the restriction that complex - numbers do not support order comparison. Within the limits of the types - involved, they compare mathematically (algorithmically) correct without loss - of precision. - - The not-a-number values :const:`float('NaN')` and :const:`Decimal('NaN')` - are special. They are identical to themselves (``x is x`` is true) but - are not equal to themselves (``x == x`` is false). Additionally, - comparing any number to a not-a-number value - will return ``False``. For example, both ``3 < float('NaN')`` and - ``float('NaN') < 3`` will return ``False``. - -* Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be - compared within and across their types. They compare lexicographically using - the numeric values of their elements. - -* Strings (instances of :class:`str`) compare lexicographically using the - numerical Unicode code points (the result of the built-in function - :func:`ord`) of their characters. [#]_ - - Strings and binary sequences cannot be directly compared. - -* Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can - be compared only within each of their types, with the restriction that ranges - do not support order comparison. Equality comparison across these types - results in inequality, and ordering comparison across these types raises - :exc:`TypeError`. - - Sequences compare lexicographically using comparison of corresponding - elements, whereby reflexivity of the elements is enforced. - - In enforcing reflexivity of elements, the comparison of collections assumes - that for a collection element ``x``, ``x == x`` is always true. Based on - that assumption, element identity is compared first, and element comparison - is performed only for distinct elements. This approach yields the same - result as a strict element comparison would, if the compared elements are - reflexive. For non-reflexive elements, the result is different than for - strict element comparison, and may be surprising: The non-reflexive - not-a-number values for example result in the following comparison behavior - when used in a list:: - - >>> nan = float('NaN') - >>> nan is nan - True - >>> nan == nan - False <-- the defined non-reflexive behavior of NaN - >>> [nan] == [nan] - True <-- list enforces reflexivity and tests identity first - - Lexicographical comparison between built-in collections works as follows: - - - For two collections to compare equal, they must be of the same type, have - the same length, and each pair of corresponding elements must compare - equal (for example, ``[1,2] == (1,2)`` is false because the type is not the - same). - - - Collections that support order comparison are ordered the same as their - first unequal elements (for example, ``[1,2,x] <= [1,2,y]`` has the same - value as ``x <= y``). If a corresponding element does not exist, the - shorter collection is ordered first (for example, ``[1,2] < [1,2,3]`` is - true). - -* Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and values - enforces reflexivity. - - Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. - -* Sets (instances of :class:`set` or :class:`frozenset`) can be compared within - and across their types. - - They define order - comparison operators to mean subset and superset tests. Those relations do - not define total orderings (for example, the two sets ``{1,2}`` and ``{2,3}`` - are not equal, nor subsets of one another, nor supersets of one - another). Accordingly, sets are not appropriate arguments for functions - which depend on total ordering (for example, :func:`min`, :func:`max`, and - :func:`sorted` produce undefined results given a list of sets as inputs). - - Comparison of sets enforces reflexivity of its elements. - -* Most other built-in types have no comparison methods implemented, so they - inherit the default comparison behavior. - -User-defined classes that customize their comparison behavior should follow -some consistency rules, if possible: - -* Equality comparison should be reflexive. - In other words, identical objects should compare equal: - - ``x is y`` implies ``x == y`` - -* Comparison should be symmetric. - In other words, the following expressions should have the same result: - - ``x == y`` and ``y == x`` - - ``x != y`` and ``y != x`` - - ``x < y`` and ``y > x`` - - ``x <= y`` and ``y >= x`` - -* Comparison should be transitive. - The following (non-exhaustive) examples illustrate that: - - ``x > y and y > z`` implies ``x > z`` - - ``x < y and y <= z`` implies ``x < z`` - -* Inverse comparison should result in the boolean negation. - In other words, the following expressions should have the same result: - - ``x == y`` and ``not x != y`` - - ``x < y`` and ``not x >= y`` (for total ordering) - - ``x > y`` and ``not x <= y`` (for total ordering) - - The last two expressions apply to totally ordered collections (e.g. to - sequences, but not to sets or mappings). See also the - :func:`~functools.total_ordering` decorator. - -* The :func:`hash` result should be consistent with equality. - Objects that are equal should either have the same hash value, - or be marked as unhashable. - -Python does not enforce these consistency rules. In fact, the not-a-number -values are an example for not following these rules. - - -.. _in: -.. _not in: -.. _membership-test-details: - -Membership test operations --------------------------- - -The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. -``x not in s`` returns the negation of ``x in s``. All built-in sequences and -set types support this as well as dictionary, for which :keyword:`in` tests -whether the dictionary has a given key. For container types such as list, tuple, -set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent -to ``any(x is e or x == e for e in y)``. - -For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a -substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are -always considered to be a substring of any other string, so ``"" in "abc"`` will -return ``True``. - -For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and -``False`` otherwise. - -For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z`` with ``x == z`` is -produced while iterating over ``y``. If an exception is raised during the -iteration, it is as if :keyword:`in` raised that exception. - -Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x == y[i]``, and all lower integer indices do not -raise :exc:`IndexError` exception. (If any other exception is raised, it is as -if :keyword:`in` raised that exception). - -.. index:: - operator: in - operator: not in - pair: membership; test - object: sequence - -The operator :keyword:`not in` is defined to have the inverse true value of -:keyword:`in`. - -.. index:: - operator: is - operator: is not - pair: identity; test - - -.. _is: -.. _is not: - -Identity comparisons --------------------- - -The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x -is y`` is true if and only if *x* and *y* are the same object. Object identity -is determined using the :meth:`id` function. ``x is not y`` yields the inverse -truth value. [#]_ - - -.. _booleans: -.. _and: -.. _or: -.. _not: - -Boolean operations -================== - -.. index:: - pair: Conditional; expression - pair: Boolean; operation - -.. productionlist:: - or_test: `and_test` | `or_test` "or" `and_test` - and_test: `not_test` | `and_test` "and" `not_test` - not_test: `comparison` | "not" `not_test` - -In the context of Boolean operations, and also when expressions are used by -control flow statements, the following values are interpreted as false: -``False``, ``None``, numeric zero of all types, and empty strings and containers -(including strings, tuples, lists, dictionaries, sets and frozensets). All -other values are interpreted as true. User-defined objects can customize their -truth value by providing a :meth:`__bool__` method. - -.. index:: operator: not - -The operator :keyword:`not` yields ``True`` if its argument is false, ``False`` -otherwise. - -.. index:: operator: and - -The expression ``x and y`` first evaluates *x*; if *x* is false, its value is -returned; otherwise, *y* is evaluated and the resulting value is returned. - -.. index:: operator: or - -The expression ``x or y`` first evaluates *x*; if *x* is true, its value is -returned; otherwise, *y* is evaluated and the resulting value is returned. - -Note that neither :keyword:`and` nor :keyword:`or` restrict the value and type -they return to ``False`` and ``True``, but rather return the last evaluated -argument. This is sometimes useful, e.g., if ``s`` is a string that should be -replaced by a default value if it is empty, the expression ``s or 'foo'`` yields -the desired value. Because :keyword:`not` has to create a new value, it -returns a boolean value regardless of the type of its argument -(for example, ``not 'foo'`` produces ``False`` rather than ``''``.) - - -Conditional expressions -======================= - -.. index:: - pair: conditional; expression - pair: ternary; operator - single: if; conditional expression - single: else; conditional expression - -.. productionlist:: - conditional_expression: `or_test` ["if" `or_test` "else" `expression`] - expression: `conditional_expression` | `lambda_expr` - expression_nocond: `or_test` | `lambda_expr_nocond` - -Conditional expressions (sometimes called a "ternary operator") have the lowest -priority of all Python operations. - -The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. -If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is -evaluated and its value is returned. - -See :pep:`308` for more details about conditional expressions. - - -.. _lambdas: -.. _lambda: - -Lambdas -======= - -.. index:: - pair: lambda; expression - pair: lambda; form - pair: anonymous; function - single: : (colon); lambda expression - -.. productionlist:: - lambda_expr: "lambda" [`parameter_list`] ":" `expression` - lambda_expr_nocond: "lambda" [`parameter_list`] ":" `expression_nocond` - -Lambda expressions (sometimes called lambda forms) are used to create anonymous -functions. The expression ``lambda parameters: expression`` yields a function -object. The unnamed object behaves like a function object defined with: - -.. code-block:: none - - def (parameters): - return expression - -See section :ref:`function` for the syntax of parameter lists. Note that -functions created with lambda expressions cannot contain statements or -annotations. - - -.. _exprlists: - -Expression lists -================ - -.. index:: - pair: expression; list - single: , (comma); expression list - -.. productionlist:: - expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `expression` | "*" `or_expr` - -.. index:: object: tuple - -Except when part of a list or set display, an expression list -containing at least one comma yields a tuple. The length of -the tuple is the number of expressions in the list. The expressions are -evaluated from left to right. - -.. index:: - pair: iterable; unpacking - single: * (asterisk); in expression lists - -An asterisk ``*`` denotes :dfn:`iterable unpacking`. Its operand must be -an :term:`iterable`. The iterable is expanded into a sequence of items, -which are included in the new tuple, list, or set, at the site of -the unpacking. - -.. versionadded:: 3.5 - Iterable unpacking in expression lists, originally proposed by :pep:`448`. - -.. index:: pair: trailing; comma - -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a -trailing comma doesn't create a tuple, but rather yields the value of that -expression. (To create an empty tuple, use an empty pair of parentheses: -``()``.) - - -.. _evalorder: - -Evaluation order -================ - -.. index:: pair: evaluation; order - -Python evaluates expressions from left to right. Notice that while evaluating -an assignment, the right-hand side is evaluated before the left-hand side. - -In the following lines, expressions will be evaluated in the arithmetic order of -their suffixes:: - - expr1, expr2, expr3, expr4 - (expr1, expr2, expr3, expr4) - {expr1: expr2, expr3: expr4} - expr1 + expr2 * (expr3 - expr4) - expr1(expr2, expr3, *expr4, **expr5) - expr3, expr4 = expr1, expr2 - - -.. _operator-summary: - -Operator precedence -=================== - -.. index:: - pair: operator; precedence - -The following table summarizes the operator precedence in Python, from lowest -precedence (least binding) to highest precedence (most binding). Operators in -the same box have the same precedence. Unless the syntax is explicitly given, -operators are binary. Operators in the same box group left to right (except for -exponentiation, which groups from right to left). - -Note that comparisons, membership tests, and identity tests, all have the same -precedence and have a left-to-right chaining feature as described in the -:ref:`comparisons` section. - - -+-----------------------------------------------+-------------------------------------+ -| Operator | Description | -+===============================================+=====================================+ -| :keyword:`lambda` | Lambda expression | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`if` -- :keyword:`else` | Conditional expression | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`or` | Boolean OR | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`and` | Boolean AND | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`not` ``x`` | Boolean NOT | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | -| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | -| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | -+-----------------------------------------------+-------------------------------------+ -| ``|`` | Bitwise OR | -+-----------------------------------------------+-------------------------------------+ -| ``^`` | Bitwise XOR | -+-----------------------------------------------+-------------------------------------+ -| ``&`` | Bitwise AND | -+-----------------------------------------------+-------------------------------------+ -| ``<<``, ``>>`` | Shifts | -+-----------------------------------------------+-------------------------------------+ -| ``+``, ``-`` | Addition and subtraction | -+-----------------------------------------------+-------------------------------------+ -| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication, division, floor | -| | division, remainder [#]_ | -+-----------------------------------------------+-------------------------------------+ -| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | -+-----------------------------------------------+-------------------------------------+ -| ``**`` | Exponentiation [#]_ | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`await` ``x`` | Await expression | -+-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | -| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | -+-----------------------------------------------+-------------------------------------+ -| ``(expressions...)``, | Binding or tuple display, | -| ``[expressions...]``, | list display, | -| ``{key: value...}``, | dictionary display, | -| ``{expressions...}`` | set display | -+-----------------------------------------------+-------------------------------------+ - - -.. rubric:: Footnotes - -.. [#] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it may not be - true numerically due to roundoff. For example, and assuming a platform on which - a Python float is an IEEE 754 double-precision number, in order that ``-1e-100 % - 1e100`` have the same sign as ``1e100``, the computed result is ``-1e-100 + - 1e100``, which is numerically exactly equal to ``1e100``. The function - :func:`math.fmod` returns a result whose sign matches the sign of the - first argument instead, and so returns ``-1e-100`` in this case. Which approach - is more appropriate depends on the application. - -.. [#] If x is very close to an exact integer multiple of y, it's possible for - ``x//y`` to be one larger than ``(x-x%y)//y`` due to rounding. In such - cases, Python returns the latter result, in order to preserve that - ``divmod(x,y)[0] * y + x % y`` be very close to ``x``. - -.. [#] The Unicode standard distinguishes between :dfn:`code points` - (e.g. U+0041) and :dfn:`abstract characters` (e.g. "LATIN CAPITAL LETTER A"). - While most abstract characters in Unicode are only represented using one - code point, there is a number of abstract characters that can in addition be - represented using a sequence of more than one code point. For example, the - abstract character "LATIN CAPITAL LETTER C WITH CEDILLA" can be represented - as a single :dfn:`precomposed character` at code position U+00C7, or as a - sequence of a :dfn:`base character` at code position U+0043 (LATIN CAPITAL - LETTER C), followed by a :dfn:`combining character` at code position U+0327 - (COMBINING CEDILLA). - - The comparison operators on strings compare at the level of Unicode code - points. This may be counter-intuitive to humans. For example, - ``"\u00C7" == "\u0043\u0327"`` is ``False``, even though both strings - represent the same abstract character "LATIN CAPITAL LETTER C WITH CEDILLA". - - To compare strings at the level of abstract characters (that is, in a way - intuitive to humans), use :func:`unicodedata.normalize`. - -.. [#] Due to automatic garbage-collection, free lists, and the dynamic nature of - descriptors, you may notice seemingly unusual behaviour in certain uses of - the :keyword:`is` operator, like those involving comparisons between instance - methods, or constants. Check their documentation for more info. - -.. [#] The ``%`` operator is also used for string formatting; the same - precedence applies. - -.. [#] The power operator ``**`` binds less tightly than an arithmetic or - bitwise unary operator on its right, that is, ``2**-1`` is ``0.5``. diff --git a/third_party/python/Doc/reference/grammar.rst b/third_party/python/Doc/reference/grammar.rst deleted file mode 100644 index 83d0f8532..000000000 --- a/third_party/python/Doc/reference/grammar.rst +++ /dev/null @@ -1,7 +0,0 @@ -Full Grammar specification -========================== - -This is the full Python grammar, as it is read by the parser generator and used -to parse Python source files: - -.. literalinclude:: ../../Grammar/Grammar diff --git a/third_party/python/Doc/reference/import.rst b/third_party/python/Doc/reference/import.rst deleted file mode 100644 index b295eed9d..000000000 --- a/third_party/python/Doc/reference/import.rst +++ /dev/null @@ -1,1007 +0,0 @@ - -.. _importsystem: - -***************** -The import system -***************** - -.. index:: single: import machinery - -Python code in one :term:`module` gains access to the code in another module -by the process of :term:`importing` it. The :keyword:`import` statement is -the most common way of invoking the import machinery, but it is not the only -way. Functions such as :func:`importlib.import_module` and built-in -:func:`__import__` can also be used to invoke the import machinery. - -The :keyword:`import` statement combines two operations; it searches for the -named module, then it binds the results of that search to a name in the local -scope. The search operation of the :keyword:`import` statement is defined as -a call to the :func:`__import__` function, with the appropriate arguments. -The return value of :func:`__import__` is used to perform the name -binding operation of the :keyword:`import` statement. See the -:keyword:`import` statement for the exact details of that name binding -operation. - -A direct call to :func:`__import__` performs only the module search and, if -found, the module creation operation. While certain side-effects may occur, -such as the importing of parent packages, and the updating of various caches -(including :data:`sys.modules`), only the :keyword:`import` statement performs -a name binding operation. - -When calling :func:`__import__` as part of an import statement, the -standard builtin :func:`__import__` is called. Other mechanisms for -invoking the import system (such as :func:`importlib.import_module`) may -choose to subvert :func:`__import__` and use its own solution to -implement import semantics. - -When a module is first imported, Python searches for the module and if found, -it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various -strategies to search for the named module when the import machinery is -invoked. These strategies can be modified and extended by using various hooks -described in the sections below. - -.. versionchanged:: 3.3 - The import system has been updated to fully implement the second phase - of :pep:`302`. There is no longer any implicit import machinery - the full - import system is exposed through :data:`sys.meta_path`. In addition, - native namespace package support has been implemented (see :pep:`420`). - - -:mod:`importlib` -================ - -The :mod:`importlib` module provides a rich API for interacting with the -import system. For example :func:`importlib.import_module` provides a -recommended, simpler API than built-in :func:`__import__` for invoking the -import machinery. Refer to the :mod:`importlib` library documentation for -additional detail. - - - -Packages -======== - -.. index:: - single: package - -Python has only one type of module object, and all modules are of this type, -regardless of whether the module is implemented in Python, C, or something -else. To help organize modules and provide a naming hierarchy, Python has a -concept of :term:`packages `. - -You can think of packages as the directories on a file system and modules as -files within directories, but don't take this analogy too literally since -packages and modules need not originate from the file system. For the -purposes of this documentation, we'll use this convenient analogy of -directories and files. Like file system directories, packages are organized -hierarchically, and packages may themselves contain subpackages, as well as -regular modules. - -It's important to keep in mind that all packages are modules, but not all -modules are packages. Or put another way, packages are just a special kind of -module. Specifically, any module that contains a ``__path__`` attribute is -considered a package. - -All modules have a name. Subpackage names are separated from their parent -package name by dots, akin to Python's standard attribute access syntax. Thus -you might have a module called :mod:`sys` and a package called :mod:`email`, -which in turn has a subpackage called :mod:`email.mime` and a module within -that subpackage called :mod:`email.mime.text`. - - -Regular packages ----------------- - -.. index:: - pair: package; regular - -Python defines two types of packages, :term:`regular packages ` and :term:`namespace packages `. Regular -packages are traditional packages as they existed in Python 3.2 and earlier. -A regular package is typically implemented as a directory containing an -``__init__.py`` file. When a regular package is imported, this -``__init__.py`` file is implicitly executed, and the objects it defines are -bound to names in the package's namespace. The ``__init__.py`` file can -contain the same Python code that any other module can contain, and Python -will add some additional attributes to the module when it is imported. - -For example, the following file system layout defines a top level ``parent`` -package with three subpackages:: - - parent/ - __init__.py - one/ - __init__.py - two/ - __init__.py - three/ - __init__.py - -Importing ``parent.one`` will implicitly execute ``parent/__init__.py`` and -``parent/one/__init__.py``. Subsequent imports of ``parent.two`` or -``parent.three`` will execute ``parent/two/__init__.py`` and -``parent/three/__init__.py`` respectively. - - -Namespace packages ------------------- - -.. index:: - pair: package; namespace - pair: package; portion - -A namespace package is a composite of various :term:`portions `, -where each portion contributes a subpackage to the parent package. Portions -may reside in different locations on the file system. Portions may also be -found in zip files, on the network, or anywhere else that Python searches -during import. Namespace packages may or may not correspond directly to -objects on the file system; they may be virtual modules that have no concrete -representation. - -Namespace packages do not use an ordinary list for their ``__path__`` -attribute. They instead use a custom iterable type which will automatically -perform a new search for package portions on the next import attempt within -that package if the path of their parent package (or :data:`sys.path` for a -top level package) changes. - -With namespace packages, there is no ``parent/__init__.py`` file. In fact, -there may be multiple ``parent`` directories found during import search, where -each one is provided by a different portion. Thus ``parent/one`` may not be -physically located next to ``parent/two``. In this case, Python will create a -namespace package for the top-level ``parent`` package whenever it or one of -its subpackages is imported. - -See also :pep:`420` for the namespace package specification. - - -Searching -========= - -To begin the search, Python needs the :term:`fully qualified ` -name of the module (or package, but for the purposes of this discussion, the -difference is immaterial) being imported. This name may come from various -arguments to the :keyword:`import` statement, or from the parameters to the -:func:`importlib.import_module` or :func:`__import__` functions. - -This name will be used in various phases of the import search, and it may be -the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python -first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. - - -The module cache ----------------- - -.. index:: - single: sys.modules - -The first place checked during import search is :data:`sys.modules`. This -mapping serves as a cache of all modules that have been previously imported, -including the intermediate paths. So if ``foo.bar.baz`` was previously -imported, :data:`sys.modules` will contain entries for ``foo``, ``foo.bar``, -and ``foo.bar.baz``. Each key will have as its value the corresponding module -object. - -During import, the module name is looked up in :data:`sys.modules` and if -present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then a -:exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will -continue searching for the module. - -:data:`sys.modules` is writable. Deleting a key may not destroy the -associated module (as other modules may hold references to it), -but it will invalidate the cache entry for the named module, causing -Python to search anew for the named module upon its next -import. The key can also be assigned to ``None``, forcing the next import -of the module to result in a :exc:`ModuleNotFoundError`. - -Beware though, as if you keep a reference to the module object, -invalidate its cache entry in :data:`sys.modules`, and then re-import the -named module, the two module objects will *not* be the same. By contrast, -:func:`importlib.reload` will reuse the *same* module object, and simply -reinitialise the module contents by rerunning the module's code. - - -Finders and loaders -------------------- - -.. index:: - single: finder - single: loader - single: module spec - -If the named module is not found in :data:`sys.modules`, then Python's import -protocol is invoked to find and load the module. This protocol consists of -two conceptual objects, :term:`finders ` and :term:`loaders `. -A finder's job is to determine whether it can find the named module using -whatever strategy it knows about. Objects that implement both of these -interfaces are referred to as :term:`importers ` - they return -themselves when they find that they can load the requested module. - -Python includes a number of default finders and importers. The first one -knows how to locate built-in modules, and the second knows how to locate -frozen modules. A third default finder searches an :term:`import path` -for modules. The :term:`import path` is a list of locations that may -name file system paths or zip files. It can also be extended to search -for any locatable resource, such as those identified by URLs. - -The import machinery is extensible, so new finders can be added to extend the -range and scope of module searching. - -Finders do not actually load modules. If they can find the named module, they -return a :dfn:`module spec`, an encapsulation of the module's import-related -information, which the import machinery then uses when loading the module. - -The following sections describe the protocol for finders and loaders in more -detail, including how you can create and register new ones to extend the -import machinery. - -.. versionchanged:: 3.4 - In previous versions of Python, finders returned :term:`loaders ` - directly, whereas now they return module specs which *contain* loaders. - Loaders are still used during import but have fewer responsibilities. - -Import hooks ------------- - -.. index:: - single: import hooks - single: meta hooks - single: path hooks - pair: hooks; import - pair: hooks; meta - pair: hooks; path - -The import machinery is designed to be extensible; the primary mechanism for -this are the *import hooks*. There are two types of import hooks: *meta -hooks* and *import path hooks*. - -Meta hooks are called at the start of import processing, before any other -import processing has occurred, other than :data:`sys.modules` cache look up. -This allows meta hooks to override :data:`sys.path` processing, frozen -modules, or even built-in modules. Meta hooks are registered by adding new -finder objects to :data:`sys.meta_path`, as described below. - -Import path hooks are called as part of :data:`sys.path` (or -``package.__path__``) processing, at the point where their associated path -item is encountered. Import path hooks are registered by adding new callables -to :data:`sys.path_hooks` as described below. - - -The meta path -------------- - -.. index:: - single: sys.meta_path - pair: finder; find_spec - -When the named module is not found in :data:`sys.modules`, Python next -searches :data:`sys.meta_path`, which contains a list of meta path finder -objects. These finders are queried in order to see if they know how to handle -the named module. Meta path finders must implement a method called -:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: -a name, an import path, and (optionally) a target module. The meta path -finder can use any strategy it wants to determine whether it can handle -the named module or not. - -If the meta path finder knows how to handle the named module, it returns a -spec object. If it cannot handle the named module, it returns ``None``. If -:data:`sys.meta_path` processing reaches the end of its list without returning -a spec, then a :exc:`ModuleNotFoundError` is raised. Any other exceptions -raised are simply propagated up, aborting the import process. - -The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path -finders is called with two or three arguments. The first is the fully -qualified name of the module being imported, for example ``foo.bar.baz``. -The second argument is the path entries to use for the module search. For -top-level modules, the second argument is ``None``, but for submodules or -subpackages, the second argument is the value of the parent package's -``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument -is an existing module object that will be the target of loading later. -The import system passes in a target module only during reload. - -The meta path may be traversed multiple times for a single import request. -For example, assuming none of the modules involved has already been cached, -importing ``foo.bar.baz`` will first perform a top level import, calling -``mpf.find_spec("foo", None, None)`` on each meta path finder (``mpf``). After -``foo`` has been imported, ``foo.bar`` will be imported by traversing the -meta path a second time, calling -``mpf.find_spec("foo.bar", foo.__path__, None)``. Once ``foo.bar`` has been -imported, the final traversal will call -``mpf.find_spec("foo.bar.baz", foo.bar.__path__, None)``. - -Some meta path finders only support top level imports. These importers will -always return ``None`` when anything other than ``None`` is passed as the -second argument. - -Python's default :data:`sys.meta_path` has three meta path finders, one that -knows how to import built-in modules, one that knows how to import frozen -modules, and one that knows how to import modules from an :term:`import path` -(i.e. the :term:`path based finder`). - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path - finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which - is now deprecated. While it will continue to work without change, the - import machinery will try it only if the finder does not implement - ``find_spec()``. - - -Loading -======= - -If and when a module spec is found, the import machinery will use it (and -the loader it contains) when loading the module. Here is an approximation -of what happens during the loading portion of import:: - - module = None - if spec.loader is not None and hasattr(spec.loader, 'create_module'): - # It is assumed 'exec_module' will also be defined on the loader. - module = spec.loader.create_module(spec) - if module is None: - module = ModuleType(spec.name) - # The import-related module attributes get set here: - _init_module_attrs(spec, module) - - if spec.loader is None: - if spec.submodule_search_locations is not None: - # namespace package - sys.modules[spec.name] = module - else: - # unsupported - raise ImportError - elif not hasattr(spec.loader, 'exec_module'): - module = spec.loader.load_module(spec.name) - # Set __loader__ and __package__ if missing. - else: - sys.modules[spec.name] = module - try: - spec.loader.exec_module(module) - except BaseException: - try: - del sys.modules[spec.name] - except KeyError: - pass - raise - return sys.modules[spec.name] - -Note the following details: - - * If there is an existing module object with the given name in - :data:`sys.modules`, import will have already returned it. - - * The module will exist in :data:`sys.modules` before the loader - executes the module code. This is crucial because the module code may - (directly or indirectly) import itself; adding it to :data:`sys.modules` - beforehand prevents unbounded recursion in the worst case and multiple - loading in the best. - - * If loading fails, the failing module -- and only the failing module -- - gets removed from :data:`sys.modules`. Any module already in the - :data:`sys.modules` cache, and any module that was successfully loaded - as a side-effect, must remain in the cache. This contrasts with - reloading where even the failing module is left in :data:`sys.modules`. - - * After the module is created but before execution, the import machinery - sets the import-related module attributes ("_init_module_attrs" in - the pseudo-code example above), as summarized in a - :ref:`later section `. - - * Module execution is the key moment of loading in which the module's - namespace gets populated. Execution is entirely delegated to the - loader, which gets to decide what gets populated and how. - - * The module created during loading and passed to exec_module() may - not be the one returned at the end of import [#fnlo]_. - -.. versionchanged:: 3.4 - The import system has taken over the boilerplate responsibilities of - loaders. These were previously performed by the - :meth:`importlib.abc.Loader.load_module` method. - -Loaders -------- - -Module loaders provide the critical function of loading: module execution. -The import machinery calls the :meth:`importlib.abc.Loader.exec_module` -method with a single argument, the module object to execute. Any value -returned from :meth:`~importlib.abc.Loader.exec_module` is ignored. - -Loaders must satisfy the following requirements: - - * If the module is a Python module (as opposed to a built-in module or a - dynamically loaded extension), the loader should execute the module's code - in the module's global name space (``module.__dict__``). - - * If the loader cannot execute the module, it should raise an - :exc:`ImportError`, although any other exception raised during - :meth:`~importlib.abc.Loader.exec_module` will be propagated. - -In many cases, the finder and loader can be the same object; in such cases the -:meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a -spec with the loader set to ``self``. - -Module loaders may opt in to creating the module object during loading -by implementing a :meth:`~importlib.abc.Loader.create_module` method. -It takes one argument, the module spec, and returns the new module object -to use during loading. ``create_module()`` does not need to set any attributes -on the module object. If the method returns ``None``, the -import machinery will create the new module itself. - -.. versionadded:: 3.4 - The :meth:`~importlib.abc.Loader.create_module` method of loaders. - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.Loader.load_module` method was replaced by - :meth:`~importlib.abc.Loader.exec_module` and the import - machinery assumed all the boilerplate responsibilities of loading. - - For compatibility with existing loaders, the import machinery will use - the ``load_module()`` method of loaders if it exists and the loader does - not also implement ``exec_module()``. However, ``load_module()`` has been - deprecated and loaders should implement ``exec_module()`` instead. - - The ``load_module()`` method must implement all the boilerplate loading - functionality described above in addition to executing the module. All - the same constraints apply, with some additional clarification: - - * If there is an existing module object with the given name in - :data:`sys.modules`, the loader must use that existing module. - (Otherwise, :func:`importlib.reload` will not work correctly.) If the - named module does not exist in :data:`sys.modules`, the loader - must create a new module object and add it to :data:`sys.modules`. - - * The module *must* exist in :data:`sys.modules` before the loader - executes the module code, to prevent unbounded recursion or multiple - loading. - - * If loading fails, the loader must remove any modules it has inserted - into :data:`sys.modules`, but it must remove **only** the failing - module(s), and only if the loader itself has loaded the module(s) - explicitly. - -.. versionchanged:: 3.5 - A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. - -.. versionchanged:: 3.6 - An :exc:`ImportError` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. - -Submodules ----------- - -When a submodule is loaded using any mechanism (e.g. ``importlib`` APIs, the -``import`` or ``import-from`` statements, or built-in ``__import__()``) a -binding is placed in the parent module's namespace to the submodule object. -For example, if package ``spam`` has a submodule ``foo``, after importing -``spam.foo``, ``spam`` will have an attribute ``foo`` which is bound to the -submodule. Let's say you have the following directory structure:: - - spam/ - __init__.py - foo.py - bar.py - -and ``spam/__init__.py`` has the following lines in it:: - - from .foo import Foo - from .bar import Bar - -then executing the following puts a name binding to ``foo`` and ``bar`` in the -``spam`` module:: - - >>> import spam - >>> spam.foo - - >>> spam.bar - - -Given Python's familiar name binding rules this might seem surprising, but -it's actually a fundamental feature of the import system. The invariant -holding is that if you have ``sys.modules['spam']`` and -``sys.modules['spam.foo']`` (as you would after the above import), the latter -must appear as the ``foo`` attribute of the former. - -Module spec ------------ - -The import machinery uses a variety of information about each module -during import, especially before loading. Most of the information is -common to all modules. The purpose of a module's spec is to encapsulate -this import-related information on a per-module basis. - -Using a spec during import allows state to be transferred between import -system components, e.g. between the finder that creates the module spec -and the loader that executes it. Most importantly, it allows the -import machinery to perform the boilerplate operations of loading, -whereas without a module spec the loader had that responsibility. - -The module's spec is exposed as the ``__spec__`` attribute on a module object. -See :class:`~importlib.machinery.ModuleSpec` for details on the contents of -the module spec. - -.. versionadded:: 3.4 - -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully-qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - -.. attribute:: __package__ - - The module's ``__package__`` attribute must be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. It is - expected to have the same value as ``__spec__.parent``. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__package__`` is not defined, ``__spec__.parent`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional. If set, this attribute's value must be a - string. The import system may opt to leave ``__file__`` unset if it - has no semantic meaning (e.g. a module loaded from a database). - - If ``__file__`` is set, it may also be appropriate to set the - ``__cached__`` attribute which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - It is also appropriate to set ``__cached__`` when ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of ``__file__`` and/or ``__cached__``. So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - -.. _package-path-rules: - -module.__path__ ---------------- - -By definition, if a module has a ``__path__`` attribute, it is a package. - -A package's ``__path__`` attribute is used during imports of its subpackages. -Within the import machinery, it functions much the same as :data:`sys.path`, -i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. - -``__path__`` must be an iterable of strings, but it may be empty. -The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. - -A package's ``__init__.py`` file may set or alter the package's ``__path__`` -attribute, and this was typically the way namespace packages were implemented -prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` -correctly for the namespace package. - -Module reprs ------------- - -By default, all modules have a usable repr, however depending on the -attributes set above, and in the module's spec, you can more explicitly -control the repr of module objects. - -If the module has a spec (``__spec__``), the import machinery will try -to generate a repr from it. If that fails or there is no spec, the import -system will craft a default repr using whatever information is available -on the module. It will try to use the ``module.__name__``, -``module.__file__``, and ``module.__loader__`` as input into the repr, -with defaults for whatever information is missing. - -Here are the exact rules used: - - * If the module has a ``__spec__`` attribute, the information in the spec - is used to generate the repr. The "name", "loader", "origin", and - "has_location" attributes are consulted. - - * If the module has a ``__file__`` attribute, this is used as part of the - module's repr. - - * If the module has no ``__file__`` but does have a ``__loader__`` that is not - ``None``, then the loader's repr is used as part of the module's repr. - - * Otherwise, just use the module's ``__name__`` in the repr. - -.. versionchanged:: 3.4 - Use of :meth:`loader.module_repr() ` - has been deprecated and the module spec is now used by the import - machinery to generate a module repr. - - For backward compatibility with Python 3.3, the module repr will be - generated by calling the loader's - :meth:`~importlib.abc.Loader.module_repr` method, if defined, before - trying either approach described above. However, the method is deprecated. - - -The Path Based Finder -===================== - -.. index:: - single: path based finder - -As mentioned previously, Python comes with several default meta path finders. -One of these, called the :term:`path based finder` -(:class:`~importlib.machinery.PathFinder`), searches an :term:`import path`, -which contains a list of :term:`path entries `. Each path -entry names a location to search for modules. - -The path based finder itself doesn't know how to import anything. Instead, it -traverses the individual path entries, associating each of them with a -path entry finder that knows how to handle that particular kind of path. - -The default set of path entry finders implement all the semantics for finding -modules on the file system, handling special file types such as Python source -code (``.py`` files), Python byte code (``.pyc`` files) and -shared libraries (e.g. ``.so`` files). When supported by the :mod:`zipimport` -module in the standard library, the default path entry finders also handle -loading all of these file types (other than shared libraries) from zipfiles. - -Path entries need not be limited to file system locations. They can refer to -URLs, database queries, or any other location that can be specified as a -string. - -The path based finder provides additional hooks and protocols so that you -can extend and customize the types of searchable path entries. For example, -if you wanted to support path entries as network URLs, you could write a hook -that implements HTTP semantics to find modules on the web. This hook (a -callable) would return a :term:`path entry finder` supporting the protocol -described below, which was then used to get a loader for the module from the -web. - -A word of warning: this section and the previous both use the term *finder*, -distinguishing between them by using the terms :term:`meta path finder` and -:term:`path entry finder`. These two types of finders are very similar, -support similar protocols, and function in similar ways during the import -process, but it's important to keep in mind that they are subtly different. -In particular, meta path finders operate at the beginning of the import -process, as keyed off the :data:`sys.meta_path` traversal. - -By contrast, path entry finders are in a sense an implementation detail -of the path based finder, and in fact, if the path based finder were to be -removed from :data:`sys.meta_path`, none of the path entry finder semantics -would be invoked. - - -Path entry finders ------------------- - -.. index:: - single: sys.path - single: sys.path_hooks - single: sys.path_importer_cache - single: PYTHONPATH - -The :term:`path based finder` is responsible for finding and loading -Python modules and packages whose location is specified with a string -:term:`path entry`. Most path entries name locations in the file system, -but they need not be limited to this. - -As a meta path finder, the :term:`path based finder` implements the -:meth:`~importlib.abc.MetaPathFinder.find_spec` protocol previously -described, however it exposes additional hooks that can be used to -customize how modules are found and loaded from the :term:`import path`. - -Three variables are used by the :term:`path based finder`, :data:`sys.path`, -:data:`sys.path_hooks` and :data:`sys.path_importer_cache`. The ``__path__`` -attributes on package objects are also used. These provide additional ways -that the import machinery can be customized. - -:data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` -environment variable and various other installation- and -implementation-specific defaults. Entries in :data:`sys.path` can name -directories on the file system, zip files, and potentially other "locations" -(see the :mod:`site` module) that should be searched for modules, such as -URLs, or database queries. Only strings and bytes should be present on -:data:`sys.path`; all other data types are ignored. The encoding of bytes -entries is determined by the individual :term:`path entry finders `. - -The :term:`path based finder` is a :term:`meta path finder`, so the import -machinery begins the :term:`import path` search by calling the path -based finder's :meth:`~importlib.machinery.PathFinder.find_spec` method as -described previously. When the ``path`` argument to -:meth:`~importlib.machinery.PathFinder.find_spec` is given, it will be a -list of string paths to traverse - typically a package's ``__path__`` -attribute for an import within that package. If the ``path`` argument is -``None``, this indicates a top level import and :data:`sys.path` is used. - -The path based finder iterates over every entry in the search path, and -for each of these, looks for an appropriate :term:`path entry finder` -(:class:`~importlib.abc.PathEntryFinder`) for the -path entry. Because this can be an expensive operation (e.g. there may be -`stat()` call overheads for this search), the path based finder maintains -a cache mapping path entries to path entry finders. This cache is maintained -in :data:`sys.path_importer_cache` (despite the name, this cache actually -stores finder objects rather than being limited to :term:`importer` objects). -In this way, the expensive search for a particular :term:`path entry` -location's :term:`path entry finder` need only be done once. User code is -free to remove cache entries from :data:`sys.path_importer_cache` forcing -the path based finder to perform the path entry search again [#fnpic]_. - -If the path entry is not present in the cache, the path based finder iterates -over every callable in :data:`sys.path_hooks`. Each of the :term:`path entry -hooks ` in this list is called with a single argument, the -path entry to be searched. This callable may either return a :term:`path -entry finder` that can handle the path entry, or it may raise -:exc:`ImportError`. An :exc:`ImportError` is used by the path based finder to -signal that the hook cannot find a :term:`path entry finder` -for that :term:`path entry`. The -exception is ignored and :term:`import path` iteration continues. The hook -should expect either a string or bytes object; the encoding of bytes objects -is up to the hook (e.g. it may be a file system encoding, UTF-8, or something -else), and if the hook cannot decode the argument, it should raise -:exc:`ImportError`. - -If :data:`sys.path_hooks` iteration ends with no :term:`path entry finder` -being returned, then the path based finder's -:meth:`~importlib.machinery.PathFinder.find_spec` method will store ``None`` -in :data:`sys.path_importer_cache` (to indicate that there is no finder for -this path entry) and return ``None``, indicating that this -:term:`meta path finder` could not find the module. - -If a :term:`path entry finder` *is* returned by one of the :term:`path entry -hook` callables on :data:`sys.path_hooks`, then the following protocol is used -to ask the finder for a module spec, which is then used when loading the -module. - -The current working directory -- denoted by an empty string -- is handled -slightly differently from other entries on :data:`sys.path`. First, if the -current working directory is found to not exist, no value is stored in -:data:`sys.path_importer_cache`. Second, the value for the current working -directory is looked up fresh for each module lookup. Third, the path used for -:data:`sys.path_importer_cache` and returned by -:meth:`importlib.machinery.PathFinder.find_spec` will be the actual current -working directory and not the empty string. - -Path entry finder protocol --------------------------- - -In order to support imports of modules and initialized packages and also to -contribute portions to namespace packages, path entry finders must implement -the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. - -:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the -fully qualified name of the module being imported, and the (optional) target -module. ``find_spec()`` returns a fully populated spec for the module. -This spec will always have "loader" set (with one exception). - -To indicate to the import machinery that the spec represents a namespace -:term:`portion`. the path entry finder sets "loader" on the spec to -``None`` and "submodule_search_locations" to a list containing the -portion. - -.. versionchanged:: 3.4 - :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced - :meth:`~importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which - are now deprecated, but will be used if ``find_spec()`` is not defined. - - Older path entry finders may implement one of these two deprecated methods - instead of ``find_spec()``. The methods are still respected for the - sake of backward compatibility. However, if ``find_spec()`` is - implemented on the path entry finder, the legacy methods are ignored. - - :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the - fully qualified name of the module being imported. ``find_loader()`` - returns a 2-tuple where the first item is the loader and the second item - is a namespace :term:`portion`. When the first item (i.e. the loader) is - ``None``, this means that while the path entry finder does not have a - loader for the named module, it knows that the path entry contributes to - a namespace portion for the named module. This will almost always be the - case where Python is asked to import a namespace package that has no - physical presence on the file system. When a path entry finder returns - ``None`` for the loader, the second item of the 2-tuple return value must - be a sequence, although it can be empty. - - If ``find_loader()`` returns a non-``None`` loader value, the portion is - ignored and the loader is returned from the path based finder, terminating - the search through the path entries. - - For backwards compatibility with other implementations of the import - protocol, many path entry finders also support the same, - traditional ``find_module()`` method that meta path finders support. - However path entry finder ``find_module()`` methods are never called - with a ``path`` argument (they are expected to record the appropriate - path information from the initial call to the path hook). - - The ``find_module()`` method on path entry finders is deprecated, - as it does not allow the path entry finder to contribute portions to - namespace packages. If both ``find_loader()`` and ``find_module()`` - exist on a path entry finder, the import system will always call - ``find_loader()`` in preference to ``find_module()``. - - -Replacing the standard import system -==================================== - -The most reliable mechanism for replacing the entire import system is to -delete the default contents of :data:`sys.meta_path`, replacing them -entirely with a custom meta path hook. - -If it is acceptable to only alter the behaviour of import statements -without affecting other APIs that access the import system, then replacing -the builtin :func:`__import__` function may be sufficient. This technique -may also be employed at the module level to only alter the behaviour of -import statements within that module. - -To selectively prevent import of some modules from a hook early on the -meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNotFoundError` directly from -:meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning -``None``. The latter indicates that the meta path search should continue, -while raising an exception terminates it immediately. - - -Special considerations for __main__ -=================================== - -The :mod:`__main__` module is a special case relative to Python's import -system. As noted :ref:`elsewhere `, the ``__main__`` module -is directly initialized at interpreter startup, much like :mod:`sys` and -:mod:`builtins`. However, unlike those two, it doesn't strictly -qualify as a built-in module. This is because the manner in which -``__main__`` is initialized depends on the flags and other options with -which the interpreter is invoked. - -.. _main_spec: - -__main__.__spec__ ------------------ - -Depending on how :mod:`__main__` is initialized, ``__main__.__spec__`` -gets set appropriately or to ``None``. - -When Python is started with the :option:`-m` option, ``__spec__`` is set -to the module spec of the corresponding module or package. ``__spec__`` is -also populated when the ``__main__`` module is loaded as part of executing a -directory, zipfile or other :data:`sys.path` entry. - -In :ref:`the remaining cases ` -``__main__.__spec__`` is set to ``None``, as the code used to populate the -:mod:`__main__` does not correspond directly with an importable module: - -- interactive prompt -- :option:`-c` option -- running from stdin -- running directly from a source or bytecode file - -Note that ``__main__.__spec__`` is always ``None`` in the last case, -*even if* the file could technically be imported directly as a module -instead. Use the :option:`-m` switch if valid module metadata is desired -in :mod:`__main__`. - -Note also that even when ``__main__`` corresponds with an importable module -and ``__main__.__spec__`` is set accordingly, they're still considered -*distinct* modules. This is due to the fact that blocks guarded by -``if __name__ == "__main__":`` checks only execute when the module is used -to populate the ``__main__`` namespace, and not during normal import. - - -Open issues -=========== - -XXX It would be really nice to have a diagram. - -XXX * (import_machinery.rst) how about a section devoted just to the -attributes of modules and packages, perhaps expanding upon or supplanting the -related entries in the data model reference page? - -XXX runpy, pkgutil, et al in the library manual should all get "See Also" -links at the top pointing to the new import system section. - -XXX Add more explanation regarding the different ways in which -``__main__`` is initialized? - -XXX Add more info on ``__main__`` quirks/pitfalls (i.e. copy from -:pep:`395`). - - -References -========== - -The import machinery has evolved considerably since Python's early days. The -original `specification for packages -`_ is still available to read, -although some details have changed since the writing of that document. - -The original specification for :data:`sys.meta_path` was :pep:`302`, with -subsequent extension in :pep:`420`. - -:pep:`420` introduced :term:`namespace packages ` for -Python 3.3. :pep:`420` also introduced the :meth:`find_loader` protocol as an -alternative to :meth:`find_module`. - -:pep:`366` describes the addition of the ``__package__`` attribute for -explicit relative imports in main modules. - -:pep:`328` introduced absolute and explicit relative imports and initially -proposed ``__name__`` for semantics :pep:`366` would eventually specify for -``__package__``. - -:pep:`338` defines executing modules as scripts. - -:pep:`451` adds the encapsulation of per-module import state in spec -objects. It also off-loads most of the boilerplate responsibilities of -loaders back onto the import machinery. These changes allow the -deprecation of several APIs in the import system and also addition of new -methods to finders and loaders. - -.. rubric:: Footnotes - -.. [#fnmo] See :class:`types.ModuleType`. - -.. [#fnlo] The importlib implementation avoids using the return value - directly. Instead, it gets the module object by looking the module name up - in :data:`sys.modules`. The indirect effect of this is that an imported - module may replace itself in :data:`sys.modules`. This is - implementation-specific behavior that is not guaranteed to work in other - Python implementations. - -.. [#fnpic] In legacy code, it is possible to find instances of - :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It - is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. diff --git a/third_party/python/Doc/reference/index.rst b/third_party/python/Doc/reference/index.rst deleted file mode 100644 index a66673b17..000000000 --- a/third_party/python/Doc/reference/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _reference-index: - -################################# - The Python Language Reference -################################# - -This reference manual describes the syntax and "core semantics" of the -language. It is terse, but attempts to be exact and complete. The semantics of -non-essential built-in object types and of the built-in functions and modules -are described in :ref:`library-index`. For an informal introduction to the -language, see :ref:`tutorial-index`. For C or C++ programmers, two additional -manuals exist: :ref:`extending-index` describes the high-level picture of how to -write a Python extension module, and the :ref:`c-api-index` describes the -interfaces available to C/C++ programmers in detail. - -.. toctree:: - :maxdepth: 2 - :numbered: - - introduction.rst - lexical_analysis.rst - datamodel.rst - executionmodel.rst - import.rst - expressions.rst - simple_stmts.rst - compound_stmts.rst - toplevel_components.rst - grammar.rst diff --git a/third_party/python/Doc/reference/introduction.rst b/third_party/python/Doc/reference/introduction.rst deleted file mode 100644 index bb7e3906d..000000000 --- a/third_party/python/Doc/reference/introduction.rst +++ /dev/null @@ -1,132 +0,0 @@ - -.. _introduction: - -************ -Introduction -************ - -This reference manual describes the Python programming language. It is not -intended as a tutorial. - -While I am trying to be as precise as possible, I chose to use English rather -than formal specifications for everything except syntax and lexical analysis. -This should make the document more understandable to the average reader, but -will leave room for ambiguities. Consequently, if you were coming from Mars and -tried to re-implement Python from this document alone, you might have to guess -things and in fact you would probably end up implementing quite a different -language. On the other hand, if you are using Python and wonder what the precise -rules about a particular area of the language are, you should definitely be able -to find them here. If you would like to see a more formal definition of the -language, maybe you could volunteer your time --- or invent a cloning machine -:-). - -It is dangerous to add too many implementation details to a language reference -document --- the implementation may change, and other implementations of the -same language may work differently. On the other hand, CPython is the one -Python implementation in widespread use (although alternate implementations -continue to gain support), and its particular quirks are sometimes worth being -mentioned, especially where the implementation imposes additional limitations. -Therefore, you'll find short "implementation notes" sprinkled throughout the -text. - -Every Python implementation comes with a number of built-in and standard -modules. These are documented in :ref:`library-index`. A few built-in modules -are mentioned when they interact in a significant way with the language -definition. - - -.. _implementations: - -Alternate Implementations -========================= - -Though there is one Python implementation which is by far the most popular, -there are some alternate implementations which are of particular interest to -different audiences. - -Known implementations include: - -CPython - This is the original and most-maintained implementation of Python, written in C. - New language features generally appear here first. - -Jython - Python implemented in Java. This implementation can be used as a scripting - language for Java applications, or can be used to create applications using the - Java class libraries. It is also often used to create tests for Java libraries. - More information can be found at `the Jython website `_. - -Python for .NET - This implementation actually uses the CPython implementation, but is a managed - .NET application and makes .NET libraries available. It was created by Brian - Lloyd. For more information, see the `Python for .NET home page - `_. - -IronPython - An alternate Python for .NET. Unlike Python.NET, this is a complete Python - implementation that generates IL, and compiles Python code directly to .NET - assemblies. It was created by Jim Hugunin, the original creator of Jython. For - more information, see `the IronPython website `_. - -PyPy - An implementation of Python written completely in Python. It supports several - advanced features not found in other implementations like stackless support - and a Just in Time compiler. One of the goals of the project is to encourage - experimentation with the language itself by making it easier to modify the - interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page `_. - -Each of these implementations varies in some way from the language as documented -in this manual, or introduces specific information beyond what's covered in the -standard Python documentation. Please refer to the implementation-specific -documentation to determine what else you need to know about the specific -implementation you're using. - - -.. _notation: - -Notation -======== - -.. index:: BNF, grammar, syntax, notation - -The descriptions of lexical analysis and syntax use a modified BNF grammar -notation. This uses the following style of definition: - -.. productionlist:: * - name: `lc_letter` (`lc_letter` | "_")* - lc_letter: "a"..."z" - -The first line says that a ``name`` is an ``lc_letter`` followed by a sequence -of zero or more ``lc_letter``\ s and underscores. An ``lc_letter`` in turn is -any of the single characters ``'a'`` through ``'z'``. (This rule is actually -adhered to for the names defined in lexical and grammar rules in this document.) - -Each rule begins with a name (which is the name defined by the rule) and -``::=``. A vertical bar (``|``) is used to separate alternatives; it is the -least binding operator in this notation. A star (``*``) means zero or more -repetitions of the preceding item; likewise, a plus (``+``) means one or more -repetitions, and a phrase enclosed in square brackets (``[ ]``) means zero or -one occurrences (in other words, the enclosed phrase is optional). The ``*`` -and ``+`` operators bind as tightly as possible; parentheses are used for -grouping. Literal strings are enclosed in quotes. White space is only -meaningful to separate tokens. Rules are normally contained on a single line; -rules with many alternatives may be formatted alternatively with each line after -the first beginning with a vertical bar. - -.. index:: lexical definitions, ASCII - -In lexical definitions (as the example above), two more conventions are used: -Two literal characters separated by three dots mean a choice of any single -character in the given (inclusive) range of ASCII characters. A phrase between -angular brackets (``<...>``) gives an informal description of the symbol -defined; e.g., this could be used to describe the notion of 'control character' -if needed. - -Even though the notation used is almost the same, there is a big difference -between the meaning of lexical and syntactic definitions: a lexical definition -operates on the individual characters of the input source, while a syntax -definition operates on the stream of tokens generated by the lexical analysis. -All uses of BNF in the next chapter ("Lexical Analysis") are lexical -definitions; uses in subsequent chapters are syntactic definitions. - diff --git a/third_party/python/Doc/reference/lexical_analysis.rst b/third_party/python/Doc/reference/lexical_analysis.rst deleted file mode 100644 index 3a03b9471..000000000 --- a/third_party/python/Doc/reference/lexical_analysis.rst +++ /dev/null @@ -1,939 +0,0 @@ - -.. _lexical: - -**************** -Lexical analysis -**************** - -.. index:: lexical analysis, parser, token - -A Python program is read by a *parser*. Input to the parser is a stream of -*tokens*, generated by the *lexical analyzer*. This chapter describes how the -lexical analyzer breaks a file into tokens. - -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. - - -.. _line-structure: - -Line structure -============== - -.. index:: line structure - -A Python program is divided into a number of *logical lines*. - - -.. _logical-lines: - -Logical lines -------------- - -.. index:: logical line, physical line, line joining, NEWLINE token - -The end of a logical line is represented by the token NEWLINE. Statements -cannot cross logical line boundaries except where NEWLINE is allowed by the -syntax (e.g., between statements in compound statements). A logical line is -constructed from one or more *physical lines* by following the explicit or -implicit *line joining* rules. - - -.. _physical-lines: - -Physical lines --------------- - -A physical line is a sequence of characters terminated by an end-of-line -sequence. In source files and strings, any of the standard platform line -termination sequences can be used - the Unix form using ASCII LF (linefeed), -the Windows form using the ASCII sequence CR LF (return followed by linefeed), -or the old Macintosh form using the ASCII CR (return) character. All of these -forms can be used equally, regardless of platform. The end of input also serves -as an implicit terminator for the final physical line. - -When embedding Python, source code strings should be passed to Python APIs using -the standard C conventions for newline characters (the ``\n`` character, -representing ASCII LF, is the line terminator). - - -.. _comments: - -Comments --------- - -.. index:: comment, hash character - single: # (hash); comment - -A comment starts with a hash character (``#``) that is not part of a string -literal, and ends at the end of the physical line. A comment signifies the end -of the logical line unless the implicit line joining rules are invoked. Comments -are ignored by the syntax; they are not tokens. - - -.. _encodings: - -Encoding declarations ---------------------- - -.. index:: source character set, encoding declarations (source file) - single: # (hash); source encoding declaration - -If a comment in the first or second line of the Python script matches the -regular expression ``coding[=:]\s*([-\w.]+)``, this comment is processed as an -encoding declaration; the first group of this expression names the encoding of -the source code file. The encoding declaration must appear on a line of its -own. If it is the second line, the first line must also be a comment-only line. -The recommended forms of an encoding expression are :: - - # -*- coding: -*- - -which is recognized also by GNU Emacs, and :: - - # vim:fileencoding= - -which is recognized by Bram Moolenaar's VIM. - -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). - -If an encoding is declared, the encoding name must be recognized by Python. The -encoding is used for all lexical analysis, including string literals, comments -and identifiers. - -.. XXX there should be a list of supported encodings. - - -.. _explicit-joining: - -Explicit line joining ---------------------- - -.. index:: physical line, line joining, line continuation, backslash character - -Two or more physical lines may be joined into logical lines using backslash -characters (``\``), as follows: when a physical line ends in a backslash that is -not part of a string literal or comment, it is joined with the following forming -a single logical line, deleting the backslash and the following end-of-line -character. For example:: - - if 1900 < year < 2100 and 1 <= month <= 12 \ - and 1 <= day <= 31 and 0 <= hour < 24 \ - and 0 <= minute < 60 and 0 <= second < 60: # Looks like a valid date - return 1 - -A line ending in a backslash cannot carry a comment. A backslash does not -continue a comment. A backslash does not continue a token except for string -literals (i.e., tokens other than string literals cannot be split across -physical lines using a backslash). A backslash is illegal elsewhere on a line -outside a string literal. - - -.. _implicit-joining: - -Implicit line joining ---------------------- - -Expressions in parentheses, square brackets or curly braces can be split over -more than one physical line without using backslashes. For example:: - - month_names = ['Januari', 'Februari', 'Maart', # These are the - 'April', 'Mei', 'Juni', # Dutch names - 'Juli', 'Augustus', 'September', # for the months - 'Oktober', 'November', 'December'] # of the year - -Implicitly continued lines can carry comments. The indentation of the -continuation lines is not important. Blank continuation lines are allowed. -There is no NEWLINE token between implicit continuation lines. Implicitly -continued lines can also occur within triple-quoted strings (see below); in that -case they cannot carry comments. - - -.. _blank-lines: - -Blank lines ------------ - -.. index:: single: blank line - -A logical line that contains only spaces, tabs, formfeeds and possibly a -comment, is ignored (i.e., no NEWLINE token is generated). During interactive -input of statements, handling of a blank line may differ depending on the -implementation of the read-eval-print loop. In the standard interactive -interpreter, an entirely blank logical line (i.e. one containing not even -whitespace or a comment) terminates a multi-line statement. - - -.. _indentation: - -Indentation ------------ - -.. index:: indentation, leading whitespace, space, tab, grouping, statement grouping - -Leading whitespace (spaces and tabs) at the beginning of a logical line is used -to compute the indentation level of the line, which in turn is used to determine -the grouping of statements. - -Tabs are replaced (from left to right) by one to eight spaces such that the -total number of characters up to and including the replacement is a multiple of -eight (this is intended to be the same rule as used by Unix). The total number -of spaces preceding the first non-blank character then determines the line's -indentation. Indentation cannot be split over multiple physical lines using -backslashes; the whitespace up to the first backslash determines the -indentation. - -Indentation is rejected as inconsistent if a source file mixes tabs and spaces -in a way that makes the meaning dependent on the worth of a tab in spaces; a -:exc:`TabError` is raised in that case. - -**Cross-platform compatibility note:** because of the nature of text editors on -non-UNIX platforms, it is unwise to use a mixture of spaces and tabs for the -indentation in a single source file. It should also be noted that different -platforms may explicitly limit the maximum indentation level. - -A formfeed character may be present at the start of the line; it will be ignored -for the indentation calculations above. Formfeed characters occurring elsewhere -in the leading whitespace have an undefined effect (for instance, they may reset -the space count to zero). - -.. index:: INDENT token, DEDENT token - -The indentation levels of consecutive lines are used to generate INDENT and -DEDENT tokens, using a stack, as follows. - -Before the first line of the file is read, a single zero is pushed on the stack; -this will never be popped off again. The numbers pushed on the stack will -always be strictly increasing from bottom to top. At the beginning of each -logical line, the line's indentation level is compared to the top of the stack. -If it is equal, nothing happens. If it is larger, it is pushed on the stack, and -one INDENT token is generated. If it is smaller, it *must* be one of the -numbers occurring on the stack; all numbers on the stack that are larger are -popped off, and for each number popped off a DEDENT token is generated. At the -end of the file, a DEDENT token is generated for each number remaining on the -stack that is larger than zero. - -Here is an example of a correctly (though confusingly) indented piece of Python -code:: - - def perm(l): - # Compute the list of all permutations of l - if len(l) <= 1: - return [l] - r = [] - for i in range(len(l)): - s = l[:i] + l[i+1:] - p = perm(s) - for x in p: - r.append(l[i:i+1] + x) - return r - -The following example shows various indentation errors:: - - def perm(l): # error: first line indented - for i in range(len(l)): # error: not indented - s = l[:i] + l[i+1:] - p = perm(l[:i] + l[i+1:]) # error: unexpected indent - for x in p: - r.append(l[i:i+1] + x) - return r # error: inconsistent dedent - -(Actually, the first three errors are detected by the parser; only the last -error is found by the lexical analyzer --- the indentation of ``return r`` does -not match a level popped off the stack.) - - -.. _whitespace: - -Whitespace between tokens -------------------------- - -Except at the beginning of a logical line or in string literals, the whitespace -characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation -could otherwise be interpreted as a different token (e.g., ab is one token, but -a b is two tokens). - - -.. _other-tokens: - -Other tokens -============ - -Besides NEWLINE, INDENT and DEDENT, the following categories of tokens exist: -*identifiers*, *keywords*, *literals*, *operators*, and *delimiters*. Whitespace -characters (other than line terminators, discussed earlier) are not tokens, but -serve to delimit tokens. Where ambiguity exists, a token comprises the longest -possible string that forms a legal token, when read from left to right. - - -.. _identifiers: - -Identifiers and keywords -======================== - -.. index:: identifier, name - -Identifiers (also referred to as *names*) are described by the following lexical -definitions. - -The syntax of identifiers in Python is based on the Unicode standard annex -UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for -further details. - -Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through -``Z``, the underscore ``_`` and, except for the first character, the digits -``0`` through ``9``. - -Python 3.0 introduces additional characters from outside the ASCII range (see -:pep:`3131`). For these characters, the classification uses the version of the -Unicode Character Database as included in the :mod:`unicodedata` module. - -Identifiers are unlimited in length. Case is significant. - -.. productionlist:: - identifier: `xid_start` `xid_continue`* - id_start: - id_continue: - xid_start: - xid_continue: - -The Unicode category codes mentioned above stand for: - -* *Lu* - uppercase letters -* *Ll* - lowercase letters -* *Lt* - titlecase letters -* *Lm* - modifier letters -* *Lo* - other letters -* *Nl* - letter numbers -* *Mn* - nonspacing marks -* *Mc* - spacing combining marks -* *Nd* - decimal numbers -* *Pc* - connector punctuations -* *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards - compatibility -* *Other_ID_Continue* - likewise - -All identifiers are converted into the normal form NFKC while parsing; comparison -of identifiers is based on NFKC. - -A non-normative HTML file listing all valid identifier characters for Unicode -4.1 can be found at -https://www.dcl.hpi.uni-potsdam.de/home/loewis/table-3131.html. - - -.. _keywords: - -Keywords --------- - -.. index:: - single: keyword - single: reserved word - -The following identifiers are used as reserved words, or *keywords* of the -language, and cannot be used as ordinary identifiers. They must be spelled -exactly as written here: - -.. sourcecode:: text - - False class finally is return - None continue for lambda try - True def from nonlocal while - and del global not with - as elif if or yield - assert else import pass - break except in raise - -.. index:: - single: _, identifiers - single: __, identifiers -.. _id-classes: - -Reserved classes of identifiers -------------------------------- - -Certain classes of identifiers (besides keywords) have special meanings. These -classes are identified by the patterns of leading and trailing underscore -characters: - -``_*`` - Not imported by ``from module import *``. The special identifier ``_`` is used - in the interactive interpreter to store the result of the last evaluation; it is - stored in the :mod:`builtins` module. When not in interactive mode, ``_`` - has no special meaning and is not defined. See section :ref:`import`. - - .. note:: - - The name ``_`` is often used in conjunction with internationalization; - refer to the documentation for the :mod:`gettext` module for more - information on this convention. - -``__*__`` - System-defined names. These names are defined by the interpreter and its - implementation (including the standard library). Current system names are - discussed in the :ref:`specialnames` section and elsewhere. More will likely - be defined in future versions of Python. *Any* use of ``__*__`` names, in - any context, that does not follow explicitly documented use, is subject to - breakage without warning. - -``__*`` - Class-private names. Names in this category, when used within the context of a - class definition, are re-written to use a mangled form to help avoid name - clashes between "private" attributes of base and derived classes. See section - :ref:`atom-identifiers`. - - -.. _literals: - -Literals -======== - -.. index:: literal, constant - -Literals are notations for constant values of some built-in types. - - -.. index:: string literal, bytes literal, ASCII - single: ' (single quote); string literal - single: " (double quote); string literal - single: u'; string literal - single: u"; string literal -.. _strings: - -String and Bytes literals -------------------------- - -String literals are described by the following lexical definitions: - -.. productionlist:: - stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" - : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' - longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' - shortstringitem: `shortstringchar` | `stringescapeseq` - longstringitem: `longstringchar` | `stringescapeseq` - shortstringchar: - longstringchar: - stringescapeseq: "\" - -.. productionlist:: - bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' - longbytes: "'''" `longbytesitem`* "'''" | '"""' `longbytesitem`* '"""' - shortbytesitem: `shortbyteschar` | `bytesescapeseq` - longbytesitem: `longbyteschar` | `bytesescapeseq` - shortbyteschar: - longbyteschar: - bytesescapeseq: "\" - -One syntactic restriction not indicated by these productions is that whitespace -is not allowed between the :token:`stringprefix` or :token:`bytesprefix` and the -rest of the literal. The source character set is defined by the encoding -declaration; it is UTF-8 if no encoding declaration is given in the source file; -see section :ref:`encodings`. - -.. index:: triple-quoted string, Unicode Consortium, raw string - single: """; string literal - single: '''; string literal - -In plain English: Both types of literals can be enclosed in matching single quotes -(``'``) or double quotes (``"``). They can also be enclosed in matching groups -of three single or double quotes (these are generally referred to as -*triple-quoted strings*). The backslash (``\``) character is used to escape -characters that otherwise have a special meaning, such as newline, backslash -itself, or the quote character. - -.. index:: - single: b'; bytes literal - single: b"; bytes literal - -Bytes literals are always prefixed with ``'b'`` or ``'B'``; they produce an -instance of the :class:`bytes` type instead of the :class:`str` type. They -may only contain ASCII characters; bytes with a numeric value of 128 or greater -must be expressed with escapes. - -.. index:: - single: r'; raw string literal - single: r"; raw string literal - -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as -literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` -escapes in raw strings are not treated specially. Given that Python 2.x's raw -unicode literals behave differently than Python 3.x's the ``'ur'`` syntax -is not supported. - -.. versionadded:: 3.3 - The ``'rb'`` prefix of raw bytes literals has been added as a synonym - of ``'br'``. - -.. versionadded:: 3.3 - Support for the unicode legacy literal (``u'value'``) was reintroduced - to simplify the maintenance of dual Python 2.x and 3.x codebases. - See :pep:`414` for more information. - -.. index:: - single: f'; formatted string literal - single: f"; formatted string literal - -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw -formatted strings are possible, but formatted bytes literals are not. - -In triple-quoted literals, unescaped newlines and quotes are allowed (and are -retained), except that three unescaped quotes in a row terminate the literal. (A -"quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) - -.. index:: physical line, escape sequence, Standard C, C - single: \ (backslash); escape sequence - single: \\; escape sequence - single: \a; escape sequence - single: \b; escape sequence - single: \f; escape sequence - single: \n; escape sequence - single: \r; escape sequence - single: \t; escape sequence - single: \v; escape sequence - single: \x; escape sequence - single: \N; escape sequence - single: \u; escape sequence - single: \U; escape sequence - -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and -bytes literals are interpreted according to rules similar to those used by -Standard C. The recognized escape sequences are: - -+-----------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=================+=================================+=======+ -| ``\newline`` | Backslash and newline ignored | | -+-----------------+---------------------------------+-------+ -| ``\\`` | Backslash (``\``) | | -+-----------------+---------------------------------+-------+ -| ``\'`` | Single quote (``'``) | | -+-----------------+---------------------------------+-------+ -| ``\"`` | Double quote (``"``) | | -+-----------------+---------------------------------+-------+ -| ``\a`` | ASCII Bell (BEL) | | -+-----------------+---------------------------------+-------+ -| ``\b`` | ASCII Backspace (BS) | | -+-----------------+---------------------------------+-------+ -| ``\f`` | ASCII Formfeed (FF) | | -+-----------------+---------------------------------+-------+ -| ``\n`` | ASCII Linefeed (LF) | | -+-----------------+---------------------------------+-------+ -| ``\r`` | ASCII Carriage Return (CR) | | -+-----------------+---------------------------------+-------+ -| ``\t`` | ASCII Horizontal Tab (TAB) | | -+-----------------+---------------------------------+-------+ -| ``\v`` | ASCII Vertical Tab (VT) | | -+-----------------+---------------------------------+-------+ -| ``\ooo`` | Character with octal value | (1,3) | -| | *ooo* | | -+-----------------+---------------------------------+-------+ -| ``\xhh`` | Character with hex value *hh* | (2,3) | -+-----------------+---------------------------------+-------+ - -Escape sequences only recognized in string literals are: - -+-----------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=================+=================================+=======+ -| ``\N{name}`` | Character named *name* in the | \(4) | -| | Unicode database | | -+-----------------+---------------------------------+-------+ -| ``\uxxxx`` | Character with 16-bit hex value | \(5) | -| | *xxxx* | | -+-----------------+---------------------------------+-------+ -| ``\Uxxxxxxxx`` | Character with 32-bit hex value | \(6) | -| | *xxxxxxxx* | | -+-----------------+---------------------------------+-------+ - -Notes: - -(1) - As in Standard C, up to three octal digits are accepted. - -(2) - Unlike in Standard C, exactly two hex digits are required. - -(3) - In a bytes literal, hexadecimal and octal escapes denote the byte with the - given value. In a string literal, these escapes denote a Unicode character - with the given value. - -(4) - .. versionchanged:: 3.3 - Support for name aliases [#]_ has been added. - -(5) - Exactly four hex digits are required. - -(6) - Any Unicode character can be encoded this way. Exactly eight hex digits - are required. - - -.. index:: unrecognized escape sequence - -Unlike Standard C, all unrecognized escape sequences are left in the string -unchanged, i.e., *the backslash is left in the result*. (This behavior is -useful when debugging: if an escape sequence is mistyped, the resulting output -is more easily recognized as broken.) It is also important to note that the -escape sequences only recognized in string literals fall into the category of -unrecognized escapes for bytes literals. - - .. versionchanged:: 3.6 - Unrecognized escape sequences produce a DeprecationWarning. In - some future version of Python they will be a SyntaxError. - -Even in a raw literal, quotes can be escaped with a backslash, but the -backslash remains in the result; for example, ``r"\""`` is a valid string -literal consisting of two characters: a backslash and a double quote; ``r"\"`` -is not a valid string literal (even a raw string cannot end in an odd number of -backslashes). Specifically, *a raw literal cannot end in a single backslash* -(since the backslash would escape the following quote character). Note also -that a single backslash followed by a newline is interpreted as those two -characters as part of the literal, *not* as a line continuation. - - -.. _string-concatenation: - -String literal concatenation ----------------------------- - -Multiple adjacent string or bytes literals (delimited by whitespace), possibly -using different quoting conventions, are allowed, and their meaning is the same -as their concatenation. Thus, ``"hello" 'world'`` is equivalent to -``"helloworld"``. This feature can be used to reduce the number of backslashes -needed, to split long strings conveniently across long lines, or even to add -comments to parts of strings, for example:: - - re.compile("[A-Za-z_]" # letter or underscore - "[A-Za-z0-9_]*" # letter, digit or underscore - ) - -Note that this feature is defined at the syntactical level, but implemented at -compile time. The '+' operator must be used to concatenate string expressions -at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings), -and formatted string literals may be concatenated with plain string literals. - - -.. index:: - single: formatted string literal - single: interpolated string literal - single: string; formatted literal - single: string; interpolated literal - single: f-string - single: {} (curly brackets); in formatted string literal - single: ! (exclamation); in formatted string literal - single: : (colon); in formatted string literal -.. _f-strings: - -Formatted string literals -------------------------- - -.. versionadded:: 3.6 - -A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. - -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: - -.. productionlist:: - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* - literal_char: - -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. After the expression, there may be a conversion field, -introduced by an exclamation point ``'!'``. A format specifier may also -be appended, introduced by a colon ``':'``. A replacement field ends -with a closing curly bracket ``'}'``. - -Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and a :keyword:`lambda` expression -must be surrounded by explicit parentheses. Replacement expressions -can contain line breaks (e.g. in triple-quoted strings), but they -cannot contain comments. Each expression is evaluated in the context -where the formatted string literal appears, in order from left to right. - -.. index:: - keyword: await - single: async for; in comprehensions - -An :keyword:`await` expression and comprehensions containing an -:keyword:`async for` clause are illegal in the expression in formatted -string literals. (The reason is a problem with the implementation --- -this restriction is lifted in Python 3.7). - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply-nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the string .format() method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - -A consequence of sharing the same syntax as regular string literals is -that characters in the replacement fields must not conflict with the -quoting used in the outer formatted string literal:: - - f"abc {a["x"]} def" # error: outer string literal ended prematurely - f"abc {a['x']} def" # workaround: use different quoting - -Backslashes are not allowed in format expressions and will raise -an error:: - - f"newline: {ord('\n')}" # raises SyntaxError - -To include a value in which a backslash escape is required, create -a temporary variable. - - >>> newline = ord('\n') - >>> f"newline: {newline}" - 'newline: 10' - -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. - -:: - - >>> def foo(): - ... f"Not a docstring" - ... - >>> foo.__doc__ is None - True - -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. - - -.. _numbers: - -Numeric literals ----------------- - -.. index:: number, numeric literal, integer literal - floating point literal, hexadecimal literal - octal literal, binary literal, decimal literal, imaginary literal, complex literal - -There are three types of numeric literals: integers, floating point numbers, and -imaginary numbers. There are no complex literals (complex numbers can be formed -by adding a real number and an imaginary number). - -Note that numeric literals do not include a sign; a phrase like ``-1`` is -actually an expression composed of the unary operator '``-``' and the literal -``1``. - - -.. index:: - single: 0b; integer literal - single: 0o; integer literal - single: 0x; integer literal - single: _ (underscore); in numeric literal - -.. _integers: - -Integer literals ----------------- - -Integer literals are described by the following lexical definitions: - -.. productionlist:: - integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` - decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* - bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ - octinteger: "0" ("o" | "O") (["_"] `octdigit`)+ - hexinteger: "0" ("x" | "X") (["_"] `hexdigit`)+ - nonzerodigit: "1"..."9" - digit: "0"..."9" - bindigit: "0" | "1" - octdigit: "0"..."7" - hexdigit: `digit` | "a"..."f" | "A"..."F" - -There is no limit for the length of integer literals apart from what can be -stored in available memory. - -Underscores are ignored for determining the numeric value of the literal. They -can be used to group digits for enhanced readability. One underscore can occur -between digits, and after base specifiers like ``0x``. - -Note that leading zeros in a non-zero decimal number are not allowed. This is -for disambiguation with C-style octal literals, which Python used before version -3.0. - -Some examples of integer literals:: - - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 - -.. versionchanged:: 3.6 - Underscores are now allowed for grouping purposes in literals. - - -.. index:: - single: . (dot); in numeric literal - single: e; in numeric literal - single: _ (underscore); in numeric literal -.. _floating: - -Floating point literals ------------------------ - -Floating point literals are described by the following lexical definitions: - -.. productionlist:: - floatnumber: `pointfloat` | `exponentfloat` - pointfloat: [`digitpart`] `fraction` | `digitpart` "." - exponentfloat: (`digitpart` | `pointfloat`) `exponent` - digitpart: `digit` (["_"] `digit`)* - fraction: "." `digitpart` - exponent: ("e" | "E") ["+" | "-"] `digitpart` - -Note that the integer and exponent parts are always interpreted using radix 10. -For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating point literals is implementation-dependent. As in -integer literals, underscores are supported for digit grouping. - -Some examples of floating point literals:: - - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 - -.. versionchanged:: 3.6 - Underscores are now allowed for grouping purposes in literals. - - -.. index:: - single: j; in numeric literal -.. _imaginary: - -Imaginary literals ------------------- - -Imaginary literals are described by the following lexical definitions: - -.. productionlist:: - imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") - -An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating point numbers and have the same -restrictions on their range. To create a complex number with a nonzero real -part, add a floating point number to it, e.g., ``(3+4j)``. Some examples of -imaginary literals:: - - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j - - -.. _operators: - -Operators -========= - -.. index:: single: operators - -The following tokens are operators: - -.. code-block:: none - - - + - * ** / // % @ - << >> & | ^ ~ - < > <= >= == != - - -.. _delimiters: - -Delimiters -========== - -.. index:: single: delimiters - -The following tokens serve as delimiters in the grammar: - -.. code-block:: none - - ( ) [ ] { } - , : . ; @ = -> - += -= *= /= //= %= @= - &= |= ^= >>= <<= **= - -The period can also occur in floating-point and imaginary literals. A sequence -of three periods has a special meaning as an ellipsis literal. The second half -of the list, the augmented assignment operators, serve lexically as delimiters, -but also perform an operation. - -The following printing ASCII characters have special meaning as part of other -tokens or are otherwise significant to the lexical analyzer: - -.. code-block:: none - - ' " # \ - -The following printing ASCII characters are not used in Python. Their -occurrence outside string literals and comments is an unconditional error: - -.. code-block:: none - - $ ? ` - - -.. rubric:: Footnotes - -.. [#] http://www.unicode.org/Public/9.0.0/ucd/NameAliases.txt diff --git a/third_party/python/Doc/reference/simple_stmts.rst b/third_party/python/Doc/reference/simple_stmts.rst deleted file mode 100644 index 8a6714dba..000000000 --- a/third_party/python/Doc/reference/simple_stmts.rst +++ /dev/null @@ -1,1004 +0,0 @@ - -.. _simple: - -***************** -Simple statements -***************** - -.. index:: pair: simple; statement - -A simple statement is comprised within a single logical line. Several simple -statements may occur on a single line separated by semicolons. The syntax for -simple statements is: - -.. productionlist:: - simple_stmt: `expression_stmt` - : | `assert_stmt` - : | `assignment_stmt` - : | `augmented_assignment_stmt` - : | `annotated_assignment_stmt` - : | `pass_stmt` - : | `del_stmt` - : | `return_stmt` - : | `yield_stmt` - : | `raise_stmt` - : | `break_stmt` - : | `continue_stmt` - : | `import_stmt` - : | `future_stmt` - : | `global_stmt` - : | `nonlocal_stmt` - - -.. _exprstmts: - -Expression statements -===================== - -.. index:: - pair: expression; statement - pair: expression; list -.. index:: pair: expression; list - -Expression statements are used (mostly interactively) to compute and write a -value, or (usually) to call a procedure (a function that returns no meaningful -result; in Python, procedures return the value ``None``). Other uses of -expression statements are allowed and occasionally useful. The syntax for an -expression statement is: - -.. productionlist:: - expression_stmt: `starred_expression` - -An expression statement evaluates the expression list (which may be a single -expression). - -.. index:: - builtin: repr - object: None - pair: string; conversion - single: output - pair: standard; output - pair: writing; values - pair: procedure; call - -In interactive mode, if the value is not ``None``, it is converted to a string -using the built-in :func:`repr` function and the resulting string is written to -standard output on a line by itself (except if the result is ``None``, so that -procedure calls do not cause any output.) - -.. _assignment: - -Assignment statements -===================== - -.. index:: - single: = (equals); assignment statement - pair: assignment; statement - pair: binding; name - pair: rebinding; name - object: mutable - pair: attribute; assignment - -Assignment statements are used to (re)bind names to values and to modify -attributes or items of mutable objects: - -.. productionlist:: - assignment_stmt: (`target_list` "=")+ (`starred_expression` | `yield_expression`) - target_list: `target` ("," `target`)* [","] - target: `identifier` - : | "(" [`target_list`] ")" - : | "[" [`target_list`] "]" - : | `attributeref` - : | `subscription` - : | `slicing` - : | "*" `target` - -(See section :ref:`primaries` for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) - -An assignment statement evaluates the expression list (remember that this can be -a single expression or a comma-separated list, the latter yielding a tuple) and -assigns the single resulting object to each of the target lists, from left to -right. - -.. index:: - single: target - pair: target; list - -Assignment is defined recursively depending on the form of the target (list). -When a target is part of a mutable object (an attribute reference, subscription -or slicing), the mutable object must ultimately perform the assignment and -decide about its validity, and may raise an exception if the assignment is -unacceptable. The rules observed by various types and the exceptions raised are -given with the definition of the object types (see section :ref:`types`). - -.. index:: triple: target; list; assignment - single: , (comma); in target list - single: * (asterisk); in assignment target list - single: [] (square brackets); in assignment target list - single: () (parentheses); in assignment target list - -Assignment of an object to a target list, optionally enclosed in parentheses or -square brackets, is recursively defined as follows. - -* If the target list is a single target with no trailing comma, - optionally in parentheses, the object is assigned to that target. - -* Else: The object must be an iterable with the same number of - items as there are targets in the target list, and the items are assigned, - from left to right, to the corresponding targets. - - * If the target list contains one target prefixed with an asterisk, called a - "starred" target: The object must be an iterable with at least as many items - as there are targets in the target list, minus one. The first items of the - iterable are assigned, from left to right, to the targets before the starred - target. The final items of the iterable are assigned to the targets after - the starred target. A list of the remaining items in the iterable is then - assigned to the starred target (the list can be empty). - - * Else: The object must be an iterable with the same number of items as there - are targets in the target list, and the items are assigned, from left to - right, to the corresponding targets. - -Assignment of an object to a single target is recursively defined as follows. - -* If the target is an identifier (name): - - * If the name does not occur in a :keyword:`global` or :keyword:`nonlocal` - statement in the current code block: the name is bound to the object in the - current local namespace. - - * Otherwise: the name is bound to the object in the global namespace or the - outer namespace determined by :keyword:`nonlocal`, respectively. - - .. index:: single: destructor - - The name is rebound if it was already bound. This may cause the reference - count for the object previously bound to the name to reach zero, causing the - object to be deallocated and its destructor (if it has one) to be called. - - .. index:: pair: attribute; assignment - -* If the target is an attribute reference: The primary expression in the - reference is evaluated. It should yield an object with assignable attributes; - if this is not the case, :exc:`TypeError` is raised. That object is then - asked to assign the assigned object to the given attribute; if it cannot - perform the assignment, it raises an exception (usually but not necessarily - :exc:`AttributeError`). - - .. _attr-target-note: - - Note: If the object is a class instance and the attribute reference occurs on - both sides of the assignment operator, the RHS expression, ``a.x`` can access - either an instance attribute or (if no instance attribute exists) a class - attribute. The LHS target ``a.x`` is always set as an instance attribute, - creating it if necessary. Thus, the two occurrences of ``a.x`` do not - necessarily refer to the same attribute: if the RHS expression refers to a - class attribute, the LHS creates a new instance attribute as the target of the - assignment:: - - class Cls: - x = 3 # class variable - inst = Cls() - inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 - - This description does not necessarily apply to descriptor attributes, such as - properties created with :func:`property`. - - .. index:: - pair: subscription; assignment - object: mutable - -* If the target is a subscription: The primary expression in the reference is - evaluated. It should yield either a mutable sequence object (such as a list) - or a mapping object (such as a dictionary). Next, the subscript expression is - evaluated. - - .. index:: - object: sequence - object: list - - If the primary is a mutable sequence object (such as a list), the subscript - must yield an integer. If it is negative, the sequence's length is added to - it. The resulting value must be a nonnegative integer less than the - sequence's length, and the sequence is asked to assign the assigned object to - its item with that index. If the index is out of range, :exc:`IndexError` is - raised (assignment to a subscripted sequence cannot add new items to a list). - - .. index:: - object: mapping - object: dictionary - - If the primary is a mapping object (such as a dictionary), the subscript must - have a type compatible with the mapping's key type, and the mapping is then - asked to create a key/datum pair which maps the subscript to the assigned - object. This can either replace an existing key/value pair with the same key - value, or insert a new key/value pair (if no key with the same value existed). - - For user-defined objects, the :meth:`__setitem__` method is called with - appropriate arguments. - - .. index:: pair: slicing; assignment - -* If the target is a slicing: The primary expression in the reference is - evaluated. It should yield a mutable sequence object (such as a list). The - assigned object should be a sequence object of the same type. Next, the lower - and upper bound expressions are evaluated, insofar they are present; defaults - are zero and the sequence's length. The bounds should evaluate to integers. - If either bound is negative, the sequence's length is added to it. The - resulting bounds are clipped to lie between zero and the sequence's length, - inclusive. Finally, the sequence object is asked to replace the slice with - the items of the assigned sequence. The length of the slice may be different - from the length of the assigned sequence, thus changing the length of the - target sequence, if the target sequence allows it. - -.. impl-detail:: - - In the current implementation, the syntax for targets is taken to be the same - as for expressions, and invalid syntax is rejected during the code generation - phase, causing less detailed error messages. - -Although the definition of assignment implies that overlaps between the -left-hand side and the right-hand side are 'simultaneous' (for example ``a, b = -b, a`` swaps two variables), overlaps *within* the collection of assigned-to -variables occur left-to-right, sometimes resulting in confusion. For instance, -the following program prints ``[0, 2]``:: - - x = [0, 1] - i = 0 - i, x[i] = 1, 2 # i is updated, then x[i] is updated - print(x) - - -.. seealso:: - - :pep:`3132` - Extended Iterable Unpacking - The specification for the ``*target`` feature. - - -.. _augassign: - -Augmented assignment statements -------------------------------- - -.. index:: - pair: augmented; assignment - single: statement; assignment, augmented - single: +=; augmented assignment - single: -=; augmented assignment - single: *=; augmented assignment - single: /=; augmented assignment - single: %=; augmented assignment - single: &=; augmented assignment - single: ^=; augmented assignment - single: |=; augmented assignment - single: **=; augmented assignment - single: //=; augmented assignment - single: >>=; augmented assignment - single: <<=; augmented assignment - -Augmented assignment is the combination, in a single statement, of a binary -operation and an assignment statement: - -.. productionlist:: - augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) - augtarget: `identifier` | `attributeref` | `subscription` | `slicing` - augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" - : | ">>=" | "<<=" | "&=" | "^=" | "|=" - -(See section :ref:`primaries` for the syntax definitions of the last three -symbols.) - -An augmented assignment evaluates the target (which, unlike normal assignment -statements, cannot be an unpacking) and the expression list, performs the binary -operation specific to the type of assignment on the two operands, and assigns -the result to the original target. The target is only evaluated once. - -An augmented assignment expression like ``x += 1`` can be rewritten as ``x = x + -1`` to achieve a similar, but not exactly equal effect. In the augmented -version, ``x`` is only evaluated once. Also, when possible, the actual operation -is performed *in-place*, meaning that rather than creating a new object and -assigning that to the target, the old object is modified instead. - -Unlike normal assignments, augmented assignments evaluate the left-hand side -*before* evaluating the right-hand side. For example, ``a[i] += f(x)`` first -looks-up ``a[i]``, then it evaluates ``f(x)`` and performs the addition, and -lastly, it writes the result back to ``a[i]``. - -With the exception of assigning to tuples and multiple targets in a single -statement, the assignment done by augmented assignment statements is handled the -same way as normal assignments. Similarly, with the exception of the possible -*in-place* behavior, the binary operation performed by augmented assignment is -the same as the normal binary operations. - -For targets which are attribute references, the same :ref:`caveat about class -and instance attributes ` applies as for regular assignments. - - -.. _annassign: - -Annotated assignment statements -------------------------------- - -.. index:: - pair: annotated; assignment - single: statement; assignment, annotated - single: : (colon); annotated variable - -Annotation assignment is the combination, in a single statement, -of a variable or attribute annotation and an optional assignment statement: - -.. productionlist:: - annotated_assignment_stmt: `augtarget` ":" `expression` ["=" `expression`] - -The difference from normal :ref:`assignment` is that only single target and -only single right hand side value is allowed. - -For simple names as assignment targets, if in class or module scope, -the annotations are evaluated and stored in a special class or module -attribute :attr:`__annotations__` -that is a dictionary mapping from variable names (mangled if private) to -evaluated annotations. This attribute is writable and is automatically -created at the start of class or module body execution, if annotations -are found statically. - -For expressions as assignment targets, the annotations are evaluated if -in class or module scope, but not stored. - -If a name is annotated in a function scope, then this name is local for -that scope. Annotations are never evaluated and stored in function scopes. - -If the right hand side is present, an annotated -assignment performs the actual assignment before evaluating annotations -(where applicable). If the right hand side is not present for an expression -target, then the interpreter evaluates the target except for the last -:meth:`__setitem__` or :meth:`__setattr__` call. - -.. seealso:: - - :pep:`526` - Syntax for Variable Annotations - The proposal that added syntax for annotating the types of variables - (including class variables and instance variables), instead of expressing - them through comments. - - :pep:`484` - Type hints - The proposal that added the :mod:`typing` module to provide a standard - syntax for type annotations that can be used in static analysis tools and - IDEs. - - -.. _assert: - -The :keyword:`assert` statement -=============================== - -.. index:: - statement: assert - pair: debugging; assertions - single: , (comma); expression list - -Assert statements are a convenient way to insert debugging assertions into a -program: - -.. productionlist:: - assert_stmt: "assert" `expression` ["," `expression`] - -The simple form, ``assert expression``, is equivalent to :: - - if __debug__: - if not expression: raise AssertionError - -The extended form, ``assert expression1, expression2``, is equivalent to :: - - if __debug__: - if not expression1: raise AssertionError(expression2) - -.. index:: - single: __debug__ - exception: AssertionError - -These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to -the built-in variables with those names. In the current implementation, the -built-in variable :const:`__debug__` is ``True`` under normal circumstances, -``False`` when optimization is requested (command line option :option:`-O`). The current -code generator emits no code for an assert statement when optimization is -requested at compile time. Note that it is unnecessary to include the source -code for the expression that failed in the error message; it will be displayed -as part of the stack trace. - -Assignments to :const:`__debug__` are illegal. The value for the built-in variable -is determined when the interpreter starts. - - -.. _pass: - -The :keyword:`pass` statement -============================= - -.. index:: - statement: pass - pair: null; operation - pair: null; operation - -.. productionlist:: - pass_stmt: "pass" - -:keyword:`pass` is a null operation --- when it is executed, nothing happens. -It is useful as a placeholder when a statement is required syntactically, but no -code needs to be executed, for example:: - - def f(arg): pass # a function that does nothing (yet) - - class C: pass # a class with no methods (yet) - - -.. _del: - -The :keyword:`del` statement -============================ - -.. index:: - statement: del - pair: deletion; target - triple: deletion; target; list - -.. productionlist:: - del_stmt: "del" `target_list` - -Deletion is recursively defined very similar to the way assignment is defined. -Rather than spelling it out in full details, here are some hints. - -Deletion of a target list recursively deletes each target, from left to right. - -.. index:: - statement: global - pair: unbinding; name - -Deletion of a name removes the binding of that name from the local or global -namespace, depending on whether the name occurs in a :keyword:`global` statement -in the same code block. If the name is unbound, a :exc:`NameError` exception -will be raised. - -.. index:: pair: attribute; deletion - -Deletion of attribute references, subscriptions and slicings is passed to the -primary object involved; deletion of a slicing is in general equivalent to -assignment of an empty slice of the right type (but even this is determined by -the sliced object). - -.. versionchanged:: 3.2 - Previously it was illegal to delete a name from the local namespace if it - occurs as a free variable in a nested block. - - -.. _return: - -The :keyword:`return` statement -=============================== - -.. index:: - statement: return - pair: function; definition - pair: class; definition - -.. productionlist:: - return_stmt: "return" [`expression_list`] - -:keyword:`return` may only occur syntactically nested in a function definition, -not within a nested class definition. - -If an expression list is present, it is evaluated, else ``None`` is substituted. - -:keyword:`return` leaves the current function call with the expression list (or -``None``) as return value. - -.. index:: keyword: finally - -When :keyword:`return` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`finally` clause is executed before -really leaving the function. - -In a generator function, the :keyword:`return` statement indicates that the -generator is done and will cause :exc:`StopIteration` to be raised. The returned -value (if any) is used as an argument to construct :exc:`StopIteration` and -becomes the :attr:`StopIteration.value` attribute. - -In an asynchronous generator function, an empty :keyword:`return` statement -indicates that the asynchronous generator is done and will cause -:exc:`StopAsyncIteration` to be raised. A non-empty :keyword:`return` -statement is a syntax error in an asynchronous generator function. - -.. _yield: - -The :keyword:`yield` statement -============================== - -.. index:: - statement: yield - single: generator; function - single: generator; iterator - single: function; generator - exception: StopIteration - -.. productionlist:: - yield_stmt: `yield_expression` - -A :keyword:`yield` statement is semantically equivalent to a :ref:`yield -expression `. The yield statement can be used to omit the parentheses -that would otherwise be required in the equivalent yield expression -statement. For example, the yield statements :: - - yield - yield from - -are equivalent to the yield expression statements :: - - (yield ) - (yield from ) - -Yield expressions and statements are only used when defining a :term:`generator` -function, and are only used in the body of the generator function. Using yield -in a function definition is sufficient to cause that definition to create a -generator function instead of a normal function. - -For full details of :keyword:`yield` semantics, refer to the -:ref:`yieldexpr` section. - -.. _raise: - -The :keyword:`raise` statement -============================== - -.. index:: - statement: raise - single: exception - pair: raising; exception - single: __traceback__ (exception attribute) - -.. productionlist:: - raise_stmt: "raise" [`expression` ["from" `expression`]] - -If no expressions are present, :keyword:`raise` re-raises the last exception -that was active in the current scope. If no exception is active in the current -scope, a :exc:`RuntimeError` exception is raised indicating that this is an -error. - -Otherwise, :keyword:`raise` evaluates the first expression as the exception -object. It must be either a subclass or an instance of :class:`BaseException`. -If it is a class, the exception instance will be obtained when needed by -instantiating the class with no arguments. - -The :dfn:`type` of the exception is the exception instance's class, the -:dfn:`value` is the instance itself. - -.. index:: object: traceback - -A traceback object is normally created automatically when an exception is raised -and attached to it as the :attr:`__traceback__` attribute, which is writable. -You can create an exception and set your own traceback in one step using the -:meth:`with_traceback` exception method (which returns the same exception -instance, with its traceback set to its argument), like so:: - - raise Exception("foo occurred").with_traceback(tracebackobj) - -.. index:: pair: exception; chaining - __cause__ (exception attribute) - __context__ (exception attribute) - -The ``from`` clause is used for exception chaining: if given, the second -*expression* must be another exception class or instance, which will then be -attached to the raised exception as the :attr:`__cause__` attribute (which is -writable). If the raised exception is not handled, both exceptions will be -printed:: - - >>> try: - ... print(1 / 0) - ... except Exception as exc: - ... raise RuntimeError("Something bad happened") from exc - ... - Traceback (most recent call last): - File "", line 2, in - ZeroDivisionError: division by zero - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -A similar mechanism works implicitly if an exception is raised inside an -exception handler or a :keyword:`finally` clause: the previous exception is then -attached as the new exception's :attr:`__context__` attribute:: - - >>> try: - ... print(1 / 0) - ... except: - ... raise RuntimeError("Something bad happened") - ... - Traceback (most recent call last): - File "", line 2, in - ZeroDivisionError: division by zero - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -Exception chaining can be explicitly suppressed by specifying :const:`None` in -the ``from`` clause:: - - >>> try: - ... print(1 / 0) - ... except: - ... raise RuntimeError("Something bad happened") from None - ... - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -Additional information on exceptions can be found in section :ref:`exceptions`, -and information about handling exceptions is in section :ref:`try`. - -.. versionchanged:: 3.3 - :const:`None` is now permitted as ``Y`` in ``raise X from Y``. - -.. versionadded:: 3.3 - The ``__suppress_context__`` attribute to suppress automatic display of the - exception context. - -.. _break: - -The :keyword:`break` statement -============================== - -.. index:: - statement: break - statement: for - statement: while - pair: loop; statement - -.. productionlist:: - break_stmt: "break" - -:keyword:`break` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition within -that loop. - -.. index:: keyword: else - pair: loop control; target - -It terminates the nearest enclosing loop, skipping the optional :keyword:`else` -clause if the loop has one. - -If a :keyword:`for` loop is terminated by :keyword:`break`, the loop control -target keeps its current value. - -.. index:: keyword: finally - -When :keyword:`break` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`finally` clause is executed before -really leaving the loop. - - -.. _continue: - -The :keyword:`continue` statement -================================= - -.. index:: - statement: continue - statement: for - statement: while - pair: loop; statement - keyword: finally - -.. productionlist:: - continue_stmt: "continue" - -:keyword:`continue` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition or -:keyword:`finally` clause within that loop. It continues with the next -cycle of the nearest enclosing loop. - -When :keyword:`continue` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`finally` clause is executed before -really starting the next loop cycle. - - -.. _import: -.. _from: - -The :keyword:`import` statement -=============================== - -.. index:: - statement: import - single: module; importing - pair: name; binding - keyword: from - keyword: as - exception: ImportError - single: , (comma); import statement - -.. productionlist:: - import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* - : | "from" `relative_module` "import" `identifier` ["as" `identifier`] - : ("," `identifier` ["as" `identifier`])* - : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] - : ("," `identifier` ["as" `identifier`])* [","] ")" - : | "from" `module` "import" "*" - module: (`identifier` ".")* `identifier` - relative_module: "."* `module` | "."+ - -The basic import statement (no :keyword:`from` clause) is executed in two -steps: - -#. find a module, loading and initializing it if necessary -#. define a name or names in the local namespace for the scope where - the :keyword:`import` statement occurs. - -When the statement contains multiple clauses (separated by -commas) the two steps are carried out separately for each clause, just -as though the clauses had been separated out into individual import -statements. - -The details of the first step, finding and loading modules are described in -greater detail in the section on the :ref:`import system `, -which also describes the various types of packages and modules that can -be imported, as well as all the hooks that can be used to customize -the import system. Note that failures in this step may indicate either -that the module could not be located, *or* that an error occurred while -initializing the module, which includes execution of the module's code. - -If the requested module is retrieved successfully, it will be made -available in the local namespace in one of three ways: - -.. index:: single: as; import statement - -* If the module name is followed by :keyword:`as`, then the name - following :keyword:`as` is bound directly to the imported module. -* If no other name is specified, and the module being imported is a top - level module, the module's name is bound in the local namespace as a - reference to the imported module -* If the module being imported is *not* a top level module, then the name - of the top level package that contains the module is bound in the local - namespace as a reference to the top level package. The imported module - must be accessed using its full qualified name rather than directly - - -.. index:: - pair: name; binding - single: from; import statement - -The :keyword:`from` form uses a slightly more complex process: - -#. find the module specified in the :keyword:`from` clause, loading and - initializing it if necessary; -#. for each of the identifiers specified in the :keyword:`import` clauses: - - #. check if the imported module has an attribute by that name - #. if not, attempt to import a submodule with that name and then - check the imported module again for that attribute - #. if the attribute is not found, :exc:`ImportError` is raised. - #. otherwise, a reference to that value is stored in the local namespace, - using the name in the :keyword:`as` clause if it is present, - otherwise using the attribute name - -Examples:: - - import foo # foo imported and bound locally - import foo.bar.baz # foo.bar.baz imported, foo bound locally - import foo.bar.baz as fbb # foo.bar.baz imported and bound as fbb - from foo.bar import baz # foo.bar.baz imported and bound as baz - from foo import attr # foo imported and foo.attr bound as attr - -.. index:: single: * (asterisk); import statement - -If the list of identifiers is replaced by a star (``'*'``), all public -names defined in the module are bound in the local namespace for the scope -where the :keyword:`import` statement occurs. - -.. index:: single: __all__ (optional module attribute) - -The *public names* defined by a module are determined by checking the module's -namespace for a variable named ``__all__``; if defined, it must be a sequence -of strings which are names defined or imported by that module. The names -given in ``__all__`` are all considered public and are required to exist. If -``__all__`` is not defined, the set of public names includes all names found -in the module's namespace which do not begin with an underscore character -(``'_'``). ``__all__`` should contain the entire public API. It is intended -to avoid accidentally exporting items that are not part of the API (such as -library modules which were imported and used within the module). - -The wild card form of import --- ``from module import *`` --- is only allowed at -the module level. Attempting to use it in class or function definitions will -raise a :exc:`SyntaxError`. - -.. index:: - single: relative; import - -When specifying what module to import you do not have to specify the absolute -name of the module. When a module or package is contained within another -package it is possible to make a relative import within the same top package -without having to mention the package name. By using leading dots in the -specified module or package after :keyword:`from` you can specify how high to -traverse up the current package hierarchy without specifying exact names. One -leading dot means the current package where the module making the import -exists. Two dots means up one package level. Three dots is up two levels, etc. -So if you execute ``from . import mod`` from a module in the ``pkg`` package -then you will end up importing ``pkg.mod``. If you execute ``from ..subpkg2 -import mod`` from within ``pkg.subpkg1`` you will import ``pkg.subpkg2.mod``. -The specification for relative imports is contained within :pep:`328`. - -:func:`importlib.import_module` is provided to support applications that -determine dynamically the modules to be loaded. - - -.. _future: - -Future statements ------------------ - -.. index:: - pair: future; statement - single: __future__; future statement - -A :dfn:`future statement` is a directive to the compiler that a particular -module should be compiled using syntax or semantics that will be available in a -specified future release of Python where the feature becomes standard. - -The future statement is intended to ease migration to future versions of Python -that introduce incompatible changes to the language. It allows use of the new -features on a per-module basis before the release in which the feature becomes -standard. - -.. productionlist:: * - future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] - : ("," `feature` ["as" `identifier`])* - : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] - : ("," `feature` ["as" `identifier`])* [","] ")" - feature: `identifier` - -A future statement must appear near the top of the module. The only lines that -can appear before a future statement are: - -* the module docstring (if any), -* comments, -* blank lines, and -* other future statements. - -.. XXX change this if future is cleaned out - -The features recognized by Python 3.0 are ``absolute_import``, ``division``, -``generators``, ``unicode_literals``, ``print_function``, ``nested_scopes`` and -``with_statement``. They are all redundant because they are always enabled, and -only kept for backwards compatibility. - -A future statement is recognized and treated specially at compile time: Changes -to the semantics of core constructs are often implemented by generating -different code. It may even be the case that a new feature introduces new -incompatible syntax (such as a new reserved word), in which case the compiler -may need to parse the module differently. Such decisions cannot be pushed off -until runtime. - -For any given release, the compiler knows which feature names have been defined, -and raises a compile-time error if a future statement contains a feature not -known to it. - -The direct runtime semantics are the same as for any import statement: there is -a standard module :mod:`__future__`, described later, and it will be imported in -the usual way at the time the future statement is executed. - -The interesting runtime semantics depend on the specific feature enabled by the -future statement. - -Note that there is nothing special about the statement:: - - import __future__ [as name] - -That is not a future statement; it's an ordinary import statement with no -special semantics or syntax restrictions. - -Code compiled by calls to the built-in functions :func:`exec` and :func:`compile` -that occur in a module :mod:`M` containing a future statement will, by default, -use the new syntax or semantics associated with the future statement. This can -be controlled by optional arguments to :func:`compile` --- see the documentation -of that function for details. - -A future statement typed at an interactive interpreter prompt will take effect -for the rest of the interpreter session. If an interpreter is started with the -:option:`-i` option, is passed a script name to execute, and the script includes -a future statement, it will be in effect in the interactive session started -after the script is executed. - -.. seealso:: - - :pep:`236` - Back to the __future__ - The original proposal for the __future__ mechanism. - - -.. _global: - -The :keyword:`global` statement -=============================== - -.. index:: - statement: global - triple: global; name; binding - single: , (comma); identifier list - -.. productionlist:: - global_stmt: "global" `identifier` ("," `identifier`)* - -The :keyword:`global` statement is a declaration which holds for the entire -current code block. It means that the listed identifiers are to be interpreted -as globals. It would be impossible to assign to a global variable without -:keyword:`global`, although free variables may refer to globals without being -declared global. - -Names listed in a :keyword:`global` statement must not be used in the same code -block textually preceding that :keyword:`global` statement. - -Names listed in a :keyword:`global` statement must not be defined as formal -parameters or in a :keyword:`for` loop control target, :keyword:`class` -definition, function definition, :keyword:`import` statement, or variable -annotation. - -.. impl-detail:: - - The current implementation does not enforce some of these restrictions, but - programs should not abuse this freedom, as future implementations may enforce - them or silently change the meaning of the program. - -.. index:: - builtin: exec - builtin: eval - builtin: compile - -**Programmer's note:** :keyword:`global` is a directive to the parser. It -applies only to code parsed at the same time as the :keyword:`global` statement. -In particular, a :keyword:`global` statement contained in a string or code -object supplied to the built-in :func:`exec` function does not affect the code -block *containing* the function call, and code contained in such a string is -unaffected by :keyword:`global` statements in the code containing the function -call. The same applies to the :func:`eval` and :func:`compile` functions. - - -.. _nonlocal: - -The :keyword:`nonlocal` statement -================================= - -.. index:: statement: nonlocal - single: , (comma); identifier list - -.. productionlist:: - nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* - -.. XXX add when implemented - : ["=" (`target_list` "=")+ starred_expression] - : | "nonlocal" identifier augop expression_list - -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. - -.. XXX not implemented - The :keyword:`nonlocal` statement may prepend an assignment or augmented - assignment, but not an expression. - -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). - -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. - -.. seealso:: - - :pep:`3104` - Access to Names in Outer Scopes - The specification for the :keyword:`nonlocal` statement. diff --git a/third_party/python/Doc/reference/toplevel_components.rst b/third_party/python/Doc/reference/toplevel_components.rst deleted file mode 100644 index d5ffb37b2..000000000 --- a/third_party/python/Doc/reference/toplevel_components.rst +++ /dev/null @@ -1,107 +0,0 @@ - -.. _top-level: - -******************** -Top-level components -******************** - -.. index:: single: interpreter - -The Python interpreter can get its input from a number of sources: from a script -passed to it as standard input or as program argument, typed in interactively, -from a module source file, etc. This chapter gives the syntax used in these -cases. - - -.. _programs: - -Complete Python programs -======================== - -.. index:: single: program - -.. index:: - module: sys - module: __main__ - module: builtins - -While a language specification need not prescribe how the language interpreter -is invoked, it is useful to have a notion of a complete Python program. A -complete Python program is executed in a minimally initialized environment: all -built-in and standard modules are available, but none have been initialized, -except for :mod:`sys` (various system services), :mod:`builtins` (built-in -functions, exceptions and ``None``) and :mod:`__main__`. The latter is used to -provide the local and global namespace for execution of the complete program. - -The syntax for a complete Python program is that for file input, described in -the next section. - -.. index:: - single: interactive mode - module: __main__ - -The interpreter may also be invoked in interactive mode; in this case, it does -not read and execute a complete program but reads and executes one statement -(possibly compound) at a time. The initial environment is identical to that of -a complete program; each statement is executed in the namespace of -:mod:`__main__`. - -.. index:: - single: UNIX - single: Windows - single: command line - single: standard input - -A complete program can be passed to the interpreter -in three forms: with the :option:`-c` *string* command line option, as a file -passed as the first command line argument, or as standard input. If the file -or standard input is a tty device, the interpreter enters interactive mode; -otherwise, it executes the file as a complete program. - - -.. _file-input: - -File input -========== - -All input read from non-interactive files has the same form: - -.. productionlist:: - file_input: (NEWLINE | `statement`)* - -This syntax is used in the following situations: - -* when parsing a complete Python program (from a file or from a string); - -* when parsing a module; - -* when parsing a string passed to the :func:`exec` function; - - -.. _interactive: - -Interactive input -================= - -Input in interactive mode is parsed using the following grammar: - -.. productionlist:: - interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE - -Note that a (top-level) compound statement must be followed by a blank line in -interactive mode; this is needed to help the parser detect the end of the input. - - -.. _expression-input: - -Expression input -================ - -.. index:: single: input -.. index:: builtin: eval - -:func:`eval` is used for expression input. It ignores leading whitespace. The -string argument to :func:`eval` must have the following form: - -.. productionlist:: - eval_input: `expression_list` NEWLINE* diff --git a/third_party/python/Doc/tools/extensions/c_annotations.py b/third_party/python/Doc/tools/extensions/c_annotations.py deleted file mode 100644 index baa39f3b4..000000000 --- a/third_party/python/Doc/tools/extensions/c_annotations.py +++ /dev/null @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: - - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. - - * stable API annotations - - Usage: Set the `refcount_file` config value to the path to the reference - count data file. - - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" - -from os import path -from docutils import nodes -from docutils.parsers.rst import directives - -from sphinx import addnodes -from sphinx.domains.c import CObject - - -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations(dict): - @classmethod - def fromfile(cls, filename): - d = cls() - fp = open(filename, 'r') - try: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = d[function] - except KeyError: - entry = d[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - finally: - fp.close() - return d - - def add_annotations(self, app, doctree): - for node in doctree.traverse(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if par['stableabi']: - node.insert(0, nodes.emphasis(' Part of the stable ABI.', - ' Part of the stable ABI.', - classes=['stableabi'])) - if par['objtype'] != 'function': - continue - if not par[0].has_key('names') or not par[0]['names']: - continue - name = par[0]['names'][0] - if name.startswith("c."): - name = name[2:] - entry = self.get(name) - if not entry: - continue - elif entry.result_type not in ("PyObject*", "PyVarObject*"): - continue - if entry.result_refs is None: - rc = 'Return value: Always NULL.' - elif entry.result_refs: - rc = 'Return value: New reference.' - else: - rc = 'Return value: Borrowed reference.' - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) - - -def init_annotations(app): - refcounts = Annotations.fromfile( - path.join(app.srcdir, app.config.refcount_file)) - app.connect('doctree-read', refcounts.add_annotations) - - -def setup(app): - app.add_config_value('refcount_file', '', True) - app.connect('builder-inited', init_annotations) - - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, - } - old_handle_signature = CObject.handle_signature - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/third_party/python/Doc/tools/extensions/escape4chm.py b/third_party/python/Doc/tools/extensions/escape4chm.py deleted file mode 100644 index 68d4e77a3..000000000 --- a/third_party/python/Doc/tools/extensions/escape4chm.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual -effect on some MBCS Windows systems. - -https://bugs.python.org/issue32174 -""" - -import re -from html.entities import codepoint2name - -try: # sphinx>=1.6 - from sphinx.util.logging import getLogger -except ImportError: # sphinx<1.6 - from logging import getLogger - -# escape the characters which codepoint > 0x7F -def _process(string): - def escape(matchobj): - codepoint = ord(matchobj.group(0)) - - name = codepoint2name.get(codepoint) - if name is None: - return '&#%d;' % codepoint - else: - return '&%s;' % name - - return re.sub(r'[^\x00-\x7F]', escape, string) - -def escape_for_chm(app, pagename, templatename, context, doctree): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp': - return - - # escape the `body` part to 7-bit ASCII - body = context.get('body') - if body is not None: - context['body'] = _process(body) - -def fixup_keywords(app, exception): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp' or exception: - return - - getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = app.builder.outdir - outname = app.builder.config.htmlhelp_basename - with app.builder.open_file(outdir, outname + '.hhk', 'r') as f: - index = f.read() - with app.builder.open_file(outdir, outname + '.hhk', 'w') as f: - f.write(index.replace(''', ''')) - -def setup(app): - # `html-page-context` event emitted when the HTML builder has - # created a context dictionary to render a template with. - app.connect('html-page-context', escape_for_chm) - # `build-finished` event emitted when all the files have been - # output. - app.connect('build-finished', fixup_keywords) - - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/third_party/python/Doc/tools/extensions/patchlevel.py b/third_party/python/Doc/tools/extensions/patchlevel.py deleted file mode 100644 index 919ba4a12..000000000 --- a/third_party/python/Doc/tools/extensions/patchlevel.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ - - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. - - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" - -from __future__ import print_function - -import os -import re -import sys - -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') - - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - - d = {} - f = open(patchlevel_h) - try: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value - finally: - f.close() - - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) - - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release - - -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) - return version, release - - -def get_version_info(): - try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print('Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release, file=sys.stderr) - return version, release - -if __name__ == '__main__': - print(get_header_version_info('.')[1]) diff --git a/third_party/python/Doc/tools/extensions/pyspecific.py b/third_party/python/Doc/tools/extensions/pyspecific.py deleted file mode 100644 index 70bdd1754..000000000 --- a/third_party/python/Doc/tools/extensions/pyspecific.py +++ /dev/null @@ -1,406 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pyspecific.py - ~~~~~~~~~~~~~ - - Sphinx extension with Python doc-specific markup. - - :copyright: 2008-2014 by Georg Brandl. - :license: Python license. -""" - -import re -import codecs -from os import getenv, path -from time import asctime -from pprint import pformat -from docutils.io import StringOutput -from docutils.parsers.rst import Directive -from docutils.utils import new_document - -from docutils import nodes, utils - -from sphinx import addnodes -from sphinx.builders import Builder -from sphinx.locale import translators -from sphinx.util.nodes import split_explicit_title -from sphinx.writers.html import HTMLTranslator -from sphinx.writers.text import TextWriter, TextTranslator -from sphinx.writers.latex import LaTeXTranslator -from sphinx.domains.python import PyModulelevel, PyClassmember - -# Support for checking for suspicious markup - -import suspicious - - -ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/3.6/%s' - -# monkey-patch reST parser to disable alphabetic and roman enumerated lists -from docutils.parsers.rst.states import Body -Body.enum.converters['loweralpha'] = \ - Body.enum.converters['upperalpha'] = \ - Body.enum.converters['lowerroman'] = \ - Body.enum.converters['upperroman'] = lambda x: None - -# monkey-patch HTML and LaTeX translators to keep doctest blocks in the -# doctest docs themselves -orig_visit_literal_block = HTMLTranslator.visit_literal_block -orig_depart_literal_block = LaTeXTranslator.depart_literal_block - - -def new_visit_literal_block(self, node): - meta = self.builder.env.metadata[self.builder.current_docname] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_visit_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -def new_depart_literal_block(self, node): - meta = self.builder.env.metadata[self.curfilestack[-1]] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_depart_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -HTMLTranslator.visit_literal_block = new_visit_literal_block -LaTeXTranslator.depart_literal_block = new_depart_literal_block - - -# Support for marking up and linking to bugs.python.org issues - -def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) - text = 'bpo-' + issue - refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) - return [refnode], [] - - -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - -# Support for marking up implementation details - -class ImplementationDetail(Directive): - - has_content = True - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True - - # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' - - def run(self): - pnode = nodes.compound(classes=['impl-detail']) - label = translators['sphinx'].gettext(self.label_text) - content = self.content - add_text = nodes.strong(label, label) - if self.arguments: - n, m = self.state.inline_text(self.arguments[0], self.lineno) - pnode.append(nodes.paragraph('', '', *(n + m))) - self.state.nested_parse(content, self.content_offset, pnode) - if pnode.children and isinstance(pnode[0], nodes.paragraph): - content = nodes.inline(pnode[0].rawsource, translatable=True) - content.source = pnode[0].source - content.line = pnode[0].line - content += pnode[0].children - pnode[0].replace_self(nodes.paragraph('', '', content, - translatable=False)) - pnode[0].insert(0, add_text) - pnode[0].insert(1, nodes.Text(' ')) - else: - pnode.insert(0, nodes.paragraph('', '', add_text)) - return [pnode] - - -# Support for documenting decorators - -class PyDecoratorMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyDecoratorMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_addname('@', '@')) - return ret - - def needs_arglist(self): - return False - - -class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): - def run(self): - # a decorator function is a function after all - self.name = 'py:function' - return PyModulelevel.run(self) - - -class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -class PyCoroutineMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyCoroutineMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine ')) - return ret - - -class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): - def run(self): - self.name = 'py:function' - return PyModulelevel.run(self) - - -class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -class PyAbstractMethod(PyClassmember): - - def handle_signature(self, sig, signode): - ret = super(PyAbstractMethod, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('abstractmethod ', - 'abstractmethod ')) - return ret - - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -# Support for documenting version of removal in deprecations - -class DeprecatedRemoved(Directive): - has_content = True - required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - - _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - - def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - label = translators['sphinx'].gettext(self._label) - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) - node.append(para) - else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified']), - translatable=False) - node.append(para) - env = self.state.document.settings.env - env.note_versionchange('deprecated', version[0], node, self.lineno) - return [node] + messages - - -# Support for including Misc/NEWS - -issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)') -whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") - - -class MiscNews(Directive): - has_content = False - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = False - option_spec = {} - - def run(self): - fname = self.arguments[0] - source = self.state_machine.input_lines.source( - self.lineno - self.state_machine.input_offset - 1) - source_dir = getenv('PY_MISC_NEWS_DIR') - if not source_dir: - source_dir = path.dirname(path.abspath(source)) - fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) - try: - fp = codecs.open(fpath, encoding='utf-8') - try: - content = fp.read() - finally: - fp.close() - except Exception: - text = 'The NEWS file is not available.' - node = nodes.strong(text, text) - return [node] - content = issue_re.sub(r'`bpo-\1 `__', - content) - content = whatsnew_re.sub(r'\1', content) - # remove first 3 lines as they are the main heading - lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] - self.state_machine.insert_input(lines, fname) - return [] - - -# Support for building "topic help" for pydoc - -pydoc_topic_labels = [ - 'assert', 'assignment', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'binary', - 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', - 'bltin-null-object', 'bltin-type-objects', 'booleans', - 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', - 'context-managers', 'continue', 'conversions', 'customization', 'debugger', - 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', - 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', - 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', - 'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types', - 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', - 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', - 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', - 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', - 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield' -] - - -class PydocTopicsBuilder(Builder): - name = 'pydoc-topics' - - default_translator_class = TextTranslator - - def init(self): - self.topics = {} - self.secnumbers = {} - - def get_outdated_docs(self): - return 'all pydoc topics' - - def get_target_uri(self, docname, typ=None): - return '' # no URIs - - def write(self, *ignored): - try: # sphinx>=1.6 - from sphinx.util import status_iterator - except ImportError: # sphinx<1.6 - status_iterator = self.status_iterator - - writer = TextWriter(self) - for label in status_iterator(pydoc_topic_labels, - 'building topics... ', - length=len(pydoc_topic_labels)): - if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) - continue - docname, labelid, sectname = self.env.domaindata['std']['labels'][label] - doctree = self.env.get_and_resolve_doctree(docname, self) - document = new_document('
    ') - document.append(doctree.ids[labelid]) - destination = StringOutput(encoding='utf-8') - writer.write(document, destination) - self.topics[label] = writer.output - - def finish(self): - f = open(path.join(self.outdir, 'topics.py'), 'wb') - try: - f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) - f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) - f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) - finally: - f.close() - - -# Support for documenting Opcodes - -opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?') - - -def parse_opcode_signature(env, sig, signode): - """Transform an opcode signature into RST nodes.""" - m = opcode_sig_re.match(sig) - if m is None: - raise ValueError - opname, arglist = m.groups() - signode += addnodes.desc_name(opname, opname) - if arglist is not None: - paramlist = addnodes.desc_parameterlist() - signode += paramlist - paramlist += addnodes.desc_parameter(arglist, arglist) - return opname.strip() - - -# Support for documenting pdb commands - -pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)') - -# later... -# pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+ | # identifiers -# [.,:]+ | # punctuation -# [\[\]()] | # parens -# \s+ # whitespace -# ''', re.X) - - -def parse_pdb_command(env, sig, signode): - """Transform a pdb command signature into RST nodes.""" - m = pdbcmd_sig_re.match(sig) - if m is None: - raise ValueError - name, args = m.groups() - fullname = name.replace('(', '').replace(')', '') - signode += addnodes.desc_name(name, name) - if args: - signode += addnodes.desc_addname(' '+args, ' '+args) - return fullname - - -def setup(app): - app.add_role('issue', issue_role) - app.add_role('source', source_role) - app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('deprecated-removed', DeprecatedRemoved) - app.add_builder(PydocTopicsBuilder) - app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) - app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) - app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) - app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)') - app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) - app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) - app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) - app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) - app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) - app.add_directive('miscnews', MiscNews) - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/third_party/python/Doc/tools/extensions/suspicious.py b/third_party/python/Doc/tools/extensions/suspicious.py deleted file mode 100644 index dfcd0da9f..000000000 --- a/third_party/python/Doc/tools/extensions/suspicious.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -Try to detect suspicious constructs, resembling markup -that has leaked into the final output. - -Suspicious lines are reported in a comma-separated-file, -``suspicious.csv``, located in the output directory. - -The file is utf-8 encoded, and each line contains four fields: - - * document name (normalized) - * line number in the source document - * problematic text - * complete line showing the problematic text in context - -It is common to find many false positives. To avoid reporting them -again and again, they may be added to the ``ignored.csv`` file -(located in the configuration directory). The file has the same -format as ``suspicious.csv`` with a few differences: - - - each line defines a rule; if the rule matches, the issue - is ignored. - - line number may be empty (that is, nothing between the - commas: ",,"). In this case, line numbers are ignored (the - rule matches anywhere in the file). - - the last field does not have to be a complete line; some - surrounding text (never more than a line) is enough for - context. - -Rules are processed sequentially. A rule matches when: - - * document names are the same - * problematic texts are the same - * line numbers are close to each other (5 lines up or down) - * the rule text is completely contained into the source line - -The simplest way to create the ignored.csv file is by copying -undesired entries from suspicious.csv (possibly trimming the last -field.) - -Copyright 2009 Gabriel A. Genellina - -""" - -import os -import re -import csv -import sys - -from docutils import nodes -from sphinx.builders import Builder -import sphinx.util - -try: # sphinx>=1.6 - from sphinx.util.logging import getLogger -except ImportError: # sphinx<1.6 - from logging import getLogger - - -detect_all = re.compile(r''' - ::(?=[^=])| # two :: (but NOT ::=) - :[a-zA-Z][a-zA-Z0-9]+| # :foo - `| # ` (seldom used by itself) - (?= (3, 0) - - -class Rule: - def __init__(self, docname, lineno, issue, line): - """A rule for ignoring issues""" - self.docname = docname # document to which this rule applies - self.lineno = lineno # line number in the original source; - # this rule matches only near that. - # None -> don't care - self.issue = issue # the markup fragment that triggered this rule - self.line = line # text of the container element (single line only) - self.used = False - - def __repr__(self): - return '{0.docname},,{0.issue},{0.line}'.format(self) - - - -class dialect(csv.excel): - """Our dialect: uses only linefeed as newline.""" - lineterminator = '\n' - - -class CheckSuspiciousMarkupBuilder(Builder): - """ - Checks for possibly invalid markup that may leak into the output. - """ - name = 'suspicious' - logger = getLogger("CheckSuspiciousMarkupBuilder") - - def init(self): - # create output file - self.log_file_name = os.path.join(self.outdir, 'suspicious.csv') - open(self.log_file_name, 'w').close() - # load database of previously ignored issues - self.load_rules(os.path.join(os.path.dirname(__file__), '..', - 'susp-ignored.csv')) - - def get_outdated_docs(self): - return self.env.found_docs - - def get_target_uri(self, docname, typ=None): - return '' - - def prepare_writing(self, docnames): - pass - - def write_doc(self, docname, doctree): - # set when any issue is encountered in this document - self.any_issue = False - self.docname = docname - visitor = SuspiciousVisitor(doctree, self) - doctree.walk(visitor) - - def finish(self): - unused_rules = [rule for rule in self.rules if not rule.used] - if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) - for rule in unused_rules: - self.logger.info(repr(rule)) - return - - def check_issue(self, line, lineno, issue): - if not self.is_ignored(line, lineno, issue): - self.report_issue(line, lineno, issue) - - def is_ignored(self, line, lineno, issue): - """Determine whether this issue should be ignored.""" - docname = self.docname - for rule in self.rules: - if rule.docname != docname: continue - if rule.issue != issue: continue - # Both lines must match *exactly*. This is rather strict, - # and probably should be improved. - # Doing fuzzy matches with levenshtein distance could work, - # but that means bringing other libraries... - # Ok, relax that requirement: just check if the rule fragment - # is contained in the document line - if rule.line not in line: continue - # Check both line numbers. If they're "near" - # this rule matches. (lineno=None means "don't care") - if (rule.lineno is not None) and \ - abs(rule.lineno - lineno) > 5: continue - # if it came this far, the rule matched - rule.used = True - return True - return False - - def report_issue(self, text, lineno, issue): - if not self.any_issue: self.logger.info() - self.any_issue = True - self.write_log_entry(lineno, issue, text) - if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) - else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( - self.docname.encode(sys.getdefaultencoding(),'replace'), - lineno, - issue.encode(sys.getdefaultencoding(),'replace'), - text.strip().encode(sys.getdefaultencoding(),'replace'))) - self.app.statuscode = 1 - - def write_log_entry(self, lineno, issue, text): - if py3: - f = open(self.log_file_name, 'a') - writer = csv.writer(f, dialect) - writer.writerow([self.docname, lineno, issue, text.strip()]) - f.close() - else: - f = open(self.log_file_name, 'ab') - writer = csv.writer(f, dialect) - writer.writerow([self.docname.encode('utf-8'), - lineno, - issue.encode('utf-8'), - text.strip().encode('utf-8')]) - f.close() - - def load_rules(self, filename): - """Load database of previously ignored issues. - - A csv file, with exactly the same format as suspicious.csv - Fields: document name (normalized), line number, issue, surrounding text - """ - self.logger.info("loading ignore rules... ", nonl=1) - self.rules = rules = [] - try: - if py3: - f = open(filename, 'r') - else: - f = open(filename, 'rb') - except IOError: - return - for i, row in enumerate(csv.reader(f)): - if len(row) != 4: - raise ValueError( - "wrong format in %s, line %d: %s" % (filename, i+1, row)) - docname, lineno, issue, text = row - if lineno: - lineno = int(lineno) - else: - lineno = None - if not py3: - docname = docname.decode('utf-8') - issue = issue.decode('utf-8') - text = text.decode('utf-8') - rule = Rule(docname, lineno, issue, text) - rules.append(rule) - f.close() - self.logger.info('done, %d rules loaded' % len(self.rules)) - - -def get_lineno(node): - """Obtain line number information for a node.""" - lineno = None - while lineno is None and node: - node = node.parent - lineno = node.line - return lineno - - -def extract_line(text, index): - """text may be a multiline string; extract - only the line containing the given character index. - - >>> extract_line("abc\ndefgh\ni", 6) - >>> 'defgh' - >>> for i in (0, 2, 3, 4, 10): - ... print extract_line("abc\ndefgh\ni", i) - abc - abc - abc - defgh - defgh - i - """ - p = text.rfind('\n', 0, index) + 1 - q = text.find('\n', index) - if q < 0: - q = len(text) - return text[p:q] - - -class SuspiciousVisitor(nodes.GenericNodeVisitor): - - lastlineno = 0 - - def __init__(self, document, builder): - nodes.GenericNodeVisitor.__init__(self, document) - self.builder = builder - - def default_visit(self, node): - if isinstance(node, (nodes.Text, nodes.image)): # direct text containers - text = node.astext() - # lineno seems to go backwards sometimes (?) - self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno) - seen = set() # don't report the same issue more than only once per line - for match in detect_all(text): - issue = match.group() - line = extract_line(text, match.start()) - if (issue, line) not in seen: - self.builder.check_issue(line, lineno, issue) - seen.add((issue, line)) - - unknown_visit = default_visit - - def visit_document(self, node): - self.lastlineno = 0 - - def visit_comment(self, node): - # ignore comments -- too much false positives. - # (although doing this could miss some errors; - # there were two sections "commented-out" by mistake - # in the Python docs that would not be caught) - raise nodes.SkipNode diff --git a/third_party/python/Doc/tools/pydoctheme/static/pydoctheme.css b/third_party/python/Doc/tools/pydoctheme/static/pydoctheme.css deleted file mode 100644 index 1d5c18e5f..000000000 --- a/third_party/python/Doc/tools/pydoctheme/static/pydoctheme.css +++ /dev/null @@ -1,194 +0,0 @@ -@import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - border-bottom: 1px solid #ccc; -} - -.inline-search { - display: inline; -} -form.inline-search input { - display: inline; -} -form.inline-search input[type="submit"] { - width: 30px; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -form.inline-search input, -div.sphinxsidebar input { - font-family: 'Lucida Grande',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #0072aa; -} - -div.body a:visited { - color: #6363bb; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, code, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt, div.body code { - border-radius: 3px; -} - -div.body tt.descname, div.body code.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { - font-weight: normal; -} - -.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -.refcount { - color: #060; -} - -.stableabi { - color: #229; -} - -.highlight { - background: none !important; -} - diff --git a/third_party/python/Doc/tools/pydoctheme/theme.conf b/third_party/python/Doc/tools/pydoctheme/theme.conf deleted file mode 100644 index 0c4388167..000000000 --- a/third_party/python/Doc/tools/pydoctheme/theme.conf +++ /dev/null @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', Arial, sans-serif -headfont = 'Lucida Grande', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff --git a/third_party/python/Doc/tools/rstlint.py b/third_party/python/Doc/tools/rstlint.py deleted file mode 100755 index 2c1816eed..000000000 --- a/third_party/python/Doc/tools/rstlint.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Check for stylistic and formal issues in .rst and .py -# files included in the documentation. -# -# 01/2009, Georg Brandl - -# TODO: - wrong versions in versionadded/changed -# - wrong markup after versionchanged directive - -import os -import re -import sys -import getopt -from os.path import join, splitext, abspath, exists -from collections import defaultdict - -directives = [ - # standard docutils ones - 'admonition', 'attention', 'caution', 'class', 'compound', 'container', - 'contents', 'csv-table', 'danger', 'date', 'default-role', 'epigraph', - 'error', 'figure', 'footer', 'header', 'highlights', 'hint', 'image', - 'important', 'include', 'line-block', 'list-table', 'meta', 'note', - 'parsed-literal', 'pull-quote', 'raw', 'replace', - 'restructuredtext-test-directive', 'role', 'rubric', 'sectnum', 'sidebar', - 'table', 'target-notes', 'tip', 'title', 'topic', 'unicode', 'warning', - # Sphinx and Python docs custom ones - 'acks', 'attribute', 'autoattribute', 'autoclass', 'autodata', - 'autoexception', 'autofunction', 'automethod', 'automodule', 'centered', - 'cfunction', 'class', 'classmethod', 'cmacro', 'cmdoption', 'cmember', - 'code-block', 'confval', 'cssclass', 'ctype', 'currentmodule', 'cvar', - 'data', 'decorator', 'decoratormethod', 'deprecated-removed', - 'deprecated(?!-removed)', 'describe', 'directive', 'doctest', 'envvar', - 'event', 'exception', 'function', 'glossary', 'highlight', 'highlightlang', - 'impl-detail', 'index', 'literalinclude', 'method', 'miscnews', 'module', - 'moduleauthor', 'opcode', 'pdbcommand', 'productionlist', - 'program', 'role', 'sectionauthor', 'seealso', 'sourcecode', 'staticmethod', - 'tabularcolumns', 'testcode', 'testoutput', 'testsetup', 'toctree', 'todo', - 'todolist', 'versionadded', 'versionchanged' -] - -all_directives = '(' + '|'.join(directives) + ')' -seems_directive_re = re.compile(r'(? 81: - # don't complain about tables, links and function signatures - if line.lstrip()[0] not in '+|' and \ - 'http://' not in line and \ - not line.lstrip().startswith(('.. function', - '.. method', - '.. cfunction')): - yield lno+1, "line too long" - - -@checker('.html', severity=2, falsepositives=True) -def check_leaked_markup(fn, lines): - """Check HTML files for leaked reST markup; this only works if - the HTML files have been built. - """ - for lno, line in enumerate(lines): - if leaked_markup_re.search(line): - yield lno+1, 'possibly leaked markup: %r' % line - - -def main(argv): - usage = '''\ -Usage: %s [-v] [-f] [-s sev] [-i path]* [path] - -Options: -v verbose (print all checked file names) - -f enable checkers that yield many false positives - -s sev only show problems with severity >= sev - -i path ignore subdir or file path -''' % argv[0] - try: - gopts, args = getopt.getopt(argv[1:], 'vfs:i:') - except getopt.GetoptError: - print(usage) - return 2 - - verbose = False - severity = 1 - ignore = [] - falsepos = False - for opt, val in gopts: - if opt == '-v': - verbose = True - elif opt == '-f': - falsepos = True - elif opt == '-s': - severity = int(val) - elif opt == '-i': - ignore.append(abspath(val)) - - if len(args) == 0: - path = '.' - elif len(args) == 1: - path = args[0] - else: - print(usage) - return 2 - - if not exists(path): - print('Error: path %s does not exist' % path) - return 2 - - count = defaultdict(int) - - for root, dirs, files in os.walk(path): - # ignore subdirs in ignore list - if abspath(root) in ignore: - del dirs[:] - continue - - for fn in files: - fn = join(root, fn) - if fn[:2] == './': - fn = fn[2:] - - # ignore files in ignore list - if abspath(fn) in ignore: - continue - - ext = splitext(fn)[1] - checkerlist = checkers.get(ext, None) - if not checkerlist: - continue - - if verbose: - print('Checking %s...' % fn) - - try: - with open(fn, 'r', encoding='utf-8') as f: - lines = list(f) - except (IOError, OSError) as err: - print('%s: cannot open: %s' % (fn, err)) - count[4] += 1 - continue - - for checker in checkerlist: - if checker.falsepositives and not falsepos: - continue - csev = checker.severity - if csev >= severity: - for lno, msg in checker(fn, lines): - print('[%d] %s:%d: %s' % (csev, fn, lno, msg)) - count[csev] += 1 - if verbose: - print() - if not count: - if severity > 1: - print('No problems with severity >= %d found.' % severity) - else: - print('No problems found.') - else: - for severity in sorted(count): - number = count[severity] - print('%d problem%s with severity %d found.' % - (number, number > 1 and 's' or '', severity)) - return int(bool(count)) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/third_party/python/Doc/tools/static/copybutton.js b/third_party/python/Doc/tools/static/copybutton.js deleted file mode 100644 index 716c9e472..000000000 --- a/third_party/python/Doc/tools/static/copybutton.js +++ /dev/null @@ -1,62 +0,0 @@ -$(document).ready(function() { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $('.highlight-python .highlight,' + - '.highlight-python3 .highlight') - var pre = div.find('pre'); - - // get the styles from the current theme - pre.parent().parent().css('position', 'relative'); - var hide_text = 'Hide the prompts and output'; - var show_text = 'Show the prompts and output'; - var border_width = pre.css('border-top-width'); - var border_style = pre.css('border-top-style'); - var border_color = pre.css('border-top-color'); - var button_styles = { - 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', - 'border-color': border_color, 'border-style': border_style, - 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', - 'border-radius': '0 3px 0 0' - } - - // create and add the button to all the code blocks that contain >>> - div.each(function(index) { - var jthis = $(this); - if (jthis.find('.gp').length > 0) { - var button = $('>>>'); - button.css(button_styles) - button.attr('title', hide_text); - button.data('hidden', 'false'); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis.find('pre:has(.gt)').contents().filter(function() { - return ((this.nodeType == 3) && (this.data.trim().length > 0)); - }).wrap(''); - }); - - // define the behavior of the button when it's clicked - $('.copybutton').click(function(e){ - e.preventDefault(); - var button = $(this); - if (button.data('hidden') === 'false') { - // hide the code output - button.parent().find('.go, .gp, .gt').hide(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); - button.css('text-decoration', 'line-through'); - button.attr('title', show_text); - button.data('hidden', 'true'); - } else { - // show the code output - button.parent().find('.go, .gp, .gt').show(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); - button.css('text-decoration', 'none'); - button.attr('title', hide_text); - button.data('hidden', 'false'); - } - }); -}); - diff --git a/third_party/python/Doc/tools/static/py.png b/third_party/python/Doc/tools/static/py.png deleted file mode 100644 index 93e4a02c3..000000000 Binary files a/third_party/python/Doc/tools/static/py.png and /dev/null differ diff --git a/third_party/python/Doc/tools/static/sidebar.js b/third_party/python/Doc/tools/static/sidebar.js deleted file mode 100644 index e8d58f4bf..000000000 --- a/third_party/python/Doc/tools/static/sidebar.js +++ /dev/null @@ -1,193 +0,0 @@ -/* - * sidebar.js - * ~~~~~~~~~~ - * - * This script makes the Sphinx sidebar collapsible and implements intelligent - * scrolling. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in - * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to - * collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the - * width of the sidebar and the margin-left of the document are decreased. - * When the sidebar is expanded the opposite happens. This script saves a - * per-browser/per-session cookie used to remember the position of the sidebar - * among the pages. Once the browser is closed the cookie is deleted and the - * position reset to the default (expanded). - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -$(function() { - // global elements used by the functions. - // the 'sidebarbutton' element is defined as global after its - // creation, in the add_sidebar_button function - var jwindow = $(window); - var jdocument = $(document); - var bodywrapper = $('.bodywrapper'); - var sidebar = $('.sphinxsidebar'); - var sidebarwrapper = $('.sphinxsidebarwrapper'); - - // original margin-left of the bodywrapper and width of the sidebar - // with the sidebar expanded - var bw_margin_expanded = bodywrapper.css('margin-left'); - var ssb_width_expanded = sidebar.width(); - - // margin-left of the bodywrapper and width of the sidebar - // with the sidebar collapsed - var bw_margin_collapsed = '.8em'; - var ssb_width_collapsed = '.8em'; - - // colors used by the current theme - var dark_color = '#AAAAAA'; - var light_color = '#CCCCCC'; - - function get_viewport_height() { - if (window.innerHeight) - return window.innerHeight; - else - return jwindow.height(); - } - - function sidebar_is_collapsed() { - return sidebarwrapper.is(':not(:visible)'); - } - - function toggle_sidebar() { - if (sidebar_is_collapsed()) - expand_sidebar(); - else - collapse_sidebar(); - // adjust the scrolling of the sidebar - scroll_sidebar(); - } - - function collapse_sidebar() { - sidebarwrapper.hide(); - sidebar.css('width', ssb_width_collapsed); - bodywrapper.css('margin-left', bw_margin_collapsed); - sidebarbutton.css({ - 'margin-left': '0', - 'height': bodywrapper.height(), - 'border-radius': '5px' - }); - sidebarbutton.find('span').text('»'); - sidebarbutton.attr('title', _('Expand sidebar')); - document.cookie = 'sidebar=collapsed'; - } - - function expand_sidebar() { - bodywrapper.css('margin-left', bw_margin_expanded); - sidebar.css('width', ssb_width_expanded); - sidebarwrapper.show(); - sidebarbutton.css({ - 'margin-left': ssb_width_expanded-12, - 'height': bodywrapper.height(), - 'border-radius': '0 5px 5px 0' - }); - sidebarbutton.find('span').text('«'); - sidebarbutton.attr('title', _('Collapse sidebar')); - //sidebarwrapper.css({'padding-top': - // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); - document.cookie = 'sidebar=expanded'; - } - - function add_sidebar_button() { - sidebarwrapper.css({ - 'float': 'left', - 'margin-right': '0', - 'width': ssb_width_expanded - 28 - }); - // create the button - sidebar.append( - '
    «
    ' - ); - var sidebarbutton = $('#sidebarbutton'); - // find the height of the viewport to center the '<<' in the page - var viewport_height = get_viewport_height(); - var sidebar_offset = sidebar.offset().top; - var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); - sidebarbutton.find('span').css({ - 'display': 'block', - 'position': 'fixed', - 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 - }); - - sidebarbutton.click(toggle_sidebar); - sidebarbutton.attr('title', _('Collapse sidebar')); - sidebarbutton.css({ - 'border-radius': '0 5px 5px 0', - 'color': '#444444', - 'background-color': '#CCCCCC', - 'font-size': '1.2em', - 'cursor': 'pointer', - 'height': sidebar_height, - 'padding-top': '1px', - 'padding-left': '1px', - 'margin-left': ssb_width_expanded - 12 - }); - - sidebarbutton.hover( - function () { - $(this).css('background-color', dark_color); - }, - function () { - $(this).css('background-color', light_color); - } - ); - } - - function set_position_from_cookie() { - if (!document.cookie) - return; - var items = document.cookie.split(';'); - for(var k=0; k wintop && curbot > winbot) { - sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); - } - else if (curtop < wintop && curbot < winbot) { - sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, - jdocument.height() - sidebar_height - 200])); - } - } - } - jwindow.scroll(scroll_sidebar); -}); diff --git a/third_party/python/Doc/tools/static/switchers.js b/third_party/python/Doc/tools/static/switchers.js deleted file mode 100644 index 30801917e..000000000 --- a/third_party/python/Doc/tools/static/switchers.js +++ /dev/null @@ -1,155 +0,0 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - var version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - var all_versions = { - '3.10': 'dev (3.10)', - '3.9': 'pre (3.9)', - '3.8': '3.8', - '3.7': '3.7', - '3.6': '3.6', - '2.7': '2.7', - }; - - var all_languages = { - 'en': 'English', - 'fr': 'French', - 'ja': 'Japanese', - 'ko': 'Korean', - 'pt-br': 'Brazilian Portuguese', - }; - - function build_version_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function build_language_select(current_language) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + current_language + selected_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - url.replace('.org/' + current_language + current_version, - '.org/' + selected_version), - 'https://docs.python.org/' + current_language + selected_version, - 'https://docs.python.org/' + selected_version, - 'https://docs.python.org/' - ]); - } - } - - function on_language_switch() { - var selected_language = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - if (selected_language == 'en/') // Special 'default' case for english. - selected_language = ''; - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + selected_language + current_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://docs.python.org/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url(url) { - var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)'; - var match = url.match(language_regexp); - if (match !== null) - return match[1]; - return ''; - } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_in_url(url) { - var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)'; - var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; - var match = url.match(version_regexp); - if (match !== null) - return match[1]; - return '' - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var language_segment = language_segment_from_url(window.location.href); - var current_language = language_segment.replace(/\/+$/g, '') || 'en'; - var version = release.substr(0, 3); - var version_select = build_version_select(version, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var language_select = build_language_select(current_language); - - $('.language_switcher_placeholder').html(language_select); - $('.language_switcher_placeholder select').bind('change', on_language_switch); - }); -})(); diff --git a/third_party/python/Doc/tools/susp-ignored.csv b/third_party/python/Doc/tools/susp-ignored.csv deleted file mode 100644 index ed434ce77..000000000 --- a/third_party/python/Doc/tools/susp-ignored.csv +++ /dev/null @@ -1,332 +0,0 @@ -c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)" -c-api/list,,:high,list[low:high] -c-api/sequence,,:i2,del o[i1:i2] -c-api/sequence,,:i2,o[i1:i2] -c-api/unicode,,:end,str[start:end] -c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,267,`,This is the description of the ``foobar`` package. -distutils/setupscript,,::, -extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" -extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" -extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {" -extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {" -faq/programming,,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" -faq/programming,,::,for x in sequence[::-1]: -faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," -faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," -faq/windows,,:d48eceb,"Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32" -howto/cporting,,:encode,"if (!PyArg_ParseTuple(args, ""O:encode_object"", &myobj))" -howto/cporting,,:say,"if (!PyArg_ParseTuple(args, ""U:say_hello"", &name))" -howto/curses,,:black,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:red,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:green,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:yellow,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:blue,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:magenta,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:cyan,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:white,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/instrumentation,,::,python$target:::function-entry -howto/instrumentation,,:function,python$target:::function-entry -howto/instrumentation,,::,python$target:::function-return -howto/instrumentation,,:function,python$target:::function-return -howto/instrumentation,,:call,156641360502280 function-entry:call_stack.py:start:23 -howto/instrumentation,,:start,156641360502280 function-entry:call_stack.py:start:23 -howto/instrumentation,,:function,156641360518804 function-entry: call_stack.py:function_1:1 -howto/instrumentation,,:function,156641360532797 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360546807 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360563367 function-return: call_stack.py:function_1:2 -howto/instrumentation,,:function,156641360578365 function-entry: call_stack.py:function_2:5 -howto/instrumentation,,:function,156641360591757 function-entry: call_stack.py:function_1:1 -howto/instrumentation,,:function,156641360605556 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360617482 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360629814 function-return: call_stack.py:function_1:2 -howto/instrumentation,,:function,156641360642285 function-return: call_stack.py:function_2:6 -howto/instrumentation,,:function,156641360656770 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360669707 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360687853 function-entry: call_stack.py:function_4:13 -howto/instrumentation,,:function,156641360700719 function-return: call_stack.py:function_4:14 -howto/instrumentation,,:function,156641360719640 function-entry: call_stack.py:function_5:18 -howto/instrumentation,,:function,156641360732567 function-return: call_stack.py:function_5:21 -howto/instrumentation,,:call,156641360747370 function-return:call_stack.py:start:28 -howto/instrumentation,,:start,156641360747370 function-return:call_stack.py:start:28 -howto/ipaddress,,:DB8,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,::,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,:db8,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('::1') -howto/ipaddress,,:db8,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/96') -howto/ipaddress,,::,IPv6Network('2001:db8::/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/128') -howto/ipaddress,,::,IPv6Network('2001:db8::/128') -howto/ipaddress,,:db8,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,::,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,::,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,:db8,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:ffff,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('::ffff:ffff') -howto/ipaddress,,:db8,'2001:db8::/96' -howto/ipaddress,,::,'2001:db8::/96' -howto/ipaddress,,:db8,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,'2001:db8::1' -howto/ipaddress,,::,'2001:db8::1' -howto/ipaddress,,:db8,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,::,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('2001:db8::ffff:ffff') -howto/logging,,:And,"WARNING:And this, too" -howto/logging,,:And,"WARNING:root:And this, too" -howto/logging,,:Doing,INFO:root:Doing something -howto/logging,,:Finished,INFO:root:Finished -howto/logging,,:logger,severity:logger name:message -howto/logging,,:Look,WARNING:root:Look before you leap! -howto/logging,,:message,severity:logger name:message -howto/logging,,:root,DEBUG:root:This message should go to the log file -howto/logging,,:root,INFO:root:Doing something -howto/logging,,:root,INFO:root:Finished -howto/logging,,:root,INFO:root:So should this -howto/logging,,:root,INFO:root:Started -howto/logging,,:root,"WARNING:root:And this, too" -howto/logging,,:root,WARNING:root:Look before you leap! -howto/logging,,:root,WARNING:root:Watch out! -howto/logging,,:So,INFO:root:So should this -howto/logging,,:So,INFO:So should this -howto/logging,,:Started,INFO:root:Started -howto/logging,,:This,DEBUG:root:This message should go to the log file -howto/logging,,:This,DEBUG:This message should appear on the console -howto/logging,,:Watch,WARNING:root:Watch out! -howto/pyporting,,::,Programming Language :: Python :: 2 -howto/pyporting,,::,Programming Language :: Python :: 3 -howto/regex,,::, -howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"""joe:password@example.com""" -library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," -library/bisect,32,:hi,all(val >= x for val in a[i:hi]) -library/bisect,42,:hi,all(val > x for val in a[i:hi]) -library/configparser,,:home,my_dir: ${Common:home_dir}/twosheds -library/configparser,,:option,${section:option} -library/configparser,,:path,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:Python,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:system,path: ${Common:system_dir}/Library/Frameworks/ -library/datetime,,:MM, -library/datetime,,:SS, -library/decimal,,:optional,"trailneg:optional trailing minus indicator" -library/difflib,,:ahi,a[alo:ahi] -library/difflib,,:bhi,b[blo:bhi] -library/difflib,,:i1, -library/difflib,,:i2, -library/difflib,,:j2, -library/doctest,,`,``factorial`` from the ``example`` module: -library/doctest,,`,The ``example`` module -library/doctest,,`,Using ``factorial`` -library/exceptions,,:err,err.object[err.start:err.end] -library/functions,,:step,a[start:stop:step] -library/functions,,:stop,"a[start:stop, i]" -library/functions,,:stop,a[start:stop:step] -library/hashlib,,:LEAF,"h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH," -library/http.client,,:port,host:port -library/http.cookies,,`,!#$%&'*+-.^_`|~: -library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS" -library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS" -library/inspect,,:int,">>> def foo(a, *, b:int, **kwargs):" -library/inspect,,:int,"'(a, *, b:int, **kwargs)'" -library/inspect,,:int,'b:int' -library/ipaddress,,:db8,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,::,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,:db8,IPv6Address('2001:db8::') -library/ipaddress,,::,IPv6Address('2001:db8::') -library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,IPv6Address('2001:db8::1000') -library/ipaddress,,::,IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,"""::abc:7:def""" -library/ipaddress,,:def,"""::abc:7:def""" -library/ipaddress,,::,::FFFF/96 -library/ipaddress,,::,2002::/16 -library/ipaddress,,::,2001::/32 -library/ipaddress,,::,>>> str(ipaddress.IPv6Address('::1')) -library/ipaddress,,::,'::1' -library/ipaddress,,:ff00,ffff:ff00:: -library/ipaddress,,:db00,2001:db00::0/24 -library/ipaddress,,::,2001:db00::0/24 -library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: -library/ipaddress,,::,2001:db00::0/ffff:ff00:: -library/itertools,,:step,elements from seq[start:stop:step] -library/itertools,,:stop,elements from seq[start:stop:step] -library/logging.handlers,,:port,host:port -library/mmap,,:i2,obj[i1:i2] -library/multiprocessing,,`,# Add more tasks using `put()` -library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)" -library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy -library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy -library/multiprocessing,,`,# register the generator function baz; use `GeneratorProxy` to make proxies -library/nntplib,,:bytes,:bytes -library/nntplib,,:lines,:lines -library/optparse,,:len,"del parser.rargs[:len(value)]" -library/os.path,,:foo,c:foo -library/pathlib,,:bar,">>> PureWindowsPath('c:/Windows', 'd:bar')" -library/pathlib,,:bar,PureWindowsPath('d:bar') -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').root -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').anchor -library/pdb,,:lineno,filename:lineno -library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")" -library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS""" -library/pprint,,::,"'Programming Language :: Python :: 2.6'," -library/pprint,,::,"'Programming Language :: Python :: 2.7'," -library/pprint,225,::,"'classifiers': ['Development Status :: 3 - Alpha'," -library/pprint,225,::,"'Intended Audience :: Developers'," -library/pprint,225,::,"'License :: OSI Approved :: MIT License'," -library/pprint,225,::,"'Programming Language :: Python :: 2'," -library/pprint,225,::,"'Programming Language :: Python :: 3'," -library/pprint,225,::,"'Programming Language :: Python :: 3.2'," -library/pprint,225,::,"'Programming Language :: Python :: 3.3'," -library/pprint,225,::,"'Programming Language :: Python :: 3.4'," -library/pprint,225,::,"'Topic :: Software Development :: Build Tools']," -library/profile,,:lineno,filename:lineno(function) -library/pyexpat,,:elem1, -library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">" -library/random,,:len,new_diff = mean(combined[:len(drug)]) - mean(combined[len(drug):]) -library/smtplib,,:port,method must support that as well as a regular host:port -library/socket,,::,'5aef:2b::8' -library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" -library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" -library/sqlite3,,:memory, -library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" -library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" -library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" -library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." -library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" -library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState -library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com -library/ssl,,:Some,"Locality Name (eg, city) []:Some City" -library/ssl,,:US,Country Name (2 letter code) [AU]:US -library/stdtypes,,:end,s[start:end] -library/stdtypes,,::,>>> hash(v[::-2]) == hash(b'abcefg'[::-2]) -library/stdtypes,,:len,s[len(s):len(s)] -library/stdtypes,,::,>>> y = m[::2] -library/stdtypes,,::,>>> z = y[::-2] -library/subprocess,,`,"output=`dmesg | grep hda`" -library/subprocess,,`,"output=`mycmd myarg`" -library/tarfile,,:bz2, -library/tarfile,,:compression,filemode[:compression] -library/tarfile,,:gz, -library/tarfile,,:xz,'a:xz' -library/tarfile,,:xz,'r:xz' -library/tarfile,,:xz,'w:xz' -library/time,,:mm, -library/time,,:ss, -library/tracemalloc,,:limit,"for index, stat in enumerate(top_stats[:limit], 1):" -library/turtle,,::,Example:: -library/unittest,,:foo,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:first,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:foo,'ERROR:foo.bar:second message']) -library/unittest,,:second,'ERROR:foo.bar:second message']) -library/urllib.request,,:close,Connection:close -library/urllib.request,,:port,:port -library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" -library/urllib.request,,:password,"""joe:password@python.org""" -library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 -library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" -library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" -library/venv,,:param,":param nopip: If True, pip is not installed into the created" -library/venv,,:param,:param context: The information for the virtual environment -library/xmlrpc.client,,:nil,ex:nil -library/xmlrpc.client,,:pass,http://user:pass@host:port/path -library/xmlrpc.client,,:pass,user:pass -library/xmlrpc.client,,:port,http://user:pass@host:port/path -license,,`,"``Software''), to deal in the Software without restriction, including" -license,,`,"THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND," -license,,`,* THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND -license,,`,THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -license,,`,* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -license,,`,THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND -license,,:zooko,mailto:zooko@zooko.com -reference/expressions,,:index,x[index:index] -reference/lexical_analysis,,`,$ ? ` -reference/lexical_analysis,,:fileencoding,# vim:fileencoding= -tutorial/datastructures,,:value,It is also possible to delete a key:value -tutorial/datastructures,,:value,key:value pairs within the braces adds initial key:value pairs -tutorial/stdlib2,,:config,"logging.warning('Warning:config file %s not found', 'server.conf')" -tutorial/stdlib2,,:config,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:Critical,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:Error,ERROR:root:Error occurred -tutorial/stdlib2,,:root,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:root,ERROR:root:Error occurred -tutorial/stdlib2,,:root,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:start,extra = data[start:start+extra_size] -tutorial/stdlib2,,:start,"fields = struct.unpack('>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo') -whatsnew/2.7,,:Sunday,'2009:4:Sunday' -whatsnew/2.7,,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/2.7,,::,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/3.2,,:affe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:affe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:beef,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:beef,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:cafe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:cafe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:deaf,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:deaf,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:directory,${buildout:directory}/downloads/dist -whatsnew/3.2,,::,"$ export PYTHONWARNINGS='ignore::RuntimeWarning::,once::UnicodeWarning::'" -whatsnew/3.2,,:feed,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" -whatsnew/3.2,,:location,zope9-location = ${zope9:location} -whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf -library/re,,`,!#$%&'*+-.^_`|~: -library/re,,`,\!\#\$\%\&\'\*\+\-\.\^_\`\|\~\: -library/tarfile,,:xz,'x:xz' -library/xml.etree.elementtree,,:sometag,prefix:sometag -library/xml.etree.elementtree,,:fictional,"Lancelot -library/xml.etree.elementtree,,:character,Archie Leach -library/xml.etree.elementtree,,:character,Sir Robin -library/xml.etree.elementtree,,:character,Gunther -library/xml.etree.elementtree,,:character,Commander Clement -library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):" -library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)" -library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):" -library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" -library/zipapp,,:fn,"pkg.mod:fn" -library/zipapp,,:callable,"pkg.module:callable" -library/stdtypes,,::,>>> m[::2].tolist() -library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: -whatsnew/3.5,,:root,'WARNING:root:warning\n' -whatsnew/3.5,,:warning,'WARNING:root:warning\n' -whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') -whatsnew/3.5,,:root,ERROR:root:exception -whatsnew/3.5,,:exception,ERROR:root:exception diff --git a/third_party/python/Doc/tools/templates/customsourcelink.html b/third_party/python/Doc/tools/templates/customsourcelink.html deleted file mode 100644 index 21af621ef..000000000 --- a/third_party/python/Doc/tools/templates/customsourcelink.html +++ /dev/null @@ -1,13 +0,0 @@ -{%- if show_source and has_source and sourcename %} - -{%- endif %} diff --git a/third_party/python/Doc/tools/templates/download.html b/third_party/python/Doc/tools/templates/download.html deleted file mode 100644 index d49ebdd14..000000000 --- a/third_party/python/Doc/tools/templates/download.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "layout.html" %} -{% set title = 'Download' %} -{% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} -{% else %} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} -{% endif %} - -{% block body %} -

    Download Python {{ release }} Documentation

    - -{% if last_updated %}

    Last updated on: {{ last_updated }}.

    {% endif %} - -

    To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table. The numbers -in the table are the size of the download files in megabytes.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FormatPacked as .zipPacked as .tar.bz2
    PDF (US-Letter paper size)Download (ca. 13 MB)Download (ca. 13 MB)
    PDF (A4 paper size)Download (ca. 13 MB)Download (ca. 13 MB)
    HTMLDownload (ca. 9 MB)Download (ca. 6 MB)
    Plain TextDownload (ca. 3 MB)Download (ca. 2 MB)
    EPUBDownload (ca. 5.5 MB)
    - -

    These archives contain all the content in the documentation.

    - -

    HTML Help (.chm) files are made available in the "Windows" section -on the Python -download page.

    - - -

    Unpacking

    - -

    Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The InfoZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.

    - -

    Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the InfoZIP zip program.

    - - -

    Problems

    - -

    If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.

    -{% endblock %} diff --git a/third_party/python/Doc/tools/templates/dummy.html b/third_party/python/Doc/tools/templates/dummy.html deleted file mode 100644 index 8d94137b0..000000000 --- a/third_party/python/Doc/tools/templates/dummy.html +++ /dev/null @@ -1,7 +0,0 @@ -This file is not an actual template, but used to add some -texts in extensions to sphinx.pot file. - -In extensions/pyspecific.py: - -{% trans %}CPython implementation detail:{% endtrans %} -{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} diff --git a/third_party/python/Doc/tools/templates/indexcontent.html b/third_party/python/Doc/tools/templates/indexcontent.html deleted file mode 100644 index d795c0a55..000000000 --- a/third_party/python/Doc/tools/templates/indexcontent.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "layout.html" %} -{%- block htmltitle -%} -{{ shorttitle }} -{%- endblock -%} -{% block body %} -

    {{ docstitle|e }}

    -

    - {% trans %}Welcome! This is the documentation for Python {{ release }}.{% endtrans %} -

    -

    {% trans %}Parts of the documentation:{% endtrans %}

    - - -
    -

    {% trans %}What's new in Python {{ version }}?{% endtrans %}
    - {% trans whatsnew_index=pathto("whatsnew/index") %}or all "What's new" documents since 2.0{% endtrans %}

    -

    {% trans %}Tutorial{% endtrans %}
    - {% trans %}start here{% endtrans %}

    -

    {% trans %}Library Reference{% endtrans %}
    - {% trans %}keep this under your pillow{% endtrans %}

    -

    {% trans %}Language Reference{% endtrans %}
    - {% trans %}describes syntax and language elements{% endtrans %}

    -

    {% trans %}Python Setup and Usage{% endtrans %}
    - {% trans %}how to use Python on different platforms{% endtrans %}

    -

    {% trans %}Python HOWTOs{% endtrans %}
    - {% trans %}in-depth documents on specific topics{% endtrans %}

    -     		-

    {% trans %}Installing Python Modules{% endtrans %}
    - {% trans %}installing from the Python Package Index & other sources{% endtrans %}

    -

    {% trans %}Distributing Python Modules{% endtrans %}
    - {% trans %}publishing modules for installation by others{% endtrans %}

    -

    {% trans %}Extending and Embedding{% endtrans %}
    - {% trans %}tutorial for C/C++ programmers{% endtrans %}

    -

    {% trans %}Python/C API{% endtrans %}
    - {% trans %}reference for C/C++ programmers{% endtrans %}

    -

    {% trans %}FAQs{% endtrans %}
    - {% trans %}frequently asked questions (with answers!){% endtrans %}

    -
    - -

    {% trans %}Indices and tables:{% endtrans %}

    - - -
    -

    {% trans %}Global Module Index{% endtrans %}
    - {% trans %}quick access to all modules{% endtrans %}

    -

    {% trans %}General Index{% endtrans %}
    - {% trans %}all functions, classes, terms{% endtrans %}

    -

    {% trans %}Glossary{% endtrans %}
    - {% trans %}the most important terms explained{% endtrans %}

    -     		-

    {% trans %}Search page{% endtrans %}
    - {% trans %}search this documentation{% endtrans %}

    -

    {% trans %}Complete Table of Contents{% endtrans %}
    - {% trans %}lists all sections and subsections{% endtrans %}

    -
    - -

    {% trans %}Meta information:{% endtrans %}

    - - -
    -

    {% trans %}Reporting bugs{% endtrans %}

    -

    {% trans %}About the documentation{% endtrans %}

    -     		-

    {% trans %}History and License of Python{% endtrans %}

    -

    {% trans %}Copyright{% endtrans %}

    -
    -{% endblock %} diff --git a/third_party/python/Doc/tools/templates/indexsidebar.html b/third_party/python/Doc/tools/templates/indexsidebar.html deleted file mode 100644 index 2ef90f094..000000000 --- a/third_party/python/Doc/tools/templates/indexsidebar.html +++ /dev/null @@ -1,21 +0,0 @@ -

    {% trans %}Download{% endtrans %}

    -

    {% trans %}Download these documents{% endtrans %}

    -

    {% trans %}Docs by version{% endtrans %}

    - - -

    {% trans %}Other resources{% endtrans %}

    - diff --git a/third_party/python/Doc/tools/templates/layout.html b/third_party/python/Doc/tools/templates/layout.html deleted file mode 100644 index 7a7feb52e..000000000 --- a/third_party/python/Doc/tools/templates/layout.html +++ /dev/null @@ -1,129 +0,0 @@ -{% extends "!layout.html" %} - -{% block header %} -{%- if outdated %} -
    - {% trans %}This document is for an old version of Python that is no longer supported. - You should upgrade, and read the {% endtrans %} - {% trans %} Python documentation for the current stable release{% endtrans %}. -
    -{%- endif %} -{% endblock %} - -{% block rootrellink %} -
    • -
  • Python{{ reldelim1 }}
    • -
  • - {%- if switchers is defined %} - {{ language or 'en' }} - {{ release }} - {% trans %}Documentation {% endtrans %}{{ reldelim1 }} - {%- else %} - {{ shorttitle }}{{ reldelim1 }} - {%- endif %} -
    • -{% endblock %} -{%- macro searchbox() %} -{# modified from sphinx/themes/basic/searchbox.html #} - {%- if builder != "htmlhelp" %} - - - {%- endif %} -{%- endmacro %} -{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbaritems %} - {%- if pagename != "search" and builder != "singlehtml" and builder != "htmlhelp" %} -
  • - {{ searchbox() }} - {{ reldelim2 }} -
    • - {%- endif %} -{% endblock %} -{% block extrahead %} - - - {% if builder != "htmlhelp" %} - {% if not embedded %}{% endif %} - {% if switchers is defined and not embedded %}{% endif %} - {% if pagename == 'whatsnew/changelog' and not embedded %} - - {% endif %} - {% endif %} -{{ super() }} -{% endblock %} -{% block footer %} -
    - © {% trans %}Copyright{% endtrans %} {{ copyright|e }}. -
    - {% trans %}The Python Software Foundation is a non-profit corporation.{% endtrans %} - {% trans %}Please donate.{% endtrans %} -
    - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {% trans pathto_bugs=pathto('bugs') %}Found a bug?{% endtrans %} -
    - {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %} -
    -{% endblock %} diff --git a/third_party/python/Doc/tools/templates/opensearch.xml b/third_party/python/Doc/tools/templates/opensearch.xml deleted file mode 100644 index 7a5cdddd2..000000000 --- a/third_party/python/Doc/tools/templates/opensearch.xml +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "!opensearch.xml" %} -{% block extra -%} -https://www.python.org/images/favicon16x16.ico -{%- endblock %} diff --git a/third_party/python/Doc/tutorial/appendix.rst b/third_party/python/Doc/tutorial/appendix.rst deleted file mode 100644 index 241a81203..000000000 --- a/third_party/python/Doc/tutorial/appendix.rst +++ /dev/null @@ -1,124 +0,0 @@ -.. _tut-appendix: - -******** -Appendix -******** - - -.. _tut-interac: - -Interactive Mode -================ - -.. _tut-error: - -Error Handling --------------- - -When an error occurs, the interpreter prints an error message and a stack trace. -In interactive mode, it then returns to the primary prompt; when input came from -a file, it exits with a nonzero exit status after printing the stack trace. -(Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement -are not errors in this context.) Some errors are unconditionally fatal and -cause an exit with a nonzero exit; this applies to internal inconsistencies and -some cases of running out of memory. All error messages are written to the -standard error stream; normal output from executed commands is written to -standard output. - -Typing the interrupt character (usually :kbd:`Control-C` or :kbd:`Delete`) to the primary or -secondary prompt cancels the input and returns to the primary prompt. [#]_ -Typing an interrupt while a command is executing raises the -:exc:`KeyboardInterrupt` exception, which may be handled by a :keyword:`try` -statement. - - -.. _tut-scripts: - -Executable Python Scripts -------------------------- - -On BSD'ish Unix systems, Python scripts can be made directly executable, like -shell scripts, by putting the line :: - - #!/usr/bin/env python3.5 - -(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning -of the script and giving the file an executable mode. The ``#!`` must be the -first two characters of the file. On some platforms, this first line must end -with a Unix-style line ending (``'\n'``), not a Windows (``'\r\n'``) line -ending. Note that the hash, or pound, character, ``'#'``, is used to start a -comment in Python. - -The script can be given an executable mode, or permission, using the -:program:`chmod` command. - -.. code-block:: shell-session - - $ chmod +x myscript.py - -On Windows systems, there is no notion of an "executable mode". The Python -installer automatically associates ``.py`` files with ``python.exe`` so that -a double-click on a Python file will run it as a script. The extension can -also be ``.pyw``, in that case, the console window that normally appears is -suppressed. - - -.. _tut-startup: - -The Interactive Startup File ----------------------------- - -When you use Python interactively, it is frequently handy to have some standard -commands executed every time the interpreter is started. You can do this by -setting an environment variable named :envvar:`PYTHONSTARTUP` to the name of a -file containing your start-up commands. This is similar to the :file:`.profile` -feature of the Unix shells. - -This file is only read in interactive sessions, not when Python reads commands -from a script, and not when :file:`/dev/tty` is given as the explicit source of -commands (which otherwise behaves like an interactive session). It is executed -in the same namespace where interactive commands are executed, so that objects -that it defines or imports can be used without qualification in the interactive -session. You can also change the prompts ``sys.ps1`` and ``sys.ps2`` in this -file. - -If you want to read an additional start-up file from the current directory, you -can program this in the global start-up file using code like ``if -os.path.isfile('.pythonrc.py'): exec(open('.pythonrc.py').read())``. -If you want to use the startup file in a script, you must do this explicitly -in the script:: - - import os - filename = os.environ.get('PYTHONSTARTUP') - if filename and os.path.isfile(filename): - with open(filename) as fobj: - startup_file = fobj.read() - exec(startup_file) - - -.. _tut-customize: - -The Customization Modules -------------------------- - -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location -of your user site-packages directory. Start Python and run this code:: - - >>> import site - >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' - -Now you can create a file named :file:`usercustomize.py` in that directory and -put anything you want in it. It will affect every invocation of Python, unless -it is started with the :option:`-s` option to disable the automatic import. - -:mod:`sitecustomize` works in the same way, but is typically created by an -administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` -module for more details. - - -.. rubric:: Footnotes - -.. [#] A problem with the GNU Readline package may prevent this. diff --git a/third_party/python/Doc/tutorial/appetite.rst b/third_party/python/Doc/tutorial/appetite.rst deleted file mode 100644 index 26e5168ab..000000000 --- a/third_party/python/Doc/tutorial/appetite.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _tut-intro: - -********************** -Whetting Your Appetite -********************** - -If you do much work on computers, eventually you find that there's some task -you'd like to automate. For example, you may wish to perform a -search-and-replace over a large number of text files, or rename and rearrange a -bunch of photo files in a complicated way. Perhaps you'd like to write a small -custom database, or a specialized GUI application, or a simple game. - -If you're a professional software developer, you may have to work with several -C/C++/Java libraries but find the usual write/compile/test/re-compile cycle is -too slow. Perhaps you're writing a test suite for such a library and find -writing the testing code a tedious task. Or maybe you've written a program that -could use an extension language, and you don't want to design and implement a -whole new language for your application. - -Python is just the language for you. - -You could write a Unix shell script or Windows batch files for some of these -tasks, but shell scripts are best at moving around files and changing text data, -not well-suited for GUI applications or games. You could write a C/C++/Java -program, but it can take a lot of development time to get even a first-draft -program. Python is simpler to use, available on Windows, Mac OS X, and Unix -operating systems, and will help you get the job done more quickly. - -Python is simple to use, but it is a real programming language, offering much -more structure and support for large programs than shell scripts or batch files -can offer. On the other hand, Python also offers much more error checking than -C, and, being a *very-high-level language*, it has high-level data types built -in, such as flexible arrays and dictionaries. Because of its more general data -types Python is applicable to a much larger problem domain than Awk or even -Perl, yet many things are at least as easy in Python as in those languages. - -Python allows you to split your program into modules that can be reused in other -Python programs. It comes with a large collection of standard modules that you -can use as the basis of your programs --- or as examples to start learning to -program in Python. Some of these modules provide things like file I/O, system -calls, sockets, and even interfaces to graphical user interface toolkits like -Tk. - -Python is an interpreted language, which can save you considerable time during -program development because no compilation and linking is necessary. The -interpreter can be used interactively, which makes it easy to experiment with -features of the language, to write throw-away programs, or to test functions -during bottom-up program development. It is also a handy desk calculator. - -Python enables programs to be written compactly and readably. Programs written -in Python are typically much shorter than equivalent C, C++, or Java programs, -for several reasons: - -* the high-level data types allow you to express complex operations in a single - statement; - -* statement grouping is done by indentation instead of beginning and ending - brackets; - -* no variable or argument declarations are necessary. - -Python is *extensible*: if you know how to program in C it is easy to add a new -built-in function or module to the interpreter, either to perform critical -operations at maximum speed, or to link Python programs to libraries that may -only be available in binary form (such as a vendor-specific graphics library). -Once you are really hooked, you can link the Python interpreter into an -application written in C and use it as an extension or command language for that -application. - -By the way, the language is named after the BBC show "Monty Python's Flying -Circus" and has nothing to do with reptiles. Making references to Monty -Python skits in documentation is not only allowed, it is encouraged! - -Now that you are all excited about Python, you'll want to examine it in some -more detail. Since the best way to learn a language is to use it, the tutorial -invites you to play with the Python interpreter as you read. - -In the next chapter, the mechanics of using the interpreter are explained. This -is rather mundane information, but essential for trying out the examples shown -later. - -The rest of the tutorial introduces various features of the Python language and -system through examples, beginning with simple expressions, statements and data -types, through functions and modules, and finally touching upon advanced -concepts like exceptions and user-defined classes. - - diff --git a/third_party/python/Doc/tutorial/classes.rst b/third_party/python/Doc/tutorial/classes.rst deleted file mode 100644 index 1acc5a8ea..000000000 --- a/third_party/python/Doc/tutorial/classes.rst +++ /dev/null @@ -1,922 +0,0 @@ -.. _tut-classes: - -******* -Classes -******* - -Classes provide a means of bundling data and functionality together. Creating -a new class creates a new *type* of object, allowing new *instances* of that -type to be made. Each class instance can have attributes attached to it for -maintaining its state. Class instances can also have methods (defined by its -class) for modifying its state. - -Compared with other programming languages, Python's class mechanism adds classes -with a minimum of new syntax and semantics. It is a mixture of the class -mechanisms found in C++ and Modula-3. Python classes provide all the standard -features of Object Oriented Programming: the class inheritance mechanism allows -multiple base classes, a derived class can override any methods of its base -class or classes, and a method can call the method of a base class with the same -name. Objects can contain arbitrary amounts and kinds of data. As is true for -modules, classes partake of the dynamic nature of Python: they are created at -runtime, and can be modified further after creation. - -In C++ terminology, normally class members (including the data members) are -*public* (except see below :ref:`tut-private`), and all member functions are -*virtual*. As in Modula-3, there are no shorthands for referencing the object's -members from its methods: the method function is declared with an explicit first -argument representing the object, which is provided implicitly by the call. As -in Smalltalk, classes themselves are objects. This provides semantics for -importing and renaming. Unlike C++ and Modula-3, built-in types can be used as -base classes for extension by the user. Also, like in C++, most built-in -operators with special syntax (arithmetic operators, subscripting etc.) can be -redefined for class instances. - -(Lacking universally accepted terminology to talk about classes, I will make -occasional use of Smalltalk and C++ terms. I would use Modula-3 terms, since -its object-oriented semantics are closer to those of Python than C++, but I -expect that few readers have heard of it.) - - -.. _tut-object: - -A Word About Names and Objects -============================== - -Objects have individuality, and multiple names (in multiple scopes) can be bound -to the same object. This is known as aliasing in other languages. This is -usually not appreciated on a first glance at Python, and can be safely ignored -when dealing with immutable basic types (numbers, strings, tuples). However, -aliasing has a possibly surprising effect on the semantics of Python code -involving mutable objects such as lists, dictionaries, and most other types. -This is usually used to the benefit of the program, since aliases behave like -pointers in some respects. For example, passing an object is cheap since only a -pointer is passed by the implementation; and if a function modifies an object -passed as an argument, the caller will see the change --- this eliminates the -need for two different argument passing mechanisms as in Pascal. - - -.. _tut-scopes: - -Python Scopes and Namespaces -============================ - -Before introducing classes, I first have to tell you something about Python's -scope rules. Class definitions play some neat tricks with namespaces, and you -need to know how scopes and namespaces work to fully understand what's going on. -Incidentally, knowledge about this subject is useful for any advanced Python -programmer. - -Let's begin with some definitions. - -A *namespace* is a mapping from names to objects. Most namespaces are currently -implemented as Python dictionaries, but that's normally not noticeable in any -way (except for performance), and it may change in the future. Examples of -namespaces are: the set of built-in names (containing functions such as :func:`abs`, and -built-in exception names); the global names in a module; and the local names in -a function invocation. In a sense the set of attributes of an object also form -a namespace. The important thing to know about namespaces is that there is -absolutely no relation between names in different namespaces; for instance, two -different modules may both define a function ``maximize`` without confusion --- -users of the modules must prefix it with the module name. - -By the way, I use the word *attribute* for any name following a dot --- for -example, in the expression ``z.real``, ``real`` is an attribute of the object -``z``. Strictly speaking, references to names in modules are attribute -references: in the expression ``modname.funcname``, ``modname`` is a module -object and ``funcname`` is an attribute of it. In this case there happens to be -a straightforward mapping between the module's attributes and the global names -defined in the module: they share the same namespace! [#]_ - -Attributes may be read-only or writable. In the latter case, assignment to -attributes is possible. Module attributes are writable: you can write -``modname.the_answer = 42``. Writable attributes may also be deleted with the -:keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. - -Namespaces are created at different moments and have different lifetimes. The -namespace containing the built-in names is created when the Python interpreter -starts up, and is never deleted. The global namespace for a module is created -when the module definition is read in; normally, module namespaces also last -until the interpreter quits. The statements executed by the top-level -invocation of the interpreter, either read from a script file or interactively, -are considered part of a module called :mod:`__main__`, so they have their own -global namespace. (The built-in names actually also live in a module; this is -called :mod:`builtins`.) - -The local namespace for a function is created when the function is called, and -deleted when the function returns or raises an exception that is not handled -within the function. (Actually, forgetting would be a better way to describe -what actually happens.) Of course, recursive invocations each have their own -local namespace. - -A *scope* is a textual region of a Python program where a namespace is directly -accessible. "Directly accessible" here means that an unqualified reference to a -name attempts to find the name in the namespace. - -Although scopes are determined statically, they are used dynamically. At any -time during execution, there are at least three nested scopes whose namespaces -are directly accessible: - -* the innermost scope, which is searched first, contains the local names -* the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contains non-local, but also non-global names -* the next-to-last scope contains the current module's global names -* the outermost scope (searched last) is the namespace containing built-in names - -If a name is declared global, then all references and assignments go directly to -the middle scope containing the module's global names. To rebind variables -found outside of the innermost scope, the :keyword:`nonlocal` statement can be -used; if not declared nonlocal, those variables are read-only (an attempt to -write to such a variable will simply create a *new* local variable in the -innermost scope, leaving the identically named outer variable unchanged). - -Usually, the local scope references the local names of the (textually) current -function. Outside functions, the local scope references the same namespace as -the global scope: the module's namespace. Class definitions place yet another -namespace in the local scope. - -It is important to realize that scopes are determined textually: the global -scope of a function defined in a module is that module's namespace, no matter -from where or by what alias the function is called. On the other hand, the -actual search for names is done dynamically, at run time --- however, the -language definition is evolving towards static name resolution, at "compile" -time, so don't rely on dynamic name resolution! (In fact, local variables are -already determined statically.) - -A special quirk of Python is that -- if no :keyword:`global` statement is in -effect -- assignments to names always go into the innermost scope. Assignments -do not copy data --- they just bind names to objects. The same is true for -deletions: the statement ``del x`` removes the binding of ``x`` from the -namespace referenced by the local scope. In fact, all operations that introduce -new names use the local scope: in particular, :keyword:`import` statements and -function definitions bind the module or function name in the local scope. - -The :keyword:`global` statement can be used to indicate that particular -variables live in the global scope and should be rebound there; the -:keyword:`nonlocal` statement indicates that particular variables live in -an enclosing scope and should be rebound there. - -.. _tut-scopeexample: - -Scopes and Namespaces Example ------------------------------ - -This is an example demonstrating how to reference the different scopes and -namespaces, and how :keyword:`global` and :keyword:`nonlocal` affect variable -binding:: - - def scope_test(): - def do_local(): - spam = "local spam" - - def do_nonlocal(): - nonlocal spam - spam = "nonlocal spam" - - def do_global(): - global spam - spam = "global spam" - - spam = "test spam" - do_local() - print("After local assignment:", spam) - do_nonlocal() - print("After nonlocal assignment:", spam) - do_global() - print("After global assignment:", spam) - - scope_test() - print("In global scope:", spam) - -The output of the example code is: - -.. code-block:: none - - After local assignment: test spam - After nonlocal assignment: nonlocal spam - After global assignment: nonlocal spam - In global scope: global spam - -Note how the *local* assignment (which is default) didn't change *scope_test*\'s -binding of *spam*. The :keyword:`nonlocal` assignment changed *scope_test*\'s -binding of *spam*, and the :keyword:`global` assignment changed the module-level -binding. - -You can also see that there was no previous binding for *spam* before the -:keyword:`global` assignment. - - -.. _tut-firstclasses: - -A First Look at Classes -======================= - -Classes introduce a little bit of new syntax, three new object types, and some -new semantics. - - -.. _tut-classdefinition: - -Class Definition Syntax ------------------------ - -The simplest form of class definition looks like this:: - - class ClassName: - - . - . - . - - -Class definitions, like function definitions (:keyword:`def` statements) must be -executed before they have any effect. (You could conceivably place a class -definition in a branch of an :keyword:`if` statement, or inside a function.) - -In practice, the statements inside a class definition will usually be function -definitions, but other statements are allowed, and sometimes useful --- we'll -come back to this later. The function definitions inside a class normally have -a peculiar form of argument list, dictated by the calling conventions for -methods --- again, this is explained later. - -When a class definition is entered, a new namespace is created, and used as the -local scope --- thus, all assignments to local variables go into this new -namespace. In particular, function definitions bind the name of the new -function here. - -When a class definition is left normally (via the end), a *class object* is -created. This is basically a wrapper around the contents of the namespace -created by the class definition; we'll learn more about class objects in the -next section. The original local scope (the one in effect just before the class -definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the -example). - - -.. _tut-classobjects: - -Class Objects -------------- - -Class objects support two kinds of operations: attribute references and -instantiation. - -*Attribute references* use the standard syntax used for all attribute references -in Python: ``obj.name``. Valid attribute names are all the names that were in -the class's namespace when the class object was created. So, if the class -definition looked like this:: - - class MyClass: - """A simple example class""" - i = 12345 - - def f(self): - return 'hello world' - -then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning -an integer and a function object, respectively. Class attributes can also be -assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`__doc__` is also a valid attribute, returning the docstring belonging to -the class: ``"A simple example class"``. - -Class *instantiation* uses function notation. Just pretend that the class -object is a parameterless function that returns a new instance of the class. -For example (assuming the above class):: - - x = MyClass() - -creates a new *instance* of the class and assigns this object to the local -variable ``x``. - -The instantiation operation ("calling" a class object) creates an empty object. -Many classes like to create objects with instances customized to a specific -initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: - - def __init__(self): - self.data = [] - -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly-created class instance. So -in this example, a new, initialized instance can be obtained by:: - - x = MyClass() - -Of course, the :meth:`__init__` method may have arguments for greater -flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: - - >>> class Complex: - ... def __init__(self, realpart, imagpart): - ... self.r = realpart - ... self.i = imagpart - ... - >>> x = Complex(3.0, -4.5) - >>> x.r, x.i - (3.0, -4.5) - - -.. _tut-instanceobjects: - -Instance Objects ----------------- - -Now what can we do with instance objects? The only operations understood by -instance objects are attribute references. There are two kinds of valid -attribute names: data attributes and methods. - -*data attributes* correspond to "instance variables" in Smalltalk, and to "data -members" in C++. Data attributes need not be declared; like local variables, -they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of -code will print the value ``16``, without leaving a trace:: - - x.counter = 1 - while x.counter < 10: - x.counter = x.counter * 2 - print(x.counter) - del x.counter - -The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) - -.. index:: object: method - -Valid method names of an instance object depend on its class. By definition, -all attributes of a class that are function objects define corresponding -methods of its instances. So in our example, ``x.f`` is a valid method -reference, since ``MyClass.f`` is a function, but ``x.i`` is not, since -``MyClass.i`` is not. But ``x.f`` is not the same thing as ``MyClass.f`` --- it -is a *method object*, not a function object. - - -.. _tut-methodobjects: - -Method Objects --------------- - -Usually, a method is called right after it is bound:: - - x.f() - -In the :class:`MyClass` example, this will return the string ``'hello world'``. -However, it is not necessary to call a method right away: ``x.f`` is a method -object, and can be stored away and called at a later time. For example:: - - xf = x.f - while True: - print(xf()) - -will continue to print ``hello world`` until the end of time. - -What exactly happens when a method is called? You may have noticed that -``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? -Surely Python raises an exception when a function that requires an argument is -called without any --- even if the argument isn't actually used... - -Actually, you may have guessed the answer: the special thing about methods is -that the instance object is passed as the first argument of the function. In our -example, the call ``x.f()`` is exactly equivalent to ``MyClass.f(x)``. In -general, calling a method with a list of *n* arguments is equivalent to calling -the corresponding function with an argument list that is created by inserting -the method's instance object before the first argument. - -If you still don't understand how methods work, a look at the implementation can -perhaps clarify matters. When a non-data attribute of an instance is -referenced, the instance's class is searched. If the name denotes a valid class -attribute that is a function object, a method object is created by packing -(pointers to) the instance object and the function object just found together in -an abstract object: this is the method object. When the method object is called -with an argument list, a new argument list is constructed from the instance -object and the argument list, and the function object is called with this new -argument list. - - -.. _tut-class-and-instance-variables: - -Class and Instance Variables ----------------------------- - -Generally speaking, instance variables are for data unique to each instance -and class variables are for attributes and methods shared by all instances -of the class:: - - class Dog: - - kind = 'canine' # class variable shared by all instances - - def __init__(self, name): - self.name = name # instance variable unique to each instance - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.kind # shared by all dogs - 'canine' - >>> e.kind # shared by all dogs - 'canine' - >>> d.name # unique to d - 'Fido' - >>> e.name # unique to e - 'Buddy' - -As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. -For example, the *tricks* list in the following code should not be used as a -class variable because just a single list would be shared by all *Dog* -instances:: - - class Dog: - - tricks = [] # mistaken use of a class variable - - def __init__(self, name): - self.name = name - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks # unexpectedly shared by all dogs - ['roll over', 'play dead'] - -Correct design of the class should use an instance variable instead:: - - class Dog: - - def __init__(self, name): - self.name = name - self.tricks = [] # creates a new empty list for each dog - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks - ['roll over'] - >>> e.tricks - ['play dead'] - - -.. _tut-remarks: - -Random Remarks -============== - -.. These should perhaps be placed more carefully... - -Data attributes override method attributes with the same name; to avoid -accidental name conflicts, which may cause hard-to-find bugs in large programs, -it is wise to use some kind of convention that minimizes the chance of -conflicts. Possible conventions include capitalizing method names, prefixing -data attribute names with a small unique string (perhaps just an underscore), or -using verbs for methods and nouns for data attributes. - -Data attributes may be referenced by methods as well as by ordinary users -("clients") of an object. In other words, classes are not usable to implement -pure abstract data types. In fact, nothing in Python makes it possible to -enforce data hiding --- it is all based upon convention. (On the other hand, -the Python implementation, written in C, can completely hide implementation -details and control access to an object if necessary; this can be used by -extensions to Python written in C.) - -Clients should use data attributes with care --- clients may mess up invariants -maintained by the methods by stamping on their data attributes. Note that -clients may add data attributes of their own to an instance object without -affecting the validity of the methods, as long as name conflicts are avoided --- -again, a naming convention can save a lot of headaches here. - -There is no shorthand for referencing data attributes (or other methods!) from -within methods. I find that this actually increases the readability of methods: -there is no chance of confusing local variables and instance variables when -glancing through a method. - -Often, the first argument of a method is called ``self``. This is nothing more -than a convention: the name ``self`` has absolutely no special meaning to -Python. Note, however, that by not following the convention your code may be -less readable to other Python programmers, and it is also conceivable that a -*class browser* program might be written that relies upon such a convention. - -Any function object that is a class attribute defines a method for instances of -that class. It is not necessary that the function definition is textually -enclosed in the class definition: assigning a function object to a local -variable in the class is also ok. For example:: - - # Function defined outside the class - def f1(self, x, y): - return min(x, x+y) - - class C: - f = f1 - - def g(self): - return 'hello world' - - h = g - -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to -function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice -usually only serves to confuse the reader of a program. - -Methods may call other methods by using method attributes of the ``self`` -argument:: - - class Bag: - def __init__(self): - self.data = [] - - def add(self, x): - self.data.append(x) - - def addtwice(self, x): - self.add(x) - self.add(x) - -Methods may reference global names in the same way as ordinary functions. The -global scope associated with a method is the module containing its -definition. (A class is never used as a global scope.) While one -rarely encounters a good reason for using global data in a method, there are -many legitimate uses of the global scope: for one thing, functions and modules -imported into the global scope can be used by methods, as well as functions and -classes defined in it. Usually, the class containing the method is itself -defined in this global scope, and in the next section we'll find some good -reasons why a method would want to reference its own class. - -Each value is an object, and therefore has a *class* (also called its *type*). -It is stored as ``object.__class__``. - - -.. _tut-inheritance: - -Inheritance -=========== - -Of course, a language feature would not be worthy of the name "class" without -supporting inheritance. The syntax for a derived class definition looks like -this:: - - class DerivedClassName(BaseClassName): - - . - . - . - - -The name :class:`BaseClassName` must be defined in a scope containing the -derived class definition. In place of a base class name, other arbitrary -expressions are also allowed. This can be useful, for example, when the base -class is defined in another module:: - - class DerivedClassName(modname.BaseClassName): - -Execution of a derived class definition proceeds the same as for a base class. -When the class object is constructed, the base class is remembered. This is -used for resolving attribute references: if a requested attribute is not found -in the class, the search proceeds to look in the base class. This rule is -applied recursively if the base class itself is derived from some other class. - -There's nothing special about instantiation of derived classes: -``DerivedClassName()`` creates a new instance of the class. Method references -are resolved as follows: the corresponding class attribute is searched, -descending down the chain of base classes if necessary, and the method reference -is valid if this yields a function object. - -Derived classes may override methods of their base classes. Because methods -have no special privileges when calling other methods of the same object, a -method of a base class that calls another method defined in the same base class -may end up calling a method of a derived class that overrides it. (For C++ -programmers: all methods in Python are effectively ``virtual``.) - -An overriding method in a derived class may in fact want to extend rather than -simply replace the base class method of the same name. There is a simple way to -call the base class method directly: just call ``BaseClassName.methodname(self, -arguments)``. This is occasionally useful to clients as well. (Note that this -only works if the base class is accessible as ``BaseClassName`` in the global -scope.) - -Python has two built-in functions that work with inheritance: - -* Use :func:`isinstance` to check an instance's type: ``isinstance(obj, int)`` - will be ``True`` only if ``obj.__class__`` is :class:`int` or some class - derived from :class:`int`. - -* Use :func:`issubclass` to check class inheritance: ``issubclass(bool, int)`` - is ``True`` since :class:`bool` is a subclass of :class:`int`. However, - ``issubclass(float, int)`` is ``False`` since :class:`float` is not a - subclass of :class:`int`. - - - -.. _tut-multiple: - -Multiple Inheritance --------------------- - -Python supports a form of multiple inheritance as well. A class definition with -multiple base classes looks like this:: - - class DerivedClassName(Base1, Base2, Base3): - - . - . - . - - -For most purposes, in the simplest cases, you can think of the search for -attributes inherited from a parent class as depth-first, left-to-right, not -searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. - -In fact, it is slightly more complex than that; the method resolution order -changes dynamically to support cooperative calls to :func:`super`. This -approach is known in some other multiple-inheritance languages as -call-next-method and is more powerful than the super call found in -single-inheritance languages. - -Dynamic ordering is necessary because all cases of multiple inheritance exhibit -one or more diamond relationships (where at least one of the parent classes -can be accessed through multiple paths from the bottommost class). For example, -all classes inherit from :class:`object`, so any case of multiple inheritance -provides more than one path to reach :class:`object`. To keep the base classes -from being accessed more than once, the dynamic algorithm linearizes the search -order in a way that preserves the left-to-right ordering specified in each -class, that calls each parent only once, and that is monotonic (meaning that a -class can be subclassed without affecting the precedence order of its parents). -Taken together, these properties make it possible to design reliable and -extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. - - -.. _tut-private: - -Private Variables -================= - -"Private" instance variables that cannot be accessed except from inside an -object don't exist in Python. However, there is a convention that is followed -by most Python code: a name prefixed with an underscore (e.g. ``_spam``) should -be treated as a non-public part of the API (whether it is a function, a method -or a data member). It should be considered an implementation detail and subject -to change without notice. - -.. index:: - pair: name; mangling - -Since there is a valid use-case for class-private members (namely to avoid name -clashes of names with names defined by subclasses), there is limited support for -such a mechanism, called :dfn:`name mangling`. Any identifier of the form -``__spam`` (at least two leading underscores, at most one trailing underscore) -is textually replaced with ``_classname__spam``, where ``classname`` is the -current class name with leading underscore(s) stripped. This mangling is done -without regard to the syntactic position of the identifier, as long as it -occurs within the definition of a class. - -Name mangling is helpful for letting subclasses override methods without -breaking intraclass method calls. For example:: - - class Mapping: - def __init__(self, iterable): - self.items_list = [] - self.__update(iterable) - - def update(self, iterable): - for item in iterable: - self.items_list.append(item) - - __update = update # private copy of original update() method - - class MappingSubclass(Mapping): - - def update(self, keys, values): - # provides new signature for update() - # but does not break __init__() - for item in zip(keys, values): - self.items_list.append(item) - -The above example would work even if ``MappingSubclass`` were to introduce a -``__update`` identifier since it is replaced with ``_Mapping__update`` in the -``Mapping`` class and ``_MappingSubclass__update`` in the ``MappingSubclass`` -class respectively. - -Note that the mangling rules are designed mostly to avoid accidents; it still is -possible to access or modify a variable that is considered private. This can -even be useful in special circumstances, such as in the debugger. - -Notice that code passed to ``exec()`` or ``eval()`` does not consider the -classname of the invoking class to be the current class; this is similar to the -effect of the ``global`` statement, the effect of which is likewise restricted -to code that is byte-compiled together. The same restriction applies to -``getattr()``, ``setattr()`` and ``delattr()``, as well as when referencing -``__dict__`` directly. - - -.. _tut-odds: - -Odds and Ends -============= - -Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. An empty class definition -will do nicely:: - - class Employee: - pass - - john = Employee() # Create an empty employee record - - # Fill the fields of the record - john.name = 'John Doe' - john.dept = 'computer lab' - john.salary = 1000 - -A piece of Python code that expects a particular abstract data type can often be -passed a class that emulates the methods of that data type instead. For -instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the -data from a string buffer instead, and pass it as an argument. - -.. (Unfortunately, this technique has its limitations: a class can't define - operations that are accessed by special syntax such as sequence subscripting - or arithmetic operators, and assigning such a "pseudo-file" to sys.stdin will - not cause the interpreter to read further input from it.) - -Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object -corresponding to the method. - - -.. _tut-iterators: - -Iterators -========= - -By now you have probably noticed that most container objects can be looped over -using a :keyword:`for` statement:: - - for element in [1, 2, 3]: - print(element) - for element in (1, 2, 3): - print(element) - for key in {'one':1, 'two':2}: - print(key) - for char in "123": - print(char) - for line in open("myfile.txt"): - print(line, end='') - -This style of access is clear, concise, and convenient. The use of iterators -pervades and unifies Python. Behind the scenes, the :keyword:`for` statement -calls :func:`iter` on the container object. The function returns an iterator -object that defines the method :meth:`~iterator.__next__` which accesses -elements in the container one at a time. When there are no more elements, -:meth:`~iterator.__next__` raises a :exc:`StopIteration` exception which tells the -:keyword:`for` loop to terminate. You can call the :meth:`~iterator.__next__` method -using the :func:`next` built-in function; this example shows how it all works:: - - >>> s = 'abc' - >>> it = iter(s) - >>> it - - >>> next(it) - 'a' - >>> next(it) - 'b' - >>> next(it) - 'c' - >>> next(it) - Traceback (most recent call last): - File "", line 1, in - next(it) - StopIteration - -Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which -returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: - - class Reverse: - """Iterator for looping over a sequence backwards.""" - def __init__(self, data): - self.data = data - self.index = len(data) - - def __iter__(self): - return self - - def __next__(self): - if self.index == 0: - raise StopIteration - self.index = self.index - 1 - return self.data[self.index] - -:: - - >>> rev = Reverse('spam') - >>> iter(rev) - <__main__.Reverse object at 0x00A1DB50> - >>> for char in rev: - ... print(char) - ... - m - a - p - s - - -.. _tut-generators: - -Generators -========== - -:term:`Generator`\s are a simple and powerful tool for creating iterators. They -are written like regular functions but use the :keyword:`yield` statement -whenever they want to return data. Each time :func:`next` is called on it, the -generator resumes where it left off (it remembers all the data values and which -statement was last executed). An example shows that generators can be trivially -easy to create:: - - def reverse(data): - for index in range(len(data)-1, -1, -1): - yield data[index] - -:: - - >>> for char in reverse('golf'): - ... print(char) - ... - f - l - o - g - -Anything that can be done with generators can also be done with class-based -iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods -are created automatically. - -Another key feature is that the local variables and execution state are -automatically saved between calls. This made the function easier to write and -much more clear than an approach using instance variables like ``self.index`` -and ``self.data``. - -In addition to automatic method creation and saving program state, when -generators terminate, they automatically raise :exc:`StopIteration`. In -combination, these features make it easy to create iterators with no more effort -than writing a regular function. - - -.. _tut-genexps: - -Generator Expressions -===================== - -Some simple generators can be coded succinctly as expressions using a syntax -similar to list comprehensions but with parentheses instead of square brackets. -These expressions are designed for situations where the generator is used right -away by an enclosing function. Generator expressions are more compact but less -versatile than full generator definitions and tend to be more memory friendly -than equivalent list comprehensions. - -Examples:: - - >>> sum(i*i for i in range(10)) # sum of squares - 285 - - >>> xvec = [10, 20, 30] - >>> yvec = [7, 5, 3] - >>> sum(x*y for x,y in zip(xvec, yvec)) # dot product - 260 - - >>> from math import pi, sin - >>> sine_table = {x: sin(x*pi/180) for x in range(0, 91)} - - >>> unique_words = set(word for line in page for word in line.split()) - - >>> valedictorian = max((student.gpa, student.name) for student in graduates) - - >>> data = 'golf' - >>> list(data[i] for i in range(len(data)-1, -1, -1)) - ['f', 'l', 'o', 'g'] - - - -.. rubric:: Footnotes - -.. [#] Except for one thing. Module objects have a secret read-only attribute called - :attr:`~object.__dict__` which returns the dictionary used to implement the module's - namespace; the name :attr:`~object.__dict__` is an attribute but not a global name. - Obviously, using this violates the abstraction of namespace implementation, and - should be restricted to things like post-mortem debuggers. diff --git a/third_party/python/Doc/tutorial/controlflow.rst b/third_party/python/Doc/tutorial/controlflow.rst deleted file mode 100644 index bf6fbe21a..000000000 --- a/third_party/python/Doc/tutorial/controlflow.rst +++ /dev/null @@ -1,762 +0,0 @@ -.. _tut-morecontrol: - -*********************** -More Control Flow Tools -*********************** - -Besides the :keyword:`while` statement just introduced, Python knows the usual -control flow statements known from other languages, with some twists. - - -.. _tut-if: - -:keyword:`if` Statements -======================== - -Perhaps the most well-known statement type is the :keyword:`if` statement. For -example:: - - >>> x = int(input("Please enter an integer: ")) - Please enter an integer: 42 - >>> if x < 0: - ... x = 0 - ... print('Negative changed to zero') - ... elif x == 0: - ... print('Zero') - ... elif x == 1: - ... print('Single') - ... else: - ... print('More') - ... - More - -There can be zero or more :keyword:`elif` parts, and the :keyword:`else` part is -optional. The keyword ':keyword:`elif`' is short for 'else if', and is useful -to avoid excessive indentation. An :keyword:`if` ... :keyword:`elif` ... -:keyword:`elif` ... sequence is a substitute for the ``switch`` or -``case`` statements found in other languages. - - -.. _tut-for: - -:keyword:`for` Statements -========================= - -.. index:: - statement: for - -The :keyword:`for` statement in Python differs a bit from what you may be used -to in C or Pascal. Rather than always iterating over an arithmetic progression -of numbers (like in Pascal), or giving the user the ability to define both the -iteration step and halting condition (as C), Python's :keyword:`for` statement -iterates over the items of any sequence (a list or a string), in the order that -they appear in the sequence. For example (no pun intended): - -.. One suggestion was to give a real C example here, but that may only serve to - confuse non-C programmers. - -:: - - >>> # Measure some strings: - ... words = ['cat', 'window', 'defenestrate'] - >>> for w in words: - ... print(w, len(w)) - ... - cat 3 - window 6 - defenestrate 12 - -If you need to modify the sequence you are iterating over while inside the loop -(for example to duplicate selected items), it is recommended that you first -make a copy. Iterating over a sequence does not implicitly make a copy. The -slice notation makes this especially convenient:: - - >>> for w in words[:]: # Loop over a slice copy of the entire list. - ... if len(w) > 6: - ... words.insert(0, w) - ... - >>> words - ['defenestrate', 'cat', 'window', 'defenestrate'] - -With ``for w in words:``, the example would attempt to create an infinite list, -inserting ``defenestrate`` over and over again. - - -.. _tut-range: - -The :func:`range` Function -========================== - -If you do need to iterate over a sequence of numbers, the built-in function -:func:`range` comes in handy. It generates arithmetic progressions:: - - >>> for i in range(5): - ... print(i) - ... - 0 - 1 - 2 - 3 - 4 - -The given end point is never part of the generated sequence; ``range(10)`` generates -10 values, the legal indices for items of a sequence of length 10. It -is possible to let the range start at another number, or to specify a different -increment (even negative; sometimes this is called the 'step'):: - - range(5, 10) - 5, 6, 7, 8, 9 - - range(0, 10, 3) - 0, 3, 6, 9 - - range(-10, -100, -30) - -10, -40, -70 - -To iterate over the indices of a sequence, you can combine :func:`range` and -:func:`len` as follows:: - - >>> a = ['Mary', 'had', 'a', 'little', 'lamb'] - >>> for i in range(len(a)): - ... print(i, a[i]) - ... - 0 Mary - 1 had - 2 a - 3 little - 4 lamb - -In most such cases, however, it is convenient to use the :func:`enumerate` -function, see :ref:`tut-loopidioms`. - -A strange thing happens if you just print a range:: - - >>> print(range(10)) - range(0, 10) - -In many ways the object returned by :func:`range` behaves as if it is a list, -but in fact it isn't. It is an object which returns the successive items of -the desired sequence when you iterate over it, but it doesn't really make -the list, thus saving space. - -We say such an object is *iterable*, that is, suitable as a target for -functions and constructs that expect something from which they can -obtain successive items until the supply is exhausted. We have seen that -the :keyword:`for` statement is such an *iterator*. The function :func:`list` -is another; it creates lists from iterables:: - - - >>> list(range(5)) - [0, 1, 2, 3, 4] - -Later we will see more functions that return iterables and take iterables as argument. - - -.. _tut-break: - -:keyword:`break` and :keyword:`continue` Statements, and :keyword:`else` Clauses on Loops -========================================================================================= - -The :keyword:`break` statement, like in C, breaks out of the innermost enclosing -:keyword:`for` or :keyword:`while` loop. - -Loop statements may have an ``else`` clause; it is executed when the loop -terminates through exhaustion of the list (with :keyword:`for`) or when the -condition becomes false (with :keyword:`while`), but not when the loop is -terminated by a :keyword:`break` statement. This is exemplified by the -following loop, which searches for prime numbers:: - - >>> for n in range(2, 10): - ... for x in range(2, n): - ... if n % x == 0: - ... print(n, 'equals', x, '*', n//x) - ... break - ... else: - ... # loop fell through without finding a factor - ... print(n, 'is a prime number') - ... - 2 is a prime number - 3 is a prime number - 4 equals 2 * 2 - 5 is a prime number - 6 equals 2 * 3 - 7 is a prime number - 8 equals 2 * 4 - 9 equals 3 * 3 - -(Yes, this is the correct code. Look closely: the ``else`` clause belongs to -the :keyword:`for` loop, **not** the :keyword:`if` statement.) - -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: - - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found a number", num) - Found an even number 2 - Found a number 3 - Found an even number 4 - Found a number 5 - Found an even number 6 - Found a number 7 - Found an even number 8 - Found a number 9 - -.. _tut-pass: - -:keyword:`pass` Statements -========================== - -The :keyword:`pass` statement does nothing. It can be used when a statement is -required syntactically but the program requires no action. For example:: - - >>> while True: - ... pass # Busy-wait for keyboard interrupt (Ctrl+C) - ... - -This is commonly used for creating minimal classes:: - - >>> class MyEmptyClass: - ... pass - ... - -Another place :keyword:`pass` can be used is as a place-holder for a function or -conditional body when you are working on new code, allowing you to keep thinking -at a more abstract level. The :keyword:`pass` is silently ignored:: - - >>> def initlog(*args): - ... pass # Remember to implement this! - ... - -.. _tut-functions: - -Defining Functions -================== - -We can create a function that writes the Fibonacci series to an arbitrary -boundary:: - - >>> def fib(n): # write Fibonacci series up to n - ... """Print a Fibonacci series up to n.""" - ... a, b = 0, 1 - ... while a < n: - ... print(a, end=' ') - ... a, b = b, a+b - ... print() - ... - >>> # Now call the function we just defined: - ... fib(2000) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 - -.. index:: - single: documentation strings - single: docstrings - single: strings, documentation - -The keyword :keyword:`def` introduces a function *definition*. It must be -followed by the function name and the parenthesized list of formal parameters. -The statements that form the body of the function start at the next line, and -must be indented. - -The first statement of the function body can optionally be a string literal; -this string literal is the function's documentation string, or :dfn:`docstring`. -(More about docstrings can be found in the section :ref:`tut-docstrings`.) -There are tools which use docstrings to automatically produce online or printed -documentation, or to let the user interactively browse through code; it's good -practice to include docstrings in code that you write, so make a habit of it. - -The *execution* of a function introduces a new symbol table used for the local -variables of the function. More precisely, all variable assignments in a -function store the value in the local symbol table; whereas variable references -first look in the local symbol table, then in the local symbol tables of -enclosing functions, then in the global symbol table, and finally in the table -of built-in names. Thus, global variables cannot be directly assigned a value -within a function (unless named in a :keyword:`global` statement), although they -may be referenced. - -The actual parameters (arguments) to a function call are introduced in the local -symbol table of the called function when it is called; thus, arguments are -passed using *call by value* (where the *value* is always an object *reference*, -not the value of the object). [#]_ When a function calls another function, a new -local symbol table is created for that call. - -A function definition introduces the function name in the current symbol table. -The value of the function name has a type that is recognized by the interpreter -as a user-defined function. This value can be assigned to another name which -can then also be used as a function. This serves as a general renaming -mechanism:: - - >>> fib - - >>> f = fib - >>> f(100) - 0 1 1 2 3 5 8 13 21 34 55 89 - -Coming from other languages, you might object that ``fib`` is not a function but -a procedure since it doesn't return a value. In fact, even functions without a -:keyword:`return` statement do return a value, albeit a rather boring one. This -value is called ``None`` (it's a built-in name). Writing the value ``None`` is -normally suppressed by the interpreter if it would be the only value written. -You can see it if you really want to using :func:`print`:: - - >>> fib(0) - >>> print(fib(0)) - None - -It is simple to write a function that returns a list of the numbers of the -Fibonacci series, instead of printing it:: - - >>> def fib2(n): # return Fibonacci series up to n - ... """Return a list containing the Fibonacci series up to n.""" - ... result = [] - ... a, b = 0, 1 - ... while a < n: - ... result.append(a) # see below - ... a, b = b, a+b - ... return result - ... - >>> f100 = fib2(100) # call it - >>> f100 # write the result - [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89] - -This example, as usual, demonstrates some new Python features: - -* The :keyword:`return` statement returns with a value from a function. - :keyword:`return` without an expression argument returns ``None``. Falling off - the end of a function also returns ``None``. - -* The statement ``result.append(a)`` calls a *method* of the list object - ``result``. A method is a function that 'belongs' to an object and is named - ``obj.methodname``, where ``obj`` is some object (this may be an expression), - and ``methodname`` is the name of a method that is defined by the object's type. - Different types define different methods. Methods of different types may have - the same name without causing ambiguity. (It is possible to define your own - object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it - adds a new element at the end of the list. In this example it is equivalent to - ``result = result + [a]``, but more efficient. - - -.. _tut-defining: - -More on Defining Functions -========================== - -It is also possible to define functions with a variable number of arguments. -There are three forms, which can be combined. - - -.. _tut-defaultargs: - -Default Argument Values ------------------------ - -The most useful form is to specify a default value for one or more arguments. -This creates a function that can be called with fewer arguments than it is -defined to allow. For example:: - - def ask_ok(prompt, retries=4, reminder='Please try again!'): - while True: - ok = input(prompt) - if ok in ('y', 'ye', 'yes'): - return True - if ok in ('n', 'no', 'nop', 'nope'): - return False - retries = retries - 1 - if retries < 0: - raise ValueError('invalid user response') - print(reminder) - -This function can be called in several ways: - -* giving only the mandatory argument: - ``ask_ok('Do you really want to quit?')`` -* giving one of the optional arguments: - ``ask_ok('OK to overwrite the file?', 2)`` -* or even giving all arguments: - ``ask_ok('OK to overwrite the file?', 2, 'Come on, only yes or no!')`` - -This example also introduces the :keyword:`in` keyword. This tests whether or -not a sequence contains a certain value. - -The default values are evaluated at the point of function definition in the -*defining* scope, so that :: - - i = 5 - - def f(arg=i): - print(arg) - - i = 6 - f() - -will print ``5``. - -**Important warning:** The default value is evaluated only once. This makes a -difference when the default is a mutable object such as a list, dictionary, or -instances of most classes. For example, the following function accumulates the -arguments passed to it on subsequent calls:: - - def f(a, L=[]): - L.append(a) - return L - - print(f(1)) - print(f(2)) - print(f(3)) - -This will print :: - - [1] - [1, 2] - [1, 2, 3] - -If you don't want the default to be shared between subsequent calls, you can -write the function like this instead:: - - def f(a, L=None): - if L is None: - L = [] - L.append(a) - return L - - -.. _tut-keywordargs: - -Keyword Arguments ------------------ - -Functions can also be called using :term:`keyword arguments ` -of the form ``kwarg=value``. For instance, the following function:: - - def parrot(voltage, state='a stiff', action='voom', type='Norwegian Blue'): - print("-- This parrot wouldn't", action, end=' ') - print("if you put", voltage, "volts through it.") - print("-- Lovely plumage, the", type) - print("-- It's", state, "!") - -accepts one required argument (``voltage``) and three optional arguments -(``state``, ``action``, and ``type``). This function can be called in any -of the following ways:: - - parrot(1000) # 1 positional argument - parrot(voltage=1000) # 1 keyword argument - parrot(voltage=1000000, action='VOOOOOM') # 2 keyword arguments - parrot(action='VOOOOOM', voltage=1000000) # 2 keyword arguments - parrot('a million', 'bereft of life', 'jump') # 3 positional arguments - parrot('a thousand', state='pushing up the daisies') # 1 positional, 1 keyword - -but all the following calls would be invalid:: - - parrot() # required argument missing - parrot(voltage=5.0, 'dead') # non-keyword argument after a keyword argument - parrot(110, voltage=220) # duplicate value for the same argument - parrot(actor='John Cleese') # unknown keyword argument - -In a function call, keyword arguments must follow positional arguments. -All the keyword arguments passed must match one of the arguments -accepted by the function (e.g. ``actor`` is not a valid argument for the -``parrot`` function), and their order is not important. This also includes -non-optional arguments (e.g. ``parrot(voltage=1000)`` is valid too). -No argument may receive a value more than once. -Here's an example that fails due to this restriction:: - - >>> def function(a): - ... pass - ... - >>> function(0, a=0) - Traceback (most recent call last): - File "", line 1, in - TypeError: function() got multiple values for keyword argument 'a' - -When a final formal parameter of the form ``**name`` is present, it receives a -dictionary (see :ref:`typesmapping`) containing all keyword arguments except for -those corresponding to a formal parameter. This may be combined with a formal -parameter of the form ``*name`` (described in the next subsection) which -receives a tuple containing the positional arguments beyond the formal parameter -list. (``*name`` must occur before ``**name``.) For example, if we define a -function like this:: - - def cheeseshop(kind, *arguments, **keywords): - print("-- Do you have any", kind, "?") - print("-- I'm sorry, we're all out of", kind) - for arg in arguments: - print(arg) - print("-" * 40) - for kw in keywords: - print(kw, ":", keywords[kw]) - -It could be called like this:: - - cheeseshop("Limburger", "It's very runny, sir.", - "It's really very, VERY runny, sir.", - shopkeeper="Michael Palin", - client="John Cleese", - sketch="Cheese Shop Sketch") - -and of course it would print: - -.. code-block:: none - - -- Do you have any Limburger ? - -- I'm sorry, we're all out of Limburger - It's very runny, sir. - It's really very, VERY runny, sir. - ---------------------------------------- - shopkeeper : Michael Palin - client : John Cleese - sketch : Cheese Shop Sketch - -Note that the order in which the keyword arguments are printed is guaranteed -to match the order in which they were provided in the function call. - - -.. _tut-arbitraryargs: - -Arbitrary Argument Lists ------------------------- - -.. index:: - single: * (asterisk); in function calls - -Finally, the least frequently used option is to specify that a function can be -called with an arbitrary number of arguments. These arguments will be wrapped -up in a tuple (see :ref:`tut-tuples`). Before the variable number of arguments, -zero or more normal arguments may occur. :: - - def write_multiple_items(file, separator, *args): - file.write(separator.join(args)) - - -Normally, these ``variadic`` arguments will be last in the list of formal -parameters, because they scoop up all remaining input arguments that are -passed to the function. Any formal parameters which occur after the ``*args`` -parameter are 'keyword-only' arguments, meaning that they can only be used as -keywords rather than positional arguments. :: - - >>> def concat(*args, sep="/"): - ... return sep.join(args) - ... - >>> concat("earth", "mars", "venus") - 'earth/mars/venus' - >>> concat("earth", "mars", "venus", sep=".") - 'earth.mars.venus' - -.. _tut-unpacking-arguments: - -Unpacking Argument Lists ------------------------- - -The reverse situation occurs when the arguments are already in a list or tuple -but need to be unpacked for a function call requiring separate positional -arguments. For instance, the built-in :func:`range` function expects separate -*start* and *stop* arguments. If they are not available separately, write the -function call with the ``*``\ -operator to unpack the arguments out of a list -or tuple:: - - >>> list(range(3, 6)) # normal call with separate arguments - [3, 4, 5] - >>> args = [3, 6] - >>> list(range(*args)) # call with arguments unpacked from a list - [3, 4, 5] - -.. index:: - single: **; in function calls - -In the same fashion, dictionaries can deliver keyword arguments with the -``**``\ -operator:: - - >>> def parrot(voltage, state='a stiff', action='voom'): - ... print("-- This parrot wouldn't", action, end=' ') - ... print("if you put", voltage, "volts through it.", end=' ') - ... print("E's", state, "!") - ... - >>> d = {"voltage": "four million", "state": "bleedin' demised", "action": "VOOM"} - >>> parrot(**d) - -- This parrot wouldn't VOOM if you put four million volts through it. E's bleedin' demised ! - - -.. _tut-lambda: - -Lambda Expressions ------------------- - -Small anonymous functions can be created with the :keyword:`lambda` keyword. -This function returns the sum of its two arguments: ``lambda a, b: a+b``. -Lambda functions can be used wherever function objects are required. They are -syntactically restricted to a single expression. Semantically, they are just -syntactic sugar for a normal function definition. Like nested function -definitions, lambda functions can reference variables from the containing -scope:: - - >>> def make_incrementor(n): - ... return lambda x: x + n - ... - >>> f = make_incrementor(42) - >>> f(0) - 42 - >>> f(1) - 43 - -The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: - - >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] - >>> pairs.sort(key=lambda pair: pair[1]) - >>> pairs - [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')] - - -.. _tut-docstrings: - -Documentation Strings ---------------------- - -.. index:: - single: docstrings - single: documentation strings - single: strings, documentation - -Here are some conventions about the content and formatting of documentation -strings. - -The first line should always be a short, concise summary of the object's -purpose. For brevity, it should not explicitly state the object's name or type, -since these are available by other means (except if the name happens to be a -verb describing a function's operation). This line should begin with a capital -letter and end with a period. - -If there are more lines in the documentation string, the second line should be -blank, visually separating the summary from the rest of the description. The -following lines should be one or more paragraphs describing the object's calling -conventions, its side effects, etc. - -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). - -Here is an example of a multi-line docstring:: - - >>> def my_function(): - ... """Do nothing, but document it. - ... - ... No, really, it doesn't do anything. - ... """ - ... pass - ... - >>> print(my_function.__doc__) - Do nothing, but document it. - - No, really, it doesn't do anything. - - -.. _tut-annotations: - -Function Annotations --------------------- - -.. sectionauthor:: Zachary Ware -.. index:: - pair: function; annotations - single: ->; function annotations - single: : (colon); function annotations - -:ref:`Function annotations ` are completely optional metadata -information about the types used by user-defined functions (see :pep:`3107` and -:pep:`484` for more information). - -Annotations are stored in the :attr:`__annotations__` attribute of the function -as a dictionary and have no effect on any other part of the function. Parameter -annotations are defined by a colon after the parameter name, followed by an -expression evaluating to the value of the annotation. Return annotations are -defined by a literal ``->``, followed by an expression, between the parameter -list and the colon denoting the end of the :keyword:`def` statement. The -following example has a positional argument, a keyword argument, and the return -value annotated:: - - >>> def f(ham: str, eggs: str = 'eggs') -> str: - ... print("Annotations:", f.__annotations__) - ... print("Arguments:", ham, eggs) - ... return ham + ' and ' + eggs - ... - >>> f('spam') - Annotations: {'ham': , 'return': , 'eggs': } - Arguments: spam eggs - 'spam and eggs' - -.. _tut-codingstyle: - -Intermezzo: Coding Style -======================== - -.. sectionauthor:: Georg Brandl -.. index:: pair: coding; style - -Now that you are about to write longer, more complex pieces of Python, it is a -good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. -Making it easy for others to read your code is always a good idea, and adopting -a nice coding style helps tremendously for that. - -For Python, :pep:`8` has emerged as the style guide that most projects adhere to; -it promotes a very readable and eye-pleasing coding style. Every Python -developer should read it at some point; here are the most important points -extracted for you: - -* Use 4-space indentation, and no tabs. - - 4 spaces are a good compromise between small indentation (allows greater - nesting depth) and large indentation (easier to read). Tabs introduce - confusion, and are best left out. - -* Wrap lines so that they don't exceed 79 characters. - - This helps users with small displays and makes it possible to have several - code files side-by-side on larger displays. - -* Use blank lines to separate functions and classes, and larger blocks of - code inside functions. - -* When possible, put comments on a line of their own. - -* Use docstrings. - -* Use spaces around operators and after commas, but not directly inside - bracketing constructs: ``a = f(1, 2) + g(3, 4)``. - -* Name your classes and functions consistently; the convention is to use - ``CamelCase`` for classes and ``lower_case_with_underscores`` for functions - and methods. Always use ``self`` as the name for the first method argument - (see :ref:`tut-firstclasses` for more on classes and methods). - -* Don't use fancy encodings if your code is meant to be used in international - environments. Python's default, UTF-8, or even plain ASCII work best in any - case. - -* Likewise, don't use non-ASCII characters in identifiers if there is only the - slightest chance people speaking a different language will read or maintain - the code. - - -.. rubric:: Footnotes - -.. [#] Actually, *call by object reference* would be a better description, - since if a mutable object is passed, the caller will see any changes the - callee makes to it (items inserted into a list). diff --git a/third_party/python/Doc/tutorial/datastructures.rst b/third_party/python/Doc/tutorial/datastructures.rst deleted file mode 100644 index f9ddf06c5..000000000 --- a/third_party/python/Doc/tutorial/datastructures.rst +++ /dev/null @@ -1,716 +0,0 @@ -.. _tut-structures: - -*************** -Data Structures -*************** - -This chapter describes some things you've learned about already in more detail, -and adds some new things as well. - -.. _tut-morelists: - -More on Lists -============= - -The list data type has some more methods. Here are all of the methods of list -objects: - - -.. method:: list.append(x) - :noindex: - - Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. - - -.. method:: list.extend(iterable) - :noindex: - - Extend the list by appending all the items from the iterable. Equivalent to - ``a[len(a):] = iterable``. - - -.. method:: list.insert(i, x) - :noindex: - - Insert an item at a given position. The first argument is the index of the - element before which to insert, so ``a.insert(0, x)`` inserts at the front of - the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. - - -.. method:: list.remove(x) - :noindex: - - Remove the first item from the list whose value is *x*. It is an error if - there is no such item. - - -.. method:: list.pop([i]) - :noindex: - - Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) - - -.. method:: list.clear() - :noindex: - - Remove all items from the list. Equivalent to ``del a[:]``. - - -.. method:: list.index(x[, start[, end]]) - :noindex: - - Return zero-based index in the list of the first item whose value is *x*. - Raises a :exc:`ValueError` if there is no such item. - - The optional arguments *start* and *end* are interpreted as in the slice - notation and are used to limit the search to a particular subsequence of - the list. The returned index is computed relative to the beginning of the full - sequence rather than the *start* argument. - - -.. method:: list.count(x) - :noindex: - - Return the number of times *x* appears in the list. - - -.. method:: list.sort(key=None, reverse=False) - :noindex: - - Sort the items of the list in place (the arguments can be used for sort - customization, see :func:`sorted` for their explanation). - - -.. method:: list.reverse() - :noindex: - - Reverse the elements of the list in place. - - -.. method:: list.copy() - :noindex: - - Return a shallow copy of the list. Equivalent to ``a[:]``. - - -An example that uses most of the list methods:: - - >>> fruits = ['orange', 'apple', 'pear', 'banana', 'kiwi', 'apple', 'banana'] - >>> fruits.count('apple') - 2 - >>> fruits.count('tangerine') - 0 - >>> fruits.index('banana') - 3 - >>> fruits.index('banana', 4) # Find next banana starting a position 4 - 6 - >>> fruits.reverse() - >>> fruits - ['banana', 'apple', 'kiwi', 'banana', 'pear', 'apple', 'orange'] - >>> fruits.append('grape') - >>> fruits - ['banana', 'apple', 'kiwi', 'banana', 'pear', 'apple', 'orange', 'grape'] - >>> fruits.sort() - >>> fruits - ['apple', 'apple', 'banana', 'banana', 'grape', 'kiwi', 'orange', 'pear'] - >>> fruits.pop() - 'pear' - -You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that -only modify the list have no return value printed -- they return the default -``None``. [1]_ This is a design principle for all mutable data structures in -Python. - - -.. _tut-lists-as-stacks: - -Using Lists as Stacks ---------------------- - -.. sectionauthor:: Ka-Ping Yee - - -The list methods make it very easy to use a list as a stack, where the last -element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: - - >>> stack = [3, 4, 5] - >>> stack.append(6) - >>> stack.append(7) - >>> stack - [3, 4, 5, 6, 7] - >>> stack.pop() - 7 - >>> stack - [3, 4, 5, 6] - >>> stack.pop() - 6 - >>> stack.pop() - 5 - >>> stack - [3, 4] - - -.. _tut-lists-as-queues: - -Using Lists as Queues ---------------------- - -.. sectionauthor:: Ka-Ping Yee - -It is also possible to use a list as a queue, where the first element added is -the first element retrieved ("first-in, first-out"); however, lists are not -efficient for this purpose. While appends and pops from the end of list are -fast, doing inserts or pops from the beginning of a list is slow (because all -of the other elements have to be shifted by one). - -To implement a queue, use :class:`collections.deque` which was designed to -have fast appends and pops from both ends. For example:: - - >>> from collections import deque - >>> queue = deque(["Eric", "John", "Michael"]) - >>> queue.append("Terry") # Terry arrives - >>> queue.append("Graham") # Graham arrives - >>> queue.popleft() # The first to arrive now leaves - 'Eric' - >>> queue.popleft() # The second to arrive now leaves - 'John' - >>> queue # Remaining queue in order of arrival - deque(['Michael', 'Terry', 'Graham']) - - -.. _tut-listcomps: - -List Comprehensions -------------------- - -List comprehensions provide a concise way to create lists. -Common applications are to make new lists where each element is the result of -some operations applied to each member of another sequence or iterable, or to -create a subsequence of those elements that satisfy a certain condition. - -For example, assume we want to create a list of squares, like:: - - >>> squares = [] - >>> for x in range(10): - ... squares.append(x**2) - ... - >>> squares - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - -Note that this creates (or overwrites) a variable named ``x`` that still exists -after the loop completes. We can calculate the list of squares without any -side effects using:: - - squares = list(map(lambda x: x**2, range(10))) - -or, equivalently:: - - squares = [x**2 for x in range(10)] - -which is more concise and readable. - -A list comprehension consists of brackets containing an expression followed -by a :keyword:`for` clause, then zero or more :keyword:`for` or :keyword:`if` -clauses. The result will be a new list resulting from evaluating the expression -in the context of the :keyword:`for` and :keyword:`if` clauses which follow it. -For example, this listcomp combines the elements of two lists if they are not -equal:: - - >>> [(x, y) for x in [1,2,3] for y in [3,1,4] if x != y] - [(1, 3), (1, 4), (2, 3), (2, 1), (2, 4), (3, 1), (3, 4)] - -and it's equivalent to:: - - >>> combs = [] - >>> for x in [1,2,3]: - ... for y in [3,1,4]: - ... if x != y: - ... combs.append((x, y)) - ... - >>> combs - [(1, 3), (1, 4), (2, 3), (2, 1), (2, 4), (3, 1), (3, 4)] - -Note how the order of the :keyword:`for` and :keyword:`if` statements is the -same in both these snippets. - -If the expression is a tuple (e.g. the ``(x, y)`` in the previous example), -it must be parenthesized. :: - - >>> vec = [-4, -2, 0, 2, 4] - >>> # create a new list with the values doubled - >>> [x*2 for x in vec] - [-8, -4, 0, 4, 8] - >>> # filter the list to exclude negative numbers - >>> [x for x in vec if x >= 0] - [0, 2, 4] - >>> # apply a function to all the elements - >>> [abs(x) for x in vec] - [4, 2, 0, 2, 4] - >>> # call a method on each element - >>> freshfruit = [' banana', ' loganberry ', 'passion fruit '] - >>> [weapon.strip() for weapon in freshfruit] - ['banana', 'loganberry', 'passion fruit'] - >>> # create a list of 2-tuples like (number, square) - >>> [(x, x**2) for x in range(6)] - [(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25)] - >>> # the tuple must be parenthesized, otherwise an error is raised - >>> [x, x**2 for x in range(6)] - File "", line 1, in - [x, x**2 for x in range(6)] - ^ - SyntaxError: invalid syntax - >>> # flatten a list using a listcomp with two 'for' - >>> vec = [[1,2,3], [4,5,6], [7,8,9]] - >>> [num for elem in vec for num in elem] - [1, 2, 3, 4, 5, 6, 7, 8, 9] - -List comprehensions can contain complex expressions and nested functions:: - - >>> from math import pi - >>> [str(round(pi, i)) for i in range(1, 6)] - ['3.1', '3.14', '3.142', '3.1416', '3.14159'] - -Nested List Comprehensions --------------------------- - -The initial expression in a list comprehension can be any arbitrary expression, -including another list comprehension. - -Consider the following example of a 3x4 matrix implemented as a list of -3 lists of length 4:: - - >>> matrix = [ - ... [1, 2, 3, 4], - ... [5, 6, 7, 8], - ... [9, 10, 11, 12], - ... ] - -The following list comprehension will transpose rows and columns:: - - >>> [[row[i] for row in matrix] for i in range(4)] - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -As we saw in the previous section, the nested listcomp is evaluated in -the context of the :keyword:`for` that follows it, so this example is -equivalent to:: - - >>> transposed = [] - >>> for i in range(4): - ... transposed.append([row[i] for row in matrix]) - ... - >>> transposed - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -which, in turn, is the same as:: - - >>> transposed = [] - >>> for i in range(4): - ... # the following 3 lines implement the nested listcomp - ... transposed_row = [] - ... for row in matrix: - ... transposed_row.append(row[i]) - ... transposed.append(transposed_row) - ... - >>> transposed - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -In the real world, you should prefer built-in functions to complex flow statements. -The :func:`zip` function would do a great job for this use case:: - - >>> list(zip(*matrix)) - [(1, 5, 9), (2, 6, 10), (3, 7, 11), (4, 8, 12)] - -See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. - -.. _tut-del: - -The :keyword:`del` statement -============================ - -There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method -which returns a value. The :keyword:`del` statement can also be used to remove -slices from a list or clear the entire list (which we did earlier by assignment -of an empty list to the slice). For example:: - - >>> a = [-1, 1, 66.25, 333, 333, 1234.5] - >>> del a[0] - >>> a - [1, 66.25, 333, 333, 1234.5] - >>> del a[2:4] - >>> a - [1, 66.25, 1234.5] - >>> del a[:] - >>> a - [] - -:keyword:`del` can also be used to delete entire variables:: - - >>> del a - -Referencing the name ``a`` hereafter is an error (at least until another value -is assigned to it). We'll find other uses for :keyword:`del` later. - - -.. _tut-tuples: - -Tuples and Sequences -==================== - -We saw that lists and strings have many common properties, such as indexing and -slicing operations. They are two examples of *sequence* data types (see -:ref:`typesseq`). Since Python is an evolving language, other sequence data -types may be added. There is also another standard sequence data type: the -*tuple*. - -A tuple consists of a number of values separated by commas, for instance:: - - >>> t = 12345, 54321, 'hello!' - >>> t[0] - 12345 - >>> t - (12345, 54321, 'hello!') - >>> # Tuples may be nested: - ... u = t, (1, 2, 3, 4, 5) - >>> u - ((12345, 54321, 'hello!'), (1, 2, 3, 4, 5)) - >>> # Tuples are immutable: - ... t[0] = 88888 - Traceback (most recent call last): - File "", line 1, in - TypeError: 'tuple' object does not support item assignment - >>> # but they can contain mutable objects: - ... v = ([1, 2, 3], [3, 2, 1]) - >>> v - ([1, 2, 3], [3, 2, 1]) - - -As you see, on output tuples are always enclosed in parentheses, so that nested -tuples are interpreted correctly; they may be input with or without surrounding -parentheses, although often parentheses are necessary anyway (if the tuple is -part of a larger expression). It is not possible to assign to the individual -items of a tuple, however it is possible to create tuples which contain mutable -objects, such as lists. - -Though tuples may seem similar to lists, they are often used in different -situations and for different purposes. -Tuples are :term:`immutable`, and usually contain a heterogeneous sequence of -elements that are accessed via unpacking (see later in this section) or indexing -(or even by attribute in the case of :func:`namedtuples `). -Lists are :term:`mutable`, and their elements are usually homogeneous and are -accessed by iterating over the list. - -A special problem is the construction of tuples containing 0 or 1 items: the -syntax has some extra quirks to accommodate these. Empty tuples are constructed -by an empty pair of parentheses; a tuple with one item is constructed by -following a value with a comma (it is not sufficient to enclose a single value -in parentheses). Ugly, but effective. For example:: - - >>> empty = () - >>> singleton = 'hello', # <-- note trailing comma - >>> len(empty) - 0 - >>> len(singleton) - 1 - >>> singleton - ('hello',) - -The statement ``t = 12345, 54321, 'hello!'`` is an example of *tuple packing*: -the values ``12345``, ``54321`` and ``'hello!'`` are packed together in a tuple. -The reverse operation is also possible:: - - >>> x, y, z = t - -This is called, appropriately enough, *sequence unpacking* and works for any -sequence on the right-hand side. Sequence unpacking requires that there are as -many variables on the left side of the equals sign as there are elements in the -sequence. Note that multiple assignment is really just a combination of tuple -packing and sequence unpacking. - - -.. _tut-sets: - -Sets -==== - -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. - -Curly braces or the :func:`set` function can be used to create sets. Note: to -create an empty set you have to use ``set()``, not ``{}``; the latter creates an -empty dictionary, a data structure that we discuss in the next section. - -Here is a brief demonstration:: - - >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} - >>> print(basket) # show that duplicates have been removed - {'orange', 'banana', 'pear', 'apple'} - >>> 'orange' in basket # fast membership testing - True - >>> 'crabgrass' in basket - False - - >>> # Demonstrate set operations on unique letters from two words - ... - >>> a = set('abracadabra') - >>> b = set('alacazam') - >>> a # unique letters in a - {'a', 'r', 'b', 'c', 'd'} - >>> a - b # letters in a but not in b - {'r', 'd', 'b'} - >>> a | b # letters in a or b or both - {'a', 'c', 'r', 'd', 'b', 'm', 'z', 'l'} - >>> a & b # letters in both a and b - {'a', 'c'} - >>> a ^ b # letters in a or b but not both - {'r', 'd', 'b', 'm', 'z', 'l'} - -Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: - - >>> a = {x for x in 'abracadabra' if x not in 'abc'} - >>> a - {'r', 'd'} - - -.. _tut-dictionaries: - -Dictionaries -============ - -Another useful data type built into Python is the *dictionary* (see -:ref:`typesmapping`). Dictionaries are sometimes found in other languages as -"associative memories" or "associative arrays". Unlike sequences, which are -indexed by a range of numbers, dictionaries are indexed by *keys*, which can be -any immutable type; strings and numbers can always be keys. Tuples can be used -as keys if they contain only strings, numbers, or tuples; if a tuple contains -any mutable object either directly or indirectly, it cannot be used as a key. -You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. - -It is best to think of a dictionary as an unordered set of *key: value* pairs, -with the requirement that the keys are unique (within one dictionary). A pair of -braces creates an empty dictionary: ``{}``. Placing a comma-separated list of -key:value pairs within the braces adds initial key:value pairs to the -dictionary; this is also the way dictionaries are written on output. - -The main operations on a dictionary are storing a value with some key and -extracting the value given the key. It is also possible to delete a key:value -pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. - -Performing ``list(d.keys())`` on a dictionary returns a list of all the keys -used in the dictionary, in arbitrary order (if you want it sorted, just use -``sorted(d.keys())`` instead). [2]_ To check whether a single key is in the -dictionary, use the :keyword:`in` keyword. - -Here is a small example using a dictionary:: - - >>> tel = {'jack': 4098, 'sape': 4139} - >>> tel['guido'] = 4127 - >>> tel - {'sape': 4139, 'guido': 4127, 'jack': 4098} - >>> tel['jack'] - 4098 - >>> del tel['sape'] - >>> tel['irv'] = 4127 - >>> tel - {'guido': 4127, 'irv': 4127, 'jack': 4098} - >>> list(tel.keys()) - ['irv', 'guido', 'jack'] - >>> sorted(tel.keys()) - ['guido', 'irv', 'jack'] - >>> 'guido' in tel - True - >>> 'jack' not in tel - False - -The :func:`dict` constructor builds dictionaries directly from sequences of -key-value pairs:: - - >>> dict([('sape', 4139), ('guido', 4127), ('jack', 4098)]) - {'sape': 4139, 'jack': 4098, 'guido': 4127} - -In addition, dict comprehensions can be used to create dictionaries from -arbitrary key and value expressions:: - - >>> {x: x**2 for x in (2, 4, 6)} - {2: 4, 4: 16, 6: 36} - -When the keys are simple strings, it is sometimes easier to specify pairs using -keyword arguments:: - - >>> dict(sape=4139, guido=4127, jack=4098) - {'sape': 4139, 'jack': 4098, 'guido': 4127} - - -.. _tut-loopidioms: - -Looping Techniques -================== - -When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: - - >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} - >>> for k, v in knights.items(): - ... print(k, v) - ... - gallahad the pure - robin the brave - -When looping through a sequence, the position index and corresponding value can -be retrieved at the same time using the :func:`enumerate` function. :: - - >>> for i, v in enumerate(['tic', 'tac', 'toe']): - ... print(i, v) - ... - 0 tic - 1 tac - 2 toe - -To loop over two or more sequences at the same time, the entries can be paired -with the :func:`zip` function. :: - - >>> questions = ['name', 'quest', 'favorite color'] - >>> answers = ['lancelot', 'the holy grail', 'blue'] - >>> for q, a in zip(questions, answers): - ... print('What is your {0}? It is {1}.'.format(q, a)) - ... - What is your name? It is lancelot. - What is your quest? It is the holy grail. - What is your favorite color? It is blue. - -To loop over a sequence in reverse, first specify the sequence in a forward -direction and then call the :func:`reversed` function. :: - - >>> for i in reversed(range(1, 10, 2)): - ... print(i) - ... - 9 - 7 - 5 - 3 - 1 - -To loop over a sequence in sorted order, use the :func:`sorted` function which -returns a new sorted list while leaving the source unaltered. :: - - >>> basket = ['apple', 'orange', 'apple', 'pear', 'orange', 'banana'] - >>> for f in sorted(set(basket)): - ... print(f) - ... - apple - banana - orange - pear - -It is sometimes tempting to change a list while you are looping over it; -however, it is often simpler and safer to create a new list instead. :: - - >>> import math - >>> raw_data = [56.2, float('NaN'), 51.7, 55.3, 52.5, float('NaN'), 47.8] - >>> filtered_data = [] - >>> for value in raw_data: - ... if not math.isnan(value): - ... filtered_data.append(value) - ... - >>> filtered_data - [56.2, 51.7, 55.3, 52.5, 47.8] - - -.. _tut-conditions: - -More on Conditions -================== - -The conditions used in ``while`` and ``if`` statements can contain any -operators, not just comparisons. - -The comparison operators ``in`` and ``not in`` check whether a value occurs -(does not occur) in a sequence. The operators ``is`` and ``is not`` compare -whether two objects are really the same object; this only matters for mutable -objects like lists. All comparison operators have the same priority, which is -lower than that of all numerical operators. - -Comparisons can be chained. For example, ``a < b == c`` tests whether ``a`` is -less than ``b`` and moreover ``b`` equals ``c``. - -Comparisons may be combined using the Boolean operators ``and`` and ``or``, and -the outcome of a comparison (or of any other Boolean expression) may be negated -with ``not``. These have lower priorities than comparison operators; between -them, ``not`` has the highest priority and ``or`` the lowest, so that ``A and -not B or C`` is equivalent to ``(A and (not B)) or C``. As always, parentheses -can be used to express the desired composition. - -The Boolean operators ``and`` and ``or`` are so-called *short-circuit* -operators: their arguments are evaluated from left to right, and evaluation -stops as soon as the outcome is determined. For example, if ``A`` and ``C`` are -true but ``B`` is false, ``A and B and C`` does not evaluate the expression -``C``. When used as a general value and not as a Boolean, the return value of a -short-circuit operator is the last evaluated argument. - -It is possible to assign the result of a comparison or other Boolean expression -to a variable. For example, :: - - >>> string1, string2, string3 = '', 'Trondheim', 'Hammer Dance' - >>> non_null = string1 or string2 or string3 - >>> non_null - 'Trondheim' - -Note that in Python, unlike C, assignment cannot occur inside expressions. C -programmers may grumble about this, but it avoids a common class of problems -encountered in C programs: typing ``=`` in an expression when ``==`` was -intended. - - -.. _tut-comparing: - -Comparing Sequences and Other Types -=================================== - -Sequence objects may be compared to other objects with the same sequence type. -The comparison uses *lexicographical* ordering: first the first two items are -compared, and if they differ this determines the outcome of the comparison; if -they are equal, the next two items are compared, and so on, until either -sequence is exhausted. If two items to be compared are themselves sequences of -the same type, the lexicographical comparison is carried out recursively. If -all items of two sequences compare equal, the sequences are considered equal. -If one sequence is an initial sub-sequence of the other, the shorter sequence is -the smaller (lesser) one. Lexicographical ordering for strings uses the Unicode -code point number to order individual characters. Some examples of comparisons -between sequences of the same type:: - - (1, 2, 3) < (1, 2, 4) - [1, 2, 3] < [1, 2, 4] - 'ABC' < 'C' < 'Pascal' < 'Python' - (1, 2, 3, 4) < (1, 2, 4) - (1, 2) < (1, 2, -1) - (1, 2, 3) == (1.0, 2.0, 3.0) - (1, 2, ('aa', 'ab')) < (1, 2, ('abc', 'a'), 4) - -Note that comparing objects of different types with ``<`` or ``>`` is legal -provided that the objects have appropriate comparison methods. For example, -mixed numeric types are compared according to their numeric value, so 0 equals -0.0, etc. Otherwise, rather than providing an arbitrary ordering, the -interpreter will raise a :exc:`TypeError` exception. - - -.. rubric:: Footnotes - -.. [1] Other languages may return the mutated object, which allows method - chaining, such as ``d->insert("a")->remove("b")->sort();``. - -.. [2] Calling ``d.keys()`` will return a :dfn:`dictionary view` object. It - supports operations like membership test and iteration, but its contents - are not independent of the original dictionary -- it is only a *view*. diff --git a/third_party/python/Doc/tutorial/errors.rst b/third_party/python/Doc/tutorial/errors.rst deleted file mode 100644 index 957cbf962..000000000 --- a/third_party/python/Doc/tutorial/errors.rst +++ /dev/null @@ -1,414 +0,0 @@ -.. _tut-errors: - -********************* -Errors and Exceptions -********************* - -Until now error messages haven't been more than mentioned, but if you have tried -out the examples you have probably seen some. There are (at least) two -distinguishable kinds of errors: *syntax errors* and *exceptions*. - - -.. _tut-syntaxerrors: - -Syntax Errors -============= - -Syntax errors, also known as parsing errors, are perhaps the most common kind of -complaint you get while you are still learning Python:: - - >>> while True print('Hello world') - File "", line 1 - while True print('Hello world') - ^ - SyntaxError: invalid syntax - -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the -example, the error is detected at the function :func:`print`, since a colon -(``':'``) is missing before it. File name and line number are printed so you -know where to look in case the input came from a script. - - -.. _tut-exceptions: - -Exceptions -========== - -Even if a statement or expression is syntactically correct, it may cause an -error when an attempt is made to execute it. Errors detected during execution -are called *exceptions* and are not unconditionally fatal: you will soon learn -how to handle them in Python programs. Most exceptions are not handled by -programs, however, and result in error messages as shown here:: - - >>> 10 * (1/0) - Traceback (most recent call last): - File "", line 1, in - ZeroDivisionError: division by zero - >>> 4 + spam*3 - Traceback (most recent call last): - File "", line 1, in - NameError: name 'spam' is not defined - >>> '2' + 2 - Traceback (most recent call last): - File "", line 1, in - TypeError: Can't convert 'int' object to str implicitly - -The last line of the error message indicates what happened. Exceptions come in -different types, and the type is printed as part of the message: the types in -the example are :exc:`ZeroDivisionError`, :exc:`NameError` and :exc:`TypeError`. -The string printed as the exception type is the name of the built-in exception -that occurred. This is true for all built-in exceptions, but need not be true -for user-defined exceptions (although it is a useful convention). Standard -exception names are built-in identifiers (not reserved keywords). - -The rest of the line provides detail based on the type of exception and what -caused it. - -The preceding part of the error message shows the context where the exception -happened, in the form of a stack traceback. In general it contains a stack -traceback listing source lines; however, it will not display lines read from -standard input. - -:ref:`bltin-exceptions` lists the built-in exceptions and their meanings. - - -.. _tut-handling: - -Handling Exceptions -=================== - -It is possible to write programs that handle selected exceptions. Look at the -following example, which asks the user for input until a valid integer has been -entered, but allows the user to interrupt the program (using :kbd:`Control-C` or -whatever the operating system supports); note that a user-generated interruption -is signalled by raising the :exc:`KeyboardInterrupt` exception. :: - - >>> while True: - ... try: - ... x = int(input("Please enter a number: ")) - ... break - ... except ValueError: - ... print("Oops! That was no valid number. Try again...") - ... - -The :keyword:`try` statement works as follows. - -* First, the *try clause* (the statement(s) between the :keyword:`try` and - :keyword:`except` keywords) is executed. - -* If no exception occurs, the *except clause* is skipped and execution of the - :keyword:`try` statement is finished. - -* If an exception occurs during execution of the try clause, the rest of the - clause is skipped. Then if its type matches the exception named after the - :keyword:`except` keyword, the except clause is executed, and then execution - continues after the :keyword:`try` statement. - -* If an exception occurs which does not match the exception named in the except - clause, it is passed on to outer :keyword:`try` statements; if no handler is - found, it is an *unhandled exception* and execution stops with a message as - shown above. - -A :keyword:`try` statement may have more than one except clause, to specify -handlers for different exceptions. At most one handler will be executed. -Handlers only handle exceptions that occur in the corresponding try clause, not -in other handlers of the same :keyword:`try` statement. An except clause may -name multiple exceptions as a parenthesized tuple, for example:: - - ... except (RuntimeError, TypeError, NameError): - ... pass - -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -except clause listing a derived class is not compatible with a base class). For -example, the following code will print B, C, D in that order:: - - class B(Exception): - pass - - class C(B): - pass - - class D(C): - pass - - for cls in [B, C, D]: - try: - raise cls() - except D: - print("D") - except C: - print("C") - except B: - print("B") - -Note that if the except clauses were reversed (with ``except B`` first), it -would have printed B, B, B --- the first matching except clause is triggered. - -The last except clause may omit the exception name(s), to serve as a wildcard. -Use this with extreme caution, since it is easy to mask a real programming error -in this way! It can also be used to print an error message and then re-raise -the exception (allowing a caller to handle the exception as well):: - - import sys - - try: - f = open('myfile.txt') - s = f.readline() - i = int(s.strip()) - except OSError as err: - print("OS error: {0}".format(err)) - except ValueError: - print("Could not convert data to an integer.") - except: - print("Unexpected error:", sys.exc_info()[0]) - raise - -The :keyword:`try` ... :keyword:`except` statement has an optional *else -clause*, which, when present, must follow all except clauses. It is useful for -code that must be executed if the try clause does not raise an exception. For -example:: - - for arg in sys.argv[1:]: - try: - f = open(arg, 'r') - except OSError: - print('cannot open', arg) - else: - print(arg, 'has', len(f.readlines()), 'lines') - f.close() - -The use of the :keyword:`else` clause is better than adding additional code to -the :keyword:`try` clause because it avoids accidentally catching an exception -that wasn't raised by the code being protected by the :keyword:`try` ... -:keyword:`except` statement. - -When an exception occurs, it may have an associated value, also known as the -exception's *argument*. The presence and type of the argument depend on the -exception type. - -The except clause may specify a variable after the exception name. The -variable is bound to an exception instance with the arguments stored in -``instance.args``. For convenience, the exception instance defines -:meth:`__str__` so the arguments can be printed directly without having to -reference ``.args``. One may also instantiate an exception first before -raising it and add any attributes to it as desired. :: - - >>> try: - ... raise Exception('spam', 'eggs') - ... except Exception as inst: - ... print(type(inst)) # the exception instance - ... print(inst.args) # arguments stored in .args - ... print(inst) # __str__ allows args to be printed directly, - ... # but may be overridden in exception subclasses - ... x, y = inst.args # unpack args - ... print('x =', x) - ... print('y =', y) - ... - - ('spam', 'eggs') - ('spam', 'eggs') - x = spam - y = eggs - -If an exception has arguments, they are printed as the last part ('detail') of -the message for unhandled exceptions. - -Exception handlers don't just handle exceptions if they occur immediately in the -try clause, but also if they occur inside functions that are called (even -indirectly) in the try clause. For example:: - - >>> def this_fails(): - ... x = 1/0 - ... - >>> try: - ... this_fails() - ... except ZeroDivisionError as err: - ... print('Handling run-time error:', err) - ... - Handling run-time error: division by zero - - -.. _tut-raising: - -Raising Exceptions -================== - -The :keyword:`raise` statement allows the programmer to force a specified -exception to occur. For example:: - - >>> raise NameError('HiThere') - Traceback (most recent call last): - File "", line 1, in - NameError: HiThere - -The sole argument to :keyword:`raise` indicates the exception to be raised. -This must be either an exception instance or an exception class (a class that -derives from :class:`Exception`). If an exception class is passed, it will -be implicitly instantiated by calling its constructor with no arguments:: - - raise ValueError # shorthand for 'raise ValueError()' - -If you need to determine whether an exception was raised but don't intend to -handle it, a simpler form of the :keyword:`raise` statement allows you to -re-raise the exception:: - - >>> try: - ... raise NameError('HiThere') - ... except NameError: - ... print('An exception flew by!') - ... raise - ... - An exception flew by! - Traceback (most recent call last): - File "", line 2, in - NameError: HiThere - - -.. _tut-userexceptions: - -User-defined Exceptions -======================= - -Programs may name their own exceptions by creating a new exception class (see -:ref:`tut-classes` for more about Python classes). Exceptions should typically -be derived from the :exc:`Exception` class, either directly or indirectly. - -Exception classes can be defined which do anything any other class can do, but -are usually kept simple, often only offering a number of attributes that allow -information about the error to be extracted by handlers for the exception. When -creating a module that can raise several distinct errors, a common practice is -to create a base class for exceptions defined by that module, and subclass that -to create specific exception classes for different error conditions:: - - class Error(Exception): - """Base class for exceptions in this module.""" - pass - - class InputError(Error): - """Exception raised for errors in the input. - - Attributes: - expression -- input expression in which the error occurred - message -- explanation of the error - """ - - def __init__(self, expression, message): - self.expression = expression - self.message = message - - class TransitionError(Error): - """Raised when an operation attempts a state transition that's not - allowed. - - Attributes: - previous -- state at beginning of transition - next -- attempted new state - message -- explanation of why the specific transition is not allowed - """ - - def __init__(self, previous, next, message): - self.previous = previous - self.next = next - self.message = message - -Most exceptions are defined with names that end in "Error", similar to the -naming of the standard exceptions. - -Many standard modules define their own exceptions to report errors that may -occur in functions they define. More information on classes is presented in -chapter :ref:`tut-classes`. - - -.. _tut-cleanup: - -Defining Clean-up Actions -========================= - -The :keyword:`try` statement has another optional clause which is intended to -define clean-up actions that must be executed under all circumstances. For -example:: - - >>> try: - ... raise KeyboardInterrupt - ... finally: - ... print('Goodbye, world!') - ... - Goodbye, world! - Traceback (most recent call last): - File "", line 2, in - KeyboardInterrupt - -A *finally clause* is always executed before leaving the :keyword:`try` -statement, whether an exception has occurred or not. When an exception has -occurred in the :keyword:`try` clause and has not been handled by an -:keyword:`except` clause (or it has occurred in an :keyword:`except` or -:keyword:`else` clause), it is re-raised after the :keyword:`finally` clause has -been executed. The :keyword:`finally` clause is also executed "on the way out" -when any other clause of the :keyword:`try` statement is left via a -:keyword:`break`, :keyword:`continue` or :keyword:`return` statement. A more -complicated example:: - - >>> def divide(x, y): - ... try: - ... result = x / y - ... except ZeroDivisionError: - ... print("division by zero!") - ... else: - ... print("result is", result) - ... finally: - ... print("executing finally clause") - ... - >>> divide(2, 1) - result is 2.0 - executing finally clause - >>> divide(2, 0) - division by zero! - executing finally clause - >>> divide("2", "1") - executing finally clause - Traceback (most recent call last): - File "", line 1, in - File "", line 3, in divide - TypeError: unsupported operand type(s) for /: 'str' and 'str' - -As you can see, the :keyword:`finally` clause is executed in any event. The -:exc:`TypeError` raised by dividing two strings is not handled by the -:keyword:`except` clause and therefore re-raised after the :keyword:`finally` -clause has been executed. - -In real world applications, the :keyword:`finally` clause is useful for -releasing external resources (such as files or network connections), regardless -of whether the use of the resource was successful. - - -.. _tut-cleanup-with: - -Predefined Clean-up Actions -=========================== - -Some objects define standard clean-up actions to be undertaken when the object -is no longer needed, regardless of whether or not the operation using the object -succeeded or failed. Look at the following example, which tries to open a file -and print its contents to the screen. :: - - for line in open("myfile.txt"): - print(line, end="") - -The problem with this code is that it leaves the file open for an indeterminate -amount of time after this part of the code has finished executing. -This is not an issue in simple scripts, but can be a problem for larger -applications. The :keyword:`with` statement allows objects like files to be -used in a way that ensures they are always cleaned up promptly and correctly. :: - - with open("myfile.txt") as f: - for line in f: - print(line, end="") - -After the statement is executed, the file *f* is always closed, even if a -problem was encountered while processing the lines. Objects which, like files, -provide predefined clean-up actions will indicate this in their documentation. - - diff --git a/third_party/python/Doc/tutorial/floatingpoint.rst b/third_party/python/Doc/tutorial/floatingpoint.rst deleted file mode 100644 index 0c0eb526f..000000000 --- a/third_party/python/Doc/tutorial/floatingpoint.rst +++ /dev/null @@ -1,300 +0,0 @@ -.. testsetup:: - - import math - -.. _tut-fp-issues: - -************************************************** -Floating Point Arithmetic: Issues and Limitations -************************************************** - -.. sectionauthor:: Tim Peters - - -Floating-point numbers are represented in computer hardware as base 2 (binary) -fractions. For example, the decimal fraction :: - - 0.125 - -has value 1/10 + 2/100 + 5/1000, and in the same way the binary fraction :: - - 0.001 - -has value 0/2 + 0/4 + 1/8. These two fractions have identical values, the only -real difference being that the first is written in base 10 fractional notation, -and the second in base 2. - -Unfortunately, most decimal fractions cannot be represented exactly as binary -fractions. A consequence is that, in general, the decimal floating-point -numbers you enter are only approximated by the binary floating-point numbers -actually stored in the machine. - -The problem is easier to understand at first in base 10. Consider the fraction -1/3. You can approximate that as a base 10 fraction:: - - 0.3 - -or, better, :: - - 0.33 - -or, better, :: - - 0.333 - -and so on. No matter how many digits you're willing to write down, the result -will never be exactly 1/3, but will be an increasingly better approximation of -1/3. - -In the same way, no matter how many base 2 digits you're willing to use, the -decimal value 0.1 cannot be represented exactly as a base 2 fraction. In base -2, 1/10 is the infinitely repeating fraction :: - - 0.0001100110011001100110011001100110011001100110011... - -Stop at any finite number of bits, and you get an approximation. On most -machines today, floats are approximated using a binary fraction with -the numerator using the first 53 bits starting with the most significant bit and -with the denominator as a power of two. In the case of 1/10, the binary fraction -is ``3602879701896397 / 2 ** 55`` which is close to but not exactly -equal to the true value of 1/10. - -Many users are not aware of the approximation because of the way values are -displayed. Python only prints a decimal approximation to the true decimal -value of the binary approximation stored by the machine. On most machines, if -Python were to print the true decimal value of the binary approximation stored -for 0.1, it would have to display :: - - >>> 0.1 - 0.1000000000000000055511151231257827021181583404541015625 - -That is more digits than most people find useful, so Python keeps the number -of digits manageable by displaying a rounded value instead :: - - >>> 1 / 10 - 0.1 - -Just remember, even though the printed result looks like the exact value -of 1/10, the actual stored value is the nearest representable binary fraction. - -Interestingly, there are many different decimal numbers that share the same -nearest approximate binary fraction. For example, the numbers ``0.1`` and -``0.10000000000000001`` and -``0.1000000000000000055511151231257827021181583404541015625`` are all -approximated by ``3602879701896397 / 2 ** 55``. Since all of these decimal -values share the same approximation, any one of them could be displayed -while still preserving the invariant ``eval(repr(x)) == x``. - -Historically, the Python prompt and built-in :func:`repr` function would choose -the one with 17 significant digits, ``0.10000000000000001``. Starting with -Python 3.1, Python (on most systems) is now able to choose the shortest of -these and simply display ``0.1``. - -Note that this is in the very nature of binary floating-point: this is not a bug -in Python, and it is not a bug in your code either. You'll see the same kind of -thing in all languages that support your hardware's floating-point arithmetic -(although some languages may not *display* the difference by default, or in all -output modes). - -For more pleasant output, you may wish to use string formatting to produce a limited number of significant digits:: - - >>> format(math.pi, '.12g') # give 12 significant digits - '3.14159265359' - - >>> format(math.pi, '.2f') # give 2 digits after the point - '3.14' - - >>> repr(math.pi) - '3.141592653589793' - - -It's important to realize that this is, in a real sense, an illusion: you're -simply rounding the *display* of the true machine value. - -One illusion may beget another. For example, since 0.1 is not exactly 1/10, -summing three values of 0.1 may not yield exactly 0.3, either:: - - >>> .1 + .1 + .1 == .3 - False - -Also, since the 0.1 cannot get any closer to the exact value of 1/10 and -0.3 cannot get any closer to the exact value of 3/10, then pre-rounding with -:func:`round` function cannot help:: - - >>> round(.1, 1) + round(.1, 1) + round(.1, 1) == round(.3, 1) - False - -Though the numbers cannot be made closer to their intended exact values, -the :func:`round` function can be useful for post-rounding so that results -with inexact values become comparable to one another:: - - >>> round(.1 + .1 + .1, 10) == round(.3, 10) - True - -Binary floating-point arithmetic holds many surprises like this. The problem -with "0.1" is explained in precise detail below, in the "Representation Error" -section. See `The Perils of Floating Point `_ -for a more complete account of other common surprises. - -As that says near the end, "there are no easy answers." Still, don't be unduly -wary of floating-point! The errors in Python float operations are inherited -from the floating-point hardware, and on most machines are on the order of no -more than 1 part in 2\*\*53 per operation. That's more than adequate for most -tasks, but you do need to keep in mind that it's not decimal arithmetic and -that every float operation can suffer a new rounding error. - -While pathological cases do exist, for most casual use of floating-point -arithmetic you'll see the result you expect in the end if you simply round the -display of your final results to the number of decimal digits you expect. -:func:`str` usually suffices, and for finer control see the :meth:`str.format` -method's format specifiers in :ref:`formatstrings`. - -For use cases which require exact decimal representation, try using the -:mod:`decimal` module which implements decimal arithmetic suitable for -accounting applications and high-precision applications. - -Another form of exact arithmetic is supported by the :mod:`fractions` module -which implements arithmetic based on rational numbers (so the numbers like -1/3 can be represented exactly). - -If you are a heavy user of floating point operations you should take a look -at the Numerical Python package and many other packages for mathematical and -statistical operations supplied by the SciPy project. See . - -Python provides tools that may help on those rare occasions when you really -*do* want to know the exact value of a float. The -:meth:`float.as_integer_ratio` method expresses the value of a float as a -fraction:: - - >>> x = 3.14159 - >>> x.as_integer_ratio() - (3537115888337719, 1125899906842624) - -Since the ratio is exact, it can be used to losslessly recreate the -original value:: - - >>> x == 3537115888337719 / 1125899906842624 - True - -The :meth:`float.hex` method expresses a float in hexadecimal (base -16), again giving the exact value stored by your computer:: - - >>> x.hex() - '0x1.921f9f01b866ep+1' - -This precise hexadecimal representation can be used to reconstruct -the float value exactly:: - - >>> x == float.fromhex('0x1.921f9f01b866ep+1') - True - -Since the representation is exact, it is useful for reliably porting values -across different versions of Python (platform independence) and exchanging -data with other languages that support the same format (such as Java and C99). - -Another helpful tool is the :func:`math.fsum` function which helps mitigate -loss-of-precision during summation. It tracks "lost digits" as values are -added onto a running total. That can make a difference in overall accuracy -so that the errors do not accumulate to the point where they affect the -final total: - - >>> sum([0.1] * 10) == 1.0 - False - >>> math.fsum([0.1] * 10) == 1.0 - True - -.. _tut-fp-error: - -Representation Error -==================== - -This section explains the "0.1" example in detail, and shows how you can perform -an exact analysis of cases like this yourself. Basic familiarity with binary -floating-point representation is assumed. - -:dfn:`Representation error` refers to the fact that some (most, actually) -decimal fractions cannot be represented exactly as binary (base 2) fractions. -This is the chief reason why Python (or Perl, C, C++, Java, Fortran, and many -others) often won't display the exact decimal number you expect. - -Why is that? 1/10 is not exactly representable as a binary fraction. Almost all -machines today (November 2000) use IEEE-754 floating point arithmetic, and -almost all platforms map Python floats to IEEE-754 "double precision". 754 -doubles contain 53 bits of precision, so on input the computer strives to -convert 0.1 to the closest fraction it can of the form *J*/2**\ *N* where *J* is -an integer containing exactly 53 bits. Rewriting :: - - 1 / 10 ~= J / (2**N) - -as :: - - J ~= 2**N / 10 - -and recalling that *J* has exactly 53 bits (is ``>= 2**52`` but ``< 2**53``), -the best value for *N* is 56:: - - >>> 2**52 <= 2**56 // 10 < 2**53 - True - -That is, 56 is the only value for *N* that leaves *J* with exactly 53 bits. The -best possible value for *J* is then that quotient rounded:: - - >>> q, r = divmod(2**56, 10) - >>> r - 6 - -Since the remainder is more than half of 10, the best approximation is obtained -by rounding up:: - - >>> q+1 - 7205759403792794 - -Therefore the best possible approximation to 1/10 in 754 double precision is:: - - 7205759403792794 / 2 ** 56 - -Dividing both the numerator and denominator by two reduces the fraction to:: - - 3602879701896397 / 2 ** 55 - -Note that since we rounded up, this is actually a little bit larger than 1/10; -if we had not rounded up, the quotient would have been a little bit smaller than -1/10. But in no case can it be *exactly* 1/10! - -So the computer never "sees" 1/10: what it sees is the exact fraction given -above, the best 754 double approximation it can get:: - - >>> 0.1 * 2 ** 55 - 3602879701896397.0 - -If we multiply that fraction by 10\*\*55, we can see the value out to -55 decimal digits:: - - >>> 3602879701896397 * 10 ** 55 // 2 ** 55 - 1000000000000000055511151231257827021181583404541015625 - -meaning that the exact number stored in the computer is equal to -the decimal value 0.1000000000000000055511151231257827021181583404541015625. -Instead of displaying the full decimal value, many languages (including -older versions of Python), round the result to 17 significant digits:: - - >>> format(0.1, '.17f') - '0.10000000000000001' - -The :mod:`fractions` and :mod:`decimal` modules make these calculations -easy:: - - >>> from decimal import Decimal - >>> from fractions import Fraction - - >>> Fraction.from_float(0.1) - Fraction(3602879701896397, 36028797018963968) - - >>> (0.1).as_integer_ratio() - (3602879701896397, 36028797018963968) - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - - >>> format(Decimal.from_float(0.1), '.17') - '0.10000000000000001' diff --git a/third_party/python/Doc/tutorial/index.rst b/third_party/python/Doc/tutorial/index.rst deleted file mode 100644 index 8ee011e28..000000000 --- a/third_party/python/Doc/tutorial/index.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. _tutorial-index: - -###################### - The Python Tutorial -###################### - -Python is an easy to learn, powerful programming language. It has efficient -high-level data structures and a simple but effective approach to -object-oriented programming. Python's elegant syntax and dynamic typing, -together with its interpreted nature, make it an ideal language for scripting -and rapid application development in many areas on most platforms. - -The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python Web site, -https://www.python.org/, and may be freely distributed. The same site also -contains distributions of and pointers to many free third party Python modules, -programs and tools, and additional documentation. - -The Python interpreter is easily extended with new functions and data types -implemented in C or C++ (or other languages callable from C). Python is also -suitable as an extension language for customizable applications. - -This tutorial introduces the reader informally to the basic concepts and -features of the Python language and system. It helps to have a Python -interpreter handy for hands-on experience, but all examples are self-contained, -so the tutorial can be read off-line as well. - -For a description of standard objects and modules, see :ref:`library-index`. -:ref:`reference-index` gives a more formal definition of the language. To write -extensions in C or C++, read :ref:`extending-index` and -:ref:`c-api-index`. There are also several books covering Python in depth. - -This tutorial does not attempt to be comprehensive and cover every single -feature, or even every commonly used feature. Instead, it introduces many of -Python's most noteworthy features, and will give you a good idea of the -language's flavor and style. After reading it, you will be able to read and -write Python modules and programs, and you will be ready to learn more about the -various Python library modules described in :ref:`library-index`. - -The :ref:`glossary` is also worth going through. - -.. toctree:: - :numbered: - - appetite.rst - interpreter.rst - introduction.rst - controlflow.rst - datastructures.rst - modules.rst - inputoutput.rst - errors.rst - classes.rst - stdlib.rst - stdlib2.rst - venv.rst - whatnow.rst - interactive.rst - floatingpoint.rst - appendix.rst diff --git a/third_party/python/Doc/tutorial/inputoutput.rst b/third_party/python/Doc/tutorial/inputoutput.rst deleted file mode 100644 index 9fd939a20..000000000 --- a/third_party/python/Doc/tutorial/inputoutput.rst +++ /dev/null @@ -1,451 +0,0 @@ -.. _tut-io: - -**************** -Input and Output -**************** - -There are several ways to present the output of a program; data can be printed -in a human-readable form, or written to a file for future use. This chapter will -discuss some of the possibilities. - - -.. _tut-formatting: - -Fancier Output Formatting -========================= - -So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method -of file objects; the standard output file can be referenced as ``sys.stdout``. -See the Library Reference for more information on this.) - -Often you'll want more control over the formatting of your output than simply -printing space-separated values. There are two ways to format your output; the -first way is to do all the string handling yourself; using string slicing and -concatenation operations you can create any layout you can imagine. The -string type has some methods that perform useful operations for padding -strings to a given column width; these will be discussed shortly. The second -way is to use :ref:`formatted string literals `, or the -:meth:`str.format` method. - -The :mod:`string` module contains a :class:`~string.Template` class which offers -yet another way to substitute values into strings. - -One question remains, of course: how do you convert values to strings? Luckily, -Python has ways to convert any value to a string: pass it to the :func:`repr` -or :func:`str` functions. - -The :func:`str` function is meant to return representations of values which are -fairly human-readable, while :func:`repr` is meant to generate representations -which can be read by the interpreter (or will force a :exc:`SyntaxError` if -there is no equivalent syntax). For objects which don't have a particular -representation for human consumption, :func:`str` will return the same value as -:func:`repr`. Many values, such as numbers or structures like lists and -dictionaries, have the same representation using either function. Strings, in -particular, have two distinct representations. - -Some examples:: - - >>> s = 'Hello, world.' - >>> str(s) - 'Hello, world.' - >>> repr(s) - "'Hello, world.'" - >>> str(1/7) - '0.14285714285714285' - >>> x = 10 * 3.25 - >>> y = 200 * 200 - >>> s = 'The value of x is ' + repr(x) + ', and y is ' + repr(y) + '...' - >>> print(s) - The value of x is 32.5, and y is 40000... - >>> # The repr() of a string adds string quotes and backslashes: - ... hello = 'hello, world\n' - >>> hellos = repr(hello) - >>> print(hellos) - 'hello, world\n' - >>> # The argument to repr() may be any Python object: - ... repr((x, y, ('spam', 'eggs'))) - "(32.5, 40000, ('spam', 'eggs'))" - -Here are two ways to write a table of squares and cubes:: - - >>> for x in range(1, 11): - ... print(repr(x).rjust(2), repr(x*x).rjust(3), end=' ') - ... # Note use of 'end' on previous line - ... print(repr(x*x*x).rjust(4)) - ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 - - >>> for x in range(1, 11): - ... print('{0:2d} {1:3d} {2:4d}'.format(x, x*x, x*x*x)) - ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 - -(Note that in the first example, one space between each column was added by the -way :func:`print` works: by default it adds spaces between its arguments.) - -This example demonstrates the :meth:`str.rjust` method of string -objects, which right-justifies a string in a field of a given width by padding -it with spaces on the left. There are similar methods :meth:`str.ljust` and -:meth:`str.center`. These methods do not write anything, they just return a -new string. If the input string is too long, they don't truncate it, but -return it unchanged; this will mess up your column lay-out but that's usually -better than the alternative, which would be lying about a value. (If you -really want truncation you can always add a slice operation, as in -``x.ljust(n)[:n]``.) - -There is another method, :meth:`str.zfill`, which pads a numeric string on the -left with zeros. It understands about plus and minus signs:: - - >>> '12'.zfill(5) - '00012' - >>> '-3.14'.zfill(7) - '-003.14' - >>> '3.14159265359'.zfill(5) - '3.14159265359' - -Basic usage of the :meth:`str.format` method looks like this:: - - >>> print('We are the {} who say "{}!"'.format('knights', 'Ni')) - We are the knights who say "Ni!" - -The brackets and characters within them (called format fields) are replaced with -the objects passed into the :meth:`str.format` method. A number in the -brackets can be used to refer to the position of the object passed into the -:meth:`str.format` method. :: - - >>> print('{0} and {1}'.format('spam', 'eggs')) - spam and eggs - >>> print('{1} and {0}'.format('spam', 'eggs')) - eggs and spam - -If keyword arguments are used in the :meth:`str.format` method, their values -are referred to by using the name of the argument. :: - - >>> print('This {food} is {adjective}.'.format( - ... food='spam', adjective='absolutely horrible')) - This spam is absolutely horrible. - -Positional and keyword arguments can be arbitrarily combined:: - - >>> print('The story of {0}, {1}, and {other}.'.format('Bill', 'Manfred', - other='Georg')) - The story of Bill, Manfred, and Georg. - -``'!a'`` (apply :func:`ascii`), ``'!s'`` (apply :func:`str`) and ``'!r'`` -(apply :func:`repr`) can be used to convert the value before it is formatted:: - - >>> contents = 'eels' - >>> print('My hovercraft is full of {}.'.format(contents)) - My hovercraft is full of eels. - >>> print('My hovercraft is full of {!r}.'.format(contents)) - My hovercraft is full of 'eels'. - -An optional ``':'`` and format specifier can follow the field name. This allows -greater control over how the value is formatted. The following example -rounds Pi to three places after the decimal. - - >>> import math - >>> print('The value of PI is approximately {0:.3f}.'.format(math.pi)) - The value of PI is approximately 3.142. - -Passing an integer after the ``':'`` will cause that field to be a minimum -number of characters wide. This is useful for making tables pretty. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 7678} - >>> for name, phone in table.items(): - ... print('{0:10} ==> {1:10d}'.format(name, phone)) - ... - Jack ==> 4098 - Dcab ==> 7678 - Sjoerd ==> 4127 - -If you have a really long format string that you don't want to split up, it -would be nice if you could reference the variables to be formatted by name -instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} - >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' - ... 'Dcab: {0[Dcab]:d}'.format(table)) - Jack: 4098; Sjoerd: 4127; Dcab: 8637678 - -This could also be done by passing the table as keyword arguments with the '**' -notation. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} - >>> print('Jack: {Jack:d}; Sjoerd: {Sjoerd:d}; Dcab: {Dcab:d}'.format(**table)) - Jack: 4098; Sjoerd: 4127; Dcab: 8637678 - -This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. - -For a complete overview of string formatting with :meth:`str.format`, see -:ref:`formatstrings`. - - -Old string formatting ---------------------- - -The ``%`` operator can also be used for string formatting. It interprets the -left argument much like a :c:func:`sprintf`\ -style format string to be applied -to the right argument, and returns the string resulting from this formatting -operation. For example:: - - >>> import math - >>> print('The value of PI is approximately %5.3f.' % math.pi) - The value of PI is approximately 3.142. - -More information can be found in the :ref:`old-string-formatting` section. - - -.. _tut-files: - -Reading and Writing Files -========================= - -.. index:: - builtin: open - object: file - -:func:`open` returns a :term:`file object`, and is most commonly used with -two arguments: ``open(filename, mode)``. - -:: - - >>> f = open('workfile', 'w') - -.. XXX str(f) is - - >>> print(f) - - -The first argument is a string containing the filename. The second argument is -another string containing a few characters describing the way in which the file -will be used. *mode* can be ``'r'`` when the file will only be read, ``'w'`` -for only writing (an existing file with the same name will be erased), and -``'a'`` opens the file for appending; any data written to the file is -automatically added to the end. ``'r+'`` opens the file for both reading and -writing. The *mode* argument is optional; ``'r'`` will be assumed if it's -omitted. - -Normally, files are opened in :dfn:`text mode`, that means, you read and write -strings from and to the file, which are encoded in a specific encoding. If -encoding is not specified, the default is platform dependent (see -:func:`open`). ``'b'`` appended to the mode opens the file in -:dfn:`binary mode`: now the data is read and written in the form of bytes -objects. This mode should be used for all files that don't contain text. - -In text mode, the default when reading is to convert platform-specific line -endings (``\n`` on Unix, ``\r\n`` on Windows) to just ``\n``. When writing in -text mode, the default is to convert occurrences of ``\n`` back to -platform-specific line endings. This behind-the-scenes modification -to file data is fine for text files, but will corrupt binary data like that in -:file:`JPEG` or :file:`EXE` files. Be very careful to use binary mode when -reading and writing such files. - -It is good practice to use the :keyword:`with` keyword when dealing -with file objects. The advantage is that the file is properly closed -after its suite finishes, even if an exception is raised at some -point. Using :keyword:`with` is also much shorter than writing -equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - - >>> with open('workfile') as f: - ... read_data = f.read() - >>> f.closed - True - -If you're not using the :keyword:`with` keyword, then you should call -``f.close()`` to close the file and immediately free up any system -resources used by it. If you don't explicitly close a file, Python's -garbage collector will eventually destroy the object and close the -open file for you, but the file may stay open for a while. Another -risk is that different Python implementations will do this clean-up at -different times. - -After a file object is closed, either by a :keyword:`with` statement -or by calling ``f.close()``, attempts to use the file object will -automatically fail. :: - - >>> f.close() - >>> f.read() - Traceback (most recent call last): - File "", line 1, in - ValueError: I/O operation on closed file. - - -.. _tut-filemethods: - -Methods of File Objects ------------------------ - -The rest of the examples in this section will assume that a file object called -``f`` has already been created. - -To read a file's contents, call ``f.read(size)``, which reads some quantity of -data and returns it as a string (in text mode) or bytes object (in binary mode). -*size* is an optional numeric argument. When *size* is omitted or negative, the -entire contents of the file will be read and returned; it's your problem if the -file is twice as large as your machine's memory. Otherwise, at most *size* bytes -are read and returned. -If the end of the file has been reached, ``f.read()`` will return an empty -string (``''``). :: - - >>> f.read() - 'This is the entire file.\n' - >>> f.read() - '' - -``f.readline()`` reads a single line from the file; a newline character (``\n``) -is left at the end of the string, and is only omitted on the last line of the -file if the file doesn't end in a newline. This makes the return value -unambiguous; if ``f.readline()`` returns an empty string, the end of the file -has been reached, while a blank line is represented by ``'\n'``, a string -containing only a single newline. :: - - >>> f.readline() - 'This is the first line of the file.\n' - >>> f.readline() - 'Second line of the file\n' - >>> f.readline() - '' - -For reading lines from a file, you can loop over the file object. This is memory -efficient, fast, and leads to simple code:: - - >>> for line in f: - ... print(line, end='') - ... - This is the first line of the file. - Second line of the file - -If you want to read all the lines of a file in a list you can also use -``list(f)`` or ``f.readlines()``. - -``f.write(string)`` writes the contents of *string* to the file, returning -the number of characters written. :: - - >>> f.write('This is a test\n') - 15 - -Other types of objects need to be converted -- either to a string (in text mode) -or a bytes object (in binary mode) -- before writing them:: - - >>> value = ('the answer', 42) - >>> s = str(value) # convert the tuple to string - >>> f.write(s) - 18 - -``f.tell()`` returns an integer giving the file object's current position in the file -represented as number of bytes from the beginning of the file when in binary mode and -an opaque number when in text mode. - -To change the file object's position, use ``f.seek(offset, from_what)``. The position is computed -from adding *offset* to a reference point; the reference point is selected by -the *from_what* argument. A *from_what* value of 0 measures from the beginning -of the file, 1 uses the current file position, and 2 uses the end of the file as -the reference point. *from_what* can be omitted and defaults to 0, using the -beginning of the file as the reference point. :: - - >>> f = open('workfile', 'rb+') - >>> f.write(b'0123456789abcdef') - 16 - >>> f.seek(5) # Go to the 6th byte in the file - 5 - >>> f.read(1) - b'5' - >>> f.seek(-3, 2) # Go to the 3rd byte before the end - 13 - >>> f.read(1) - b'd' - -In text files (those opened without a ``b`` in the mode string), only seeks -relative to the beginning of the file are allowed (the exception being seeking -to the very file end with ``seek(0, 2)``) and the only valid *offset* values are -those returned from the ``f.tell()``, or zero. Any other *offset* value produces -undefined behaviour. - -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library -Reference for a complete guide to file objects. - - -.. _tut-json: - -Saving structured data with :mod:`json` ---------------------------------------- - -.. index:: module: json - -Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to -be passed to a function like :func:`int`, which takes a string like ``'123'`` -and returns its numeric value 123. When you want to save more complex data -types like nested lists and dictionaries, parsing and serializing by hand -becomes complicated. - -Rather than having users constantly writing and debugging code to save -complicated data types to files, Python allows you to use the popular data -interchange format called `JSON (JavaScript Object Notation) -`_. The standard module called :mod:`json` can take Python -data hierarchies, and convert them to string representations; this process is -called :dfn:`serializing`. Reconstructing the data from the string representation -is called :dfn:`deserializing`. Between serializing and deserializing, the -string representing the object may have been stored in a file or data, or -sent over a network connection to some distant machine. - -.. note:: - The JSON format is commonly used by modern applications to allow for data - exchange. Many programmers are already familiar with it, which makes - it a good choice for interoperability. - -If you have an object ``x``, you can view its JSON string representation with a -simple line of code:: - - >>> import json - >>> json.dumps([1, 'simple', 'list']) - '[1, "simple", "list"]' - -Another variant of the :func:`~json.dumps` function, called :func:`~json.dump`, -simply serializes the object to a :term:`text file`. So if ``f`` is a -:term:`text file` object opened for writing, we can do this:: - - json.dump(x, f) - -To decode the object again, if ``f`` is a :term:`text file` object which has -been opened for reading:: - - x = json.load(f) - -This simple serialization technique can handle lists and dictionaries, but -serializing arbitrary class instances in JSON requires a bit of extra effort. -The reference for the :mod:`json` module contains an explanation of this. - -.. seealso:: - - :mod:`pickle` - the pickle module - - Contrary to :ref:`JSON `, *pickle* is a protocol which allows - the serialization of arbitrarily complex Python objects. As such, it is - specific to Python and cannot be used to communicate with applications - written in other languages. It is also insecure by default: - deserializing pickle data coming from an untrusted source can execute - arbitrary code, if the data was crafted by a skilled attacker. diff --git a/third_party/python/Doc/tutorial/interactive.rst b/third_party/python/Doc/tutorial/interactive.rst deleted file mode 100644 index d73cfeb34..000000000 --- a/third_party/python/Doc/tutorial/interactive.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _tut-interacting: - -************************************************** -Interactive Input Editing and History Substitution -************************************************** - -Some versions of the Python interpreter support editing of the current input -line and history substitution, similar to facilities found in the Korn shell and -the GNU Bash shell. This is implemented using the `GNU Readline`_ library, -which supports various styles of editing. This library has its own -documentation which we won't duplicate here. - - -.. _tut-keybindings: - -Tab Completion and History Editing -================================== - -Completion of variable and module names is -:ref:`automatically enabled ` at interpreter startup so -that the :kbd:`Tab` key invokes the completion function; it looks at -Python statement names, the current local variables, and the available -module names. For dotted expressions such as ``string.a``, it will evaluate -the expression up to the final ``'.'`` and then suggest completions from -the attributes of the resulting object. Note that this may execute -application-defined code if an object with a :meth:`__getattr__` method -is part of the expression. The default configuration also saves your -history into a file named :file:`.python_history` in your user directory. -The history will be available again during the next interactive interpreter -session. - - -.. _tut-commentary: - -Alternatives to the Interactive Interpreter -=========================================== - -This facility is an enormous step forward compared to earlier versions of the -interpreter; however, some wishes are left: It would be nice if the proper -indentation were suggested on continuation lines (the parser knows if an indent -token is required next). The completion mechanism might use the interpreter's -symbol table. A command to check (or even suggest) matching parentheses, -quotes, etc., would also be useful. - -One alternative enhanced interactive interpreter that has been around for quite -some time is IPython_, which features tab completion, object exploration and -advanced history management. It can also be thoroughly customized and embedded -into other applications. Another similar enhanced interactive environment is -bpython_. - - -.. _GNU Readline: https://tiswww.case.edu/php/chet/readline/rltop.html -.. _IPython: https://ipython.org/ -.. _bpython: http://www.bpython-interpreter.org/ diff --git a/third_party/python/Doc/tutorial/interpreter.rst b/third_party/python/Doc/tutorial/interpreter.rst deleted file mode 100644 index d04f7cefa..000000000 --- a/third_party/python/Doc/tutorial/interpreter.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. _tut-using: - -**************************** -Using the Python Interpreter -**************************** - - -.. _tut-invoking: - -Invoking the Interpreter -======================== - -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.6` -on those machines where it is available; putting :file:`/usr/local/bin` in your -Unix shell's search path makes it possible to start it by typing the command: - -.. code-block:: text - - python3.6 - -to the shell. [#]_ Since the choice of the directory where the interpreter lives -is an installation option, other places are possible; check with your local -Python guru or system administrator. (E.g., :file:`/usr/local/python` is a -popular alternative location.) - -On Windows machines, the Python installation is usually placed in -:file:`C:\\Python36`, though you can change this when you're running the -installer. To add this directory to your path, you can type the following -command into the command prompt in a DOS box:: - - set path=%path%;C:\python36 - -Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on -Windows) at the primary prompt causes the interpreter to exit with a zero exit -status. If that doesn't work, you can exit the interpreter by typing the -following command: ``quit()``. - -The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support readline. Perhaps the -quickest check to see whether command line editing is supported is typing -:kbd:`Control-P` to the first Python prompt you get. If it beeps, you have command -line editing; see Appendix :ref:`tut-interacting` for an introduction to the -keys. If nothing appears to happen, or if ``^P`` is echoed, command line -editing isn't available; you'll only be able to use backspace to remove -characters from the current line. - -The interpreter operates somewhat like the Unix shell: when called with standard -input connected to a tty device, it reads and executes commands interactively; -when called with a file name argument or with a file as standard input, it reads -and executes a *script* from that file. - -A second way of starting the interpreter is ``python -c command [arg] ...``, -which executes the statement(s) in *command*, analogous to the shell's -:option:`-c` option. Since Python statements often contain spaces or other -characters that are special to the shell, it is usually advised to quote -*command* in its entirety with single quotes. - -Some Python modules are also useful as scripts. These can be invoked using -``python -m module [arg] ...``, which executes the source file for *module* as -if you had spelled out its full name on the command line. - -When a script file is used, it is sometimes useful to be able to run the script -and enter interactive mode afterwards. This can be done by passing :option:`-i` -before the script. - -All command line options are described in :ref:`using-on-general`. - - -.. _tut-argpassing: - -Argument Passing ----------------- - -When known to the interpreter, the script name and additional arguments -thereafter are turned into a list of strings and assigned to the ``argv`` -variable in the ``sys`` module. You can access this list by executing ``import -sys``. The length of the list is at least one; when no script and no arguments -are given, ``sys.argv[0]`` is an empty string. When the script name is given as -``'-'`` (meaning standard input), ``sys.argv[0]`` is set to ``'-'``. When -:option:`-c` *command* is used, ``sys.argv[0]`` is set to ``'-c'``. When -:option:`-m` *module* is used, ``sys.argv[0]`` is set to the full name of the -located module. Options found after :option:`-c` *command* or :option:`-m` -*module* are not consumed by the Python interpreter's option processing but -left in ``sys.argv`` for the command or module to handle. - - -.. _tut-interactive: - -Interactive Mode ----------------- - -When commands are read from a tty, the interpreter is said to be in *interactive -mode*. In this mode it prompts for the next command with the *primary prompt*, -usually three greater-than signs (``>>>``); for continuation lines it prompts -with the *secondary prompt*, by default three dots (``...``). The interpreter -prints a welcome message stating its version number and a copyright notice -before printing the first prompt: - -.. code-block:: shell-session - - $ python3.6 - Python 3.6 (default, Sep 16 2015, 09:25:04) - [GCC 4.8.2] on linux - Type "help", "copyright", "credits" or "license" for more information. - >>> - -.. XXX update for new releases - -Continuation lines are needed when entering a multi-line construct. As an -example, take a look at this :keyword:`if` statement:: - - >>> the_world_is_flat = True - >>> if the_world_is_flat: - ... print("Be careful not to fall off!") - ... - Be careful not to fall off! - - -For more on interactive mode, see :ref:`tut-interac`. - - -.. _tut-interp: - -The Interpreter and Its Environment -=================================== - - -.. _tut-source-encoding: - -Source Code Encoding --------------------- - -By default, Python source files are treated as encoded in UTF-8. In that -encoding, characters of most languages in the world can be used simultaneously -in string literals, identifiers and comments --- although the standard library -only uses ASCII characters for identifiers, a convention that any portable code -should follow. To display all these characters properly, your editor must -recognize that the file is UTF-8, and it must use a font that supports all the -characters in the file. - -To declare an encoding other than the default one, a special comment line -should be added as the *first* line of the file. The syntax is as follows:: - - # -*- coding: encoding -*- - -where *encoding* is one of the valid :mod:`codecs` supported by Python. - -For example, to declare that Windows-1252 encoding is to be used, the first -line of your source code file should be:: - - # -*- coding: cp1252 -*- - -One exception to the *first line* rule is when the source code starts with a -:ref:`UNIX "shebang" line `. In this case, the encoding -declaration should be added as the second line of the file. For example:: - - #!/usr/bin/env python3 - # -*- coding: cp1252 -*- - -.. rubric:: Footnotes - -.. [#] On Unix, the Python 3.x interpreter is by default not installed with the - executable named ``python``, so that it does not conflict with a - simultaneously installed Python 2.x executable. diff --git a/third_party/python/Doc/tutorial/introduction.rst b/third_party/python/Doc/tutorial/introduction.rst deleted file mode 100644 index 5d6812dd3..000000000 --- a/third_party/python/Doc/tutorial/introduction.rst +++ /dev/null @@ -1,537 +0,0 @@ -.. _tut-informal: - -********************************** -An Informal Introduction to Python -********************************** - -In the following examples, input and output are distinguished by the presence or -absence of prompts (:term:`>>>` and :term:`...`): to repeat the example, you must type -everything after the prompt, when the prompt appears; lines that do not begin -with a prompt are output from the interpreter. Note that a secondary prompt on a -line by itself in an example means you must type a blank line; this is used to -end a multi-line command. - -.. index:: single: # (hash); comment - -Many of the examples in this manual, even those entered at the interactive -prompt, include comments. Comments in Python start with the hash character, -``#``, and extend to the end of the physical line. A comment may appear at the -start of a line or following whitespace or code, but not within a string -literal. A hash character within a string literal is just a hash character. -Since comments are to clarify code and are not interpreted by Python, they may -be omitted when typing in examples. - -Some examples:: - - # this is the first comment - spam = 1 # and this is the second comment - # ... and now a third! - text = "# This is not a comment because it's inside quotes." - - -.. _tut-calculator: - -Using Python as a Calculator -============================ - -Let's try some simple Python commands. Start the interpreter and wait for the -primary prompt, ``>>>``. (It shouldn't take long.) - - -.. _tut-numbers: - -Numbers -------- - -The interpreter acts as a simple calculator: you can type an expression at it -and it will write the value. Expression syntax is straightforward: the -operators ``+``, ``-``, ``*`` and ``/`` work just like in most other languages -(for example, Pascal or C); parentheses (``()``) can be used for grouping. -For example:: - - >>> 2 + 2 - 4 - >>> 50 - 5*6 - 20 - >>> (50 - 5*6) / 4 - 5.0 - >>> 8 / 5 # division always returns a floating point number - 1.6 - -The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, -the ones with a fractional part (e.g. ``5.0``, ``1.6``) have type -:class:`float`. We will see more about numeric types later in the tutorial. - -Division (``/``) always returns a float. To do :term:`floor division` and -get an integer result (discarding any fractional result) you can use the ``//`` -operator; to calculate the remainder you can use ``%``:: - - >>> 17 / 3 # classic division returns a float - 5.666666666666667 - >>> - >>> 17 // 3 # floor division discards the fractional part - 5 - >>> 17 % 3 # the % operator returns the remainder of the division - 2 - >>> 5 * 3 + 2 # result * divisor + remainder - 17 - -With Python, it is possible to use the ``**`` operator to calculate powers [#]_:: - - >>> 5 ** 2 # 5 squared - 25 - >>> 2 ** 7 # 2 to the power of 7 - 128 - -The equal sign (``=``) is used to assign a value to a variable. Afterwards, no -result is displayed before the next interactive prompt:: - - >>> width = 20 - >>> height = 5 * 9 - >>> width * height - 900 - -If a variable is not "defined" (assigned a value), trying to use it will -give you an error:: - - >>> n # try to access an undefined variable - Traceback (most recent call last): - File "", line 1, in - NameError: name 'n' is not defined - -There is full support for floating point; operators with mixed type operands -convert the integer operand to floating point:: - - >>> 4 * 3.75 - 1 - 14.0 - -In interactive mode, the last printed expression is assigned to the variable -``_``. This means that when you are using Python as a desk calculator, it is -somewhat easier to continue calculations, for example:: - - >>> tax = 12.5 / 100 - >>> price = 100.50 - >>> price * tax - 12.5625 - >>> price + _ - 113.0625 - >>> round(_, 2) - 113.06 - -This variable should be treated as read-only by the user. Don't explicitly -assign a value to it --- you would create an independent local variable with the -same name masking the built-in variable with its magic behavior. - -In addition to :class:`int` and :class:`float`, Python supports other types of -numbers, such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Python also has built-in support for :ref:`complex numbers `, -and uses the ``j`` or ``J`` suffix to indicate the imaginary part -(e.g. ``3+5j``). - - -.. _tut-strings: - -Strings -------- - -Besides numbers, Python can also manipulate strings, which can be expressed -in several ways. They can be enclosed in single quotes (``'...'``) or -double quotes (``"..."``) with the same result [#]_. ``\`` can be used -to escape quotes:: - - >>> 'spam eggs' # single quotes - 'spam eggs' - >>> 'doesn\'t' # use \' to escape the single quote... - "doesn't" - >>> "doesn't" # ...or use double quotes instead - "doesn't" - >>> '"Yes," they said.' - '"Yes," they said.' - >>> "\"Yes,\" they said." - '"Yes," they said.' - >>> '"Isn\'t," they said.' - '"Isn\'t," they said.' - -In the interactive interpreter, the output string is enclosed in quotes and -special characters are escaped with backslashes. While this might sometimes -look different from the input (the enclosing quotes could change), the two -strings are equivalent. The string is enclosed in double quotes if -the string contains a single quote and no double quotes, otherwise it is -enclosed in single quotes. The :func:`print` function produces a more -readable output, by omitting the enclosing quotes and by printing escaped -and special characters:: - - >>> '"Isn\'t," they said.' - '"Isn\'t," they said.' - >>> print('"Isn\'t," they said.') - "Isn't," they said. - >>> s = 'First line.\nSecond line.' # \n means newline - >>> s # without print(), \n is included in the output - 'First line.\nSecond line.' - >>> print(s) # with print(), \n produces a new line - First line. - Second line. - -If you don't want characters prefaced by ``\`` to be interpreted as -special characters, you can use *raw strings* by adding an ``r`` before -the first quote:: - - >>> print('C:\some\name') # here \n means newline! - C:\some - ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name - -String literals can span multiple lines. One way is using triple-quotes: -``"""..."""`` or ``'''...'''``. End of lines are automatically -included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: - - print("""\ - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to - """) - -produces the following output (note that the initial newline is not included): - -.. code-block:: text - - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to - -Strings can be concatenated (glued together) with the ``+`` operator, and -repeated with ``*``:: - - >>> # 3 times 'un', followed by 'ium' - >>> 3 * 'un' + 'ium' - 'unununium' - -Two or more *string literals* (i.e. the ones enclosed between quotes) next -to each other are automatically concatenated. :: - - >>> 'Py' 'thon' - 'Python' - -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - -This only works with two literals though, not with variables or expressions:: - - >>> prefix = 'Py' - >>> prefix 'thon' # can't concatenate a variable and a string literal - ... - SyntaxError: invalid syntax - >>> ('un' * 3) 'ium' - ... - SyntaxError: invalid syntax - -If you want to concatenate variables or a variable and a literal, use ``+``:: - - >>> prefix + 'thon' - 'Python' - -Strings can be *indexed* (subscripted), with the first character having index 0. -There is no separate character type; a character is simply a string of size -one:: - - >>> word = 'Python' - >>> word[0] # character in position 0 - 'P' - >>> word[5] # character in position 5 - 'n' - -Indices may also be negative numbers, to start counting from the right:: - - >>> word[-1] # last character - 'n' - >>> word[-2] # second-last character - 'o' - >>> word[-6] - 'P' - -Note that since -0 is the same as 0, negative indices start from -1. - -In addition to indexing, *slicing* is also supported. While indexing is used -to obtain individual characters, *slicing* allows you to obtain substring:: - - >>> word[0:2] # characters from position 0 (included) to 2 (excluded) - 'Py' - >>> word[2:5] # characters from position 2 (included) to 5 (excluded) - 'tho' - -Note how the start is always included, and the end always excluded. This -makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: - - >>> word[:2] + word[2:] - 'Python' - >>> word[:4] + word[4:] - 'Python' - -Slice indices have useful defaults; an omitted first index defaults to zero, an -omitted second index defaults to the size of the string being sliced. :: - - >>> word[:2] # character from the beginning to position 2 (excluded) - 'Py' - >>> word[4:] # characters from position 4 (included) to the end - 'on' - >>> word[-2:] # characters from the second-last (included) to the end - 'on' - -One way to remember how slices work is to think of the indices as pointing -*between* characters, with the left edge of the first character numbered 0. -Then the right edge of the last character of a string of *n* characters has -index *n*, for example:: - - +---+---+---+---+---+---+ - | P | y | t | h | o | n | - +---+---+---+---+---+---+ - 0 1 2 3 4 5 6 - -6 -5 -4 -3 -2 -1 - -The first row of numbers gives the position of the indices 0...6 in the string; -the second row gives the corresponding negative indices. The slice from *i* to -*j* consists of all characters between the edges labeled *i* and *j*, -respectively. - -For non-negative indices, the length of a slice is the difference of the -indices, if both are within bounds. For example, the length of ``word[1:3]`` is -2. - -Attempting to use an index that is too large will result in an error:: - - >>> word[42] # the word only has 6 characters - Traceback (most recent call last): - File "", line 1, in - IndexError: string index out of range - -However, out of range slice indexes are handled gracefully when used for -slicing:: - - >>> word[4:42] - 'on' - >>> word[42:] - '' - -Python strings cannot be changed --- they are :term:`immutable`. -Therefore, assigning to an indexed position in the string results in an error:: - - >>> word[0] = 'J' - ... - TypeError: 'str' object does not support item assignment - >>> word[2:] = 'py' - ... - TypeError: 'str' object does not support item assignment - -If you need a different string, you should create a new one:: - - >>> 'J' + word[1:] - 'Jython' - >>> word[:2] + 'py' - 'Pypy' - -The built-in function :func:`len` returns the length of a string:: - - >>> s = 'supercalifragilisticexpialidocious' - >>> len(s) - 34 - - -.. seealso:: - - :ref:`textseq` - Strings are examples of *sequence types*, and support the common - operations supported by such types. - - :ref:`string-methods` - Strings support a large number of methods for - basic transformations and searching. - - :ref:`f-strings` - String literals that have embedded expressions. - - :ref:`formatstrings` - Information about string formatting with :meth:`str.format`. - - :ref:`old-string-formatting` - The old formatting operations invoked when strings are - the left operand of the ``%`` operator are described in more detail here. - - -.. _tut-lists: - -Lists ------ - -Python knows a number of *compound* data types, used to group together other -values. The most versatile is the *list*, which can be written as a list of -comma-separated values (items) between square brackets. Lists might contain -items of different types, but usually the items all have the same type. :: - - >>> squares = [1, 4, 9, 16, 25] - >>> squares - [1, 4, 9, 16, 25] - -Like strings (and all other built-in :term:`sequence` type), lists can be -indexed and sliced:: - - >>> squares[0] # indexing returns the item - 1 - >>> squares[-1] - 25 - >>> squares[-3:] # slicing returns a new list - [9, 16, 25] - -All slice operations return a new list containing the requested elements. This -means that the following slice returns a new (shallow) copy of the list:: - - >>> squares[:] - [1, 4, 9, 16, 25] - -Lists also support operations like concatenation:: - - >>> squares + [36, 49, 64, 81, 100] - [1, 4, 9, 16, 25, 36, 49, 64, 81, 100] - -Unlike strings, which are :term:`immutable`, lists are a :term:`mutable` -type, i.e. it is possible to change their content:: - - >>> cubes = [1, 8, 27, 65, 125] # something's wrong here - >>> 4 ** 3 # the cube of 4 is 64, not 65! - 64 - >>> cubes[3] = 64 # replace the wrong value - >>> cubes - [1, 8, 27, 64, 125] - -You can also add new items at the end of the list, by using -the :meth:`~list.append` *method* (we will see more about methods later):: - - >>> cubes.append(216) # add the cube of 6 - >>> cubes.append(7 ** 3) # and the cube of 7 - >>> cubes - [1, 8, 27, 64, 125, 216, 343] - -Assignment to slices is also possible, and this can even change the size of the -list or clear it entirely:: - - >>> letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> letters - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> # replace some values - >>> letters[2:5] = ['C', 'D', 'E'] - >>> letters - ['a', 'b', 'C', 'D', 'E', 'f', 'g'] - >>> # now remove them - >>> letters[2:5] = [] - >>> letters - ['a', 'b', 'f', 'g'] - >>> # clear the list by replacing all the elements with an empty list - >>> letters[:] = [] - >>> letters - [] - -The built-in function :func:`len` also applies to lists:: - - >>> letters = ['a', 'b', 'c', 'd'] - >>> len(letters) - 4 - -It is possible to nest lists (create lists containing other lists), for -example:: - - >>> a = ['a', 'b', 'c'] - >>> n = [1, 2, 3] - >>> x = [a, n] - >>> x - [['a', 'b', 'c'], [1, 2, 3]] - >>> x[0] - ['a', 'b', 'c'] - >>> x[0][1] - 'b' - -.. _tut-firststeps: - -First Steps Towards Programming -=============================== - -Of course, we can use Python for more complicated tasks than adding two and two -together. For instance, we can write an initial sub-sequence of the *Fibonacci* -series as follows:: - - >>> # Fibonacci series: - ... # the sum of two elements defines the next - ... a, b = 0, 1 - >>> while b < 10: - ... print(b) - ... a, b = b, a+b - ... - 1 - 1 - 2 - 3 - 5 - 8 - -This example introduces several new features. - -* The first line contains a *multiple assignment*: the variables ``a`` and ``b`` - simultaneously get the new values 0 and 1. On the last line this is used again, - demonstrating that the expressions on the right-hand side are all evaluated - first before any of the assignments take place. The right-hand side expressions - are evaluated from the left to the right. - -* The :keyword:`while` loop executes as long as the condition (here: ``b < 10``) - remains true. In Python, like in C, any non-zero integer value is true; zero is - false. The condition may also be a string or list value, in fact any sequence; - anything with a non-zero length is true, empty sequences are false. The test - used in the example is a simple comparison. The standard comparison operators - are written the same as in C: ``<`` (less than), ``>`` (greater than), ``==`` - (equal to), ``<=`` (less than or equal to), ``>=`` (greater than or equal to) - and ``!=`` (not equal to). - -* The *body* of the loop is *indented*: indentation is Python's way of grouping - statements. At the interactive prompt, you have to type a tab or space(s) for - each indented line. In practice you will prepare more complicated input - for Python with a text editor; all decent text editors have an auto-indent - facility. When a compound statement is entered interactively, it must be - followed by a blank line to indicate completion (since the parser cannot - guess when you have typed the last line). Note that each line within a basic - block must be indented by the same amount. - -* The :func:`print` function writes the value of the argument(s) it is given. - It differs from just writing the expression you want to write (as we did - earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, - and a space is inserted between items, so you can format things nicely, like - this:: - - >>> i = 256*256 - >>> print('The value of i is', i) - The value of i is 65536 - - The keyword argument *end* can be used to avoid the newline after the output, - or end the output with a different string:: - - >>> a, b = 0, 1 - >>> while b < 1000: - ... print(b, end=',') - ... a, b = b, a+b - ... - 1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987, - - -.. rubric:: Footnotes - -.. [#] Since ``**`` has higher precedence than ``-``, ``-3**2`` will be - interpreted as ``-(3**2)`` and thus result in ``-9``. To avoid this - and get ``9``, you can use ``(-3)**2``. - -.. [#] Unlike other languages, special characters such as ``\n`` have the - same meaning with both single (``'...'``) and double (``"..."``) quotes. - The only difference between the two is that within single quotes you don't - need to escape ``"`` (but you have to escape ``\'``) and vice versa. diff --git a/third_party/python/Doc/tutorial/modules.rst b/third_party/python/Doc/tutorial/modules.rst deleted file mode 100644 index 04352cf6e..000000000 --- a/third_party/python/Doc/tutorial/modules.rst +++ /dev/null @@ -1,572 +0,0 @@ -.. _tut-modules: - -******* -Modules -******* - -If you quit from the Python interpreter and enter it again, the definitions you -have made (functions and variables) are lost. Therefore, if you want to write a -somewhat longer program, you are better off using a text editor to prepare the -input for the interpreter and running it with that file as input instead. This -is known as creating a *script*. As your program gets longer, you may want to -split it into several files for easier maintenance. You may also want to use a -handy function that you've written in several programs without copying its -definition into each program. - -To support this, Python has a way to put definitions in a file and use them in a -script or in an interactive instance of the interpreter. Such a file is called a -*module*; definitions from a module can be *imported* into other modules or into -the *main* module (the collection of variables that you have access to in a -script executed at the top level and in calculator mode). - -A module is a file containing Python definitions and statements. The file name -is the module name with the suffix :file:`.py` appended. Within a module, the -module's name (as a string) is available as the value of the global variable -``__name__``. For instance, use your favorite text editor to create a file -called :file:`fibo.py` in the current directory with the following contents:: - - # Fibonacci numbers module - - def fib(n): # write Fibonacci series up to n - a, b = 0, 1 - while b < n: - print(b, end=' ') - a, b = b, a+b - print() - - def fib2(n): # return Fibonacci series up to n - result = [] - a, b = 0, 1 - while b < n: - result.append(b) - a, b = b, a+b - return result - -Now enter the Python interpreter and import this module with the following -command:: - - >>> import fibo - -This does not enter the names of the functions defined in ``fibo`` directly in -the current symbol table; it only enters the module name ``fibo`` there. Using -the module name you can access the functions:: - - >>> fibo.fib(1000) - 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 - >>> fibo.fib2(100) - [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89] - >>> fibo.__name__ - 'fibo' - -If you intend to use a function often you can assign it to a local name:: - - >>> fib = fibo.fib - >>> fib(500) - 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - - -.. _tut-moremodules: - -More on Modules -=============== - -A module can contain executable statements as well as function definitions. -These statements are intended to initialize the module. They are executed only -the *first* time the module name is encountered in an import statement. [#]_ -(They are also run if the file is executed as a script.) - -Each module has its own private symbol table, which is used as the global symbol -table by all functions defined in the module. Thus, the author of a module can -use global variables in the module without worrying about accidental clashes -with a user's global variables. On the other hand, if you know what you are -doing you can touch a module's global variables with the same notation used to -refer to its functions, ``modname.itemname``. - -Modules can import other modules. It is customary but not required to place all -:keyword:`import` statements at the beginning of a module (or script, for that -matter). The imported module names are placed in the importing module's global -symbol table. - -There is a variant of the :keyword:`import` statement that imports names from a -module directly into the importing module's symbol table. For example:: - - >>> from fibo import fib, fib2 - >>> fib(500) - 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This does not introduce the module name from which the imports are taken in the -local symbol table (so in the example, ``fibo`` is not defined). - -There is even a variant to import all names that a module defines:: - - >>> from fibo import * - >>> fib(500) - 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This imports all names except those beginning with an underscore (``_``). -In most cases Python programmers do not use this facility since it introduces -an unknown set of names into the interpreter, possibly hiding some things -you have already defined. - -Note that in general the practice of importing ``*`` from a module or package is -frowned upon, since it often causes poorly readable code. However, it is okay to -use it to save typing in interactive sessions. - -If the module name is followed by :keyword:`as`, then the name -following :keyword:`as` is bound directly to the imported module. - -:: - - >>> import fibo as fib - >>> fib.fib(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This is effectively importing the module in the same way that ``import fibo`` -will do, with the only difference of it being available as ``fib``. - -It can also be used when utilising :keyword:`from` with similar effects:: - - >>> from fibo import fib as fibonacci - >>> fibonacci(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - - -.. note:: - - For efficiency reasons, each module is only imported once per interpreter - session. Therefore, if you change your modules, you must restart the - interpreter -- or, if it's just one module you want to test interactively, - use :func:`importlib.reload`, e.g. ``import importlib; - importlib.reload(modulename)``. - - -.. _tut-modulesasscripts: - -Executing modules as scripts ----------------------------- - -When you run a Python module with :: - - python fibo.py - -the code in the module will be executed, just as if you imported it, but with -the ``__name__`` set to ``"__main__"``. That means that by adding this code at -the end of your module:: - - if __name__ == "__main__": - import sys - fib(int(sys.argv[1])) - -you can make the file usable as a script as well as an importable module, -because the code that parses the command line only runs if the module is -executed as the "main" file: - -.. code-block:: shell-session - - $ python fibo.py 50 - 1 1 2 3 5 8 13 21 34 - -If the module is imported, the code is not run:: - - >>> import fibo - >>> - -This is often used either to provide a convenient user interface to a module, or -for testing purposes (running the module as a script executes a test suite). - - -.. _tut-searchpath: - -The Module Search Path ----------------------- - -.. index:: triple: module; search; path - -When a module named :mod:`spam` is imported, the interpreter first searches for -a built-in module with that name. If not found, it then searches for a file -named :file:`spam.py` in a list of directories given by the variable -:data:`sys.path`. :data:`sys.path` is initialized from these locations: - -* The directory containing the input script (or the current directory when no - file is specified). -* :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the - shell variable :envvar:`PATH`). -* The installation-dependent default. - -.. note:: - On file systems which support symlinks, the directory containing the input - script is calculated after the symlink is followed. In other words the - directory containing the symlink is **not** added to the module search path. - -After initialization, Python programs can modify :data:`sys.path`. The -directory containing the script being run is placed at the beginning of the -search path, ahead of the standard library path. This means that scripts in that -directory will be loaded instead of modules of the same name in the library -directory. This is an error unless the replacement is intended. See section -:ref:`tut-standardmodules` for more information. - -.. % - Do we need stuff on zip files etc. ? DUBOIS - -"Compiled" Python files ------------------------ - -To speed up loading modules, Python caches the compiled version of each module -in the ``__pycache__`` directory under the name :file:`module.{version}.pyc`, -where the version encodes the format of the compiled file; it generally contains -the Python version number. For example, in CPython release 3.3 the compiled -version of spam.py would be cached as ``__pycache__/spam.cpython-33.pyc``. This -naming convention allows compiled modules from different releases and different -versions of Python to coexist. - -Python checks the modification date of the source against the compiled version -to see if it's out of date and needs to be recompiled. This is a completely -automatic process. Also, the compiled modules are platform-independent, so the -same library can be shared among systems with different architectures. - -Python does not check the cache in two circumstances. First, it always -recompiles and does not store the result for the module that's loaded directly -from the command line. Second, it does not check the cache if there is no -source module. To support a non-source (compiled only) distribution, the -compiled module must be in the source directory, and there must not be a source -module. - -Some tips for experts: - -* You can use the :option:`-O` or :option:`-OO` switches on the Python command - to reduce the size of a compiled module. The ``-O`` switch removes assert - statements, the ``-OO`` switch removes both assert statements and __doc__ - strings. Since some programs may rely on having these available, you should - only use this option if you know what you're doing. "Optimized" modules have - an ``opt-`` tag and are usually smaller. Future releases may - change the effects of optimization. - -* A program doesn't run any faster when it is read from a ``.pyc`` - file than when it is read from a ``.py`` file; the only thing that's faster - about ``.pyc`` files is the speed with which they are loaded. - -* The module :mod:`compileall` can create .pyc files for all modules in a - directory. - -* There is more detail on this process, including a flow chart of the - decisions, in :pep:`3147`. - - -.. _tut-standardmodules: - -Standard Modules -================ - -.. index:: module: sys - -Python comes with a library of standard modules, described in a separate -document, the Python Library Reference ("Library Reference" hereafter). Some -modules are built into the interpreter; these provide access to operations that -are not part of the core of the language but are nevertheless built in, either -for efficiency or to provide access to operating system primitives such as -system calls. The set of such modules is a configuration option which also -depends on the underlying platform. For example, the :mod:`winreg` module is only -provided on Windows systems. One particular module deserves some attention: -:mod:`sys`, which is built into every Python interpreter. The variables -``sys.ps1`` and ``sys.ps2`` define the strings used as primary and secondary -prompts:: - - >>> import sys - >>> sys.ps1 - '>>> ' - >>> sys.ps2 - '... ' - >>> sys.ps1 = 'C> ' - C> print('Yuck!') - Yuck! - C> - - -These two variables are only defined if the interpreter is in interactive mode. - -The variable ``sys.path`` is a list of strings that determines the interpreter's -search path for modules. It is initialized to a default path taken from the -environment variable :envvar:`PYTHONPATH`, or from a built-in default if -:envvar:`PYTHONPATH` is not set. You can modify it using standard list -operations:: - - >>> import sys - >>> sys.path.append('/ufs/guido/lib/python') - - -.. _tut-dir: - -The :func:`dir` Function -======================== - -The built-in function :func:`dir` is used to find out which names a module -defines. It returns a sorted list of strings:: - - >>> import fibo, sys - >>> dir(fibo) - ['__name__', 'fib', 'fib2'] - >>> dir(sys) # doctest: +NORMALIZE_WHITESPACE - ['__displayhook__', '__doc__', '__excepthook__', '__loader__', '__name__', - '__package__', '__stderr__', '__stdin__', '__stdout__', - '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe', - '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv', - 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder', - 'call_tracing', 'callstats', 'copyright', 'displayhook', - 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix', - 'executable', 'exit', 'flags', 'float_info', 'float_repr_style', - 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags', - 'getfilesystemencoding', 'getobjects', 'getprofile', 'getrecursionlimit', - 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettotalrefcount', - 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info', - 'intern', 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', - 'path_hooks', 'path_importer_cache', 'platform', 'prefix', 'ps1', - 'setcheckinterval', 'setdlopenflags', 'setprofile', 'setrecursionlimit', - 'setswitchinterval', 'settrace', 'stderr', 'stdin', 'stdout', - 'thread_info', 'version', 'version_info', 'warnoptions'] - -Without arguments, :func:`dir` lists the names you have defined currently:: - - >>> a = [1, 2, 3, 4, 5] - >>> import fibo - >>> fib = fibo.fib - >>> dir() - ['__builtins__', '__name__', 'a', 'fib', 'fibo', 'sys'] - -Note that it lists all types of names: variables, modules, functions, etc. - -.. index:: module: builtins - -:func:`dir` does not list the names of built-in functions and variables. If you -want a list of those, they are defined in the standard module -:mod:`builtins`:: - - >>> import builtins - >>> dir(builtins) # doctest: +NORMALIZE_WHITESPACE - ['ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', - 'BlockingIOError', 'BrokenPipeError', 'BufferError', 'BytesWarning', - 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', - 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning', - 'EOFError', 'Ellipsis', 'EnvironmentError', 'Exception', 'False', - 'FileExistsError', 'FileNotFoundError', 'FloatingPointError', - 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', - 'ImportWarning', 'IndentationError', 'IndexError', 'InterruptedError', - 'IsADirectoryError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'None', 'NotADirectoryError', 'NotImplemented', - 'NotImplementedError', 'OSError', 'OverflowError', - 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError', - 'ReferenceError', 'ResourceWarning', 'RuntimeError', 'RuntimeWarning', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', - 'SystemExit', 'TabError', 'TimeoutError', 'True', 'TypeError', - 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', - 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', - 'ValueError', 'Warning', 'ZeroDivisionError', '_', '__build_class__', - '__debug__', '__doc__', '__import__', '__name__', '__package__', 'abs', - 'all', 'any', 'ascii', 'bin', 'bool', 'bytearray', 'bytes', 'callable', - 'chr', 'classmethod', 'compile', 'complex', 'copyright', 'credits', - 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'exec', 'exit', - 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', - 'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', - 'iter', 'len', 'license', 'list', 'locals', 'map', 'max', 'memoryview', - 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property', - 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', - 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', - 'zip'] - -.. _tut-packages: - -Packages -======== - -Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule -named ``B`` in a package named ``A``. Just like the use of modules saves the -authors of different modules from having to worry about each other's global -variable names, the use of dotted module names saves the authors of multi-module -packages like NumPy or Pillow from having to worry about -each other's module names. - -Suppose you want to design a collection of modules (a "package") for the uniform -handling of sound files and sound data. There are many different sound file -formats (usually recognized by their extension, for example: :file:`.wav`, -:file:`.aiff`, :file:`.au`), so you may need to create and maintain a growing -collection of modules for the conversion between the various file formats. -There are also many different operations you might want to perform on sound data -(such as mixing, adding echo, applying an equalizer function, creating an -artificial stereo effect), so in addition you will be writing a never-ending -stream of modules to perform these operations. Here's a possible structure for -your package (expressed in terms of a hierarchical filesystem): - -.. code-block:: text - - sound/ Top-level package - __init__.py Initialize the sound package - formats/ Subpackage for file format conversions - __init__.py - wavread.py - wavwrite.py - aiffread.py - aiffwrite.py - auread.py - auwrite.py - ... - effects/ Subpackage for sound effects - __init__.py - echo.py - surround.py - reverse.py - ... - filters/ Subpackage for filters - __init__.py - equalizer.py - vocoder.py - karaoke.py - ... - -When importing the package, Python searches through the directories on -``sys.path`` looking for the package subdirectory. - -The :file:`__init__.py` files are required to make Python treat the directories -as containing packages; this is done to prevent directories with a common name, -such as ``string``, from unintentionally hiding valid modules that occur later -on the module search path. In the simplest case, :file:`__init__.py` can just be -an empty file, but it can also execute initialization code for the package or -set the ``__all__`` variable, described later. - -Users of the package can import individual modules from the package, for -example:: - - import sound.effects.echo - -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with -its full name. :: - - sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) - -An alternative way of importing the submodule is:: - - from sound.effects import echo - -This also loads the submodule :mod:`echo`, and makes it available without its -package prefix, so it can be used as follows:: - - echo.echofilter(input, output, delay=0.7, atten=4) - -Yet another variation is to import the desired function or variable directly:: - - from sound.effects.echo import echofilter - -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: - - echofilter(input, output, delay=0.7, atten=4) - -Note that when using ``from package import item``, the item can be either a -submodule (or subpackage) of the package, or some other name defined in the -package, like a function, class or variable. The ``import`` statement first -tests whether the item is defined in the package; if not, it assumes it is a -module and attempts to load it. If it fails to find it, an :exc:`ImportError` -exception is raised. - -Contrarily, when using syntax like ``import item.subitem.subsubitem``, each item -except for the last must be a package; the last item can be a module or a -package but can't be a class or function or variable defined in the previous -item. - - -.. _tut-pkg-import-star: - -Importing \* From a Package ---------------------------- - -.. index:: single: __all__ - -Now what happens when the user writes ``from sound.effects import *``? Ideally, -one would hope that this somehow goes out to the filesystem, finds which -submodules are present in the package, and imports them all. This could take a -long time and importing sub-modules might have unwanted side-effects that should -only happen when the sub-module is explicitly imported. - -The only solution is for the package author to provide an explicit index of the -package. The :keyword:`import` statement uses the following convention: if a package's -:file:`__init__.py` code defines a list named ``__all__``, it is taken to be the -list of module names that should be imported when ``from package import *`` is -encountered. It is up to the package author to keep this list up-to-date when a -new version of the package is released. Package authors may also decide not to -support it, if they don't see a use for importing \* from their package. For -example, the file :file:`sound/effects/__init__.py` could contain the following -code:: - - __all__ = ["echo", "surround", "reverse"] - -This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound` package. - -If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has -been imported (possibly running any initialization code in :file:`__init__.py`) -and then imports whatever names are defined in the package. This includes any -names defined (and submodules explicitly loaded) by :file:`__init__.py`. It -also includes any submodules of the package that were explicitly loaded by -previous :keyword:`import` statements. Consider this code:: - - import sound.effects.echo - import sound.effects.surround - from sound.effects import * - -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package -when the ``from...import`` statement is executed. (This also works when -``__all__`` is defined.) - -Although certain modules are designed to export only names that follow certain -patterns when you use ``import *``, it is still considered bad practice in -production code. - -Remember, there is nothing wrong with using ``from Package import -specific_submodule``! In fact, this is the recommended notation unless the -importing module needs to use submodules with the same name from different -packages. - - -Intra-package References ------------------------- - -When packages are structured into subpackages (as with the :mod:`sound` package -in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from -sound.effects import echo``. - -You can also write relative imports, with the ``from module import name`` form -of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` -module for example, you might use:: - - from . import echo - from .. import formats - from ..filters import equalizer - -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use -as the main module of a Python application must always use absolute imports. - - -Packages in Multiple Directories --------------------------------- - -Packages support one more special attribute, :attr:`__path__`. This is -initialized to be a list containing the name of the directory holding the -package's :file:`__init__.py` before the code in that file is executed. This -variable can be modified; doing so affects future searches for modules and -subpackages contained in the package. - -While this feature is not often needed, it can be used to extend the set of -modules found in a package. - - -.. rubric:: Footnotes - -.. [#] In fact function definitions are also 'statements' that are 'executed'; the - execution of a module-level function definition enters the function name in - the module's global symbol table. diff --git a/third_party/python/Doc/tutorial/stdlib.rst b/third_party/python/Doc/tutorial/stdlib.rst deleted file mode 100644 index 110e6e546..000000000 --- a/third_party/python/Doc/tutorial/stdlib.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _tut-brieftour: - -********************************** -Brief Tour of the Standard Library -********************************** - - -.. _tut-os-interface: - -Operating System Interface -========================== - -The :mod:`os` module provides dozens of functions for interacting with the -operating system:: - - >>> import os - >>> os.getcwd() # Return the current working directory - 'C:\\Python36' - >>> os.chdir('/server/accesslogs') # Change current working directory - >>> os.system('mkdir today') # Run the command mkdir in the system shell - 0 - -Be sure to use the ``import os`` style instead of ``from os import *``. This -will keep :func:`os.open` from shadowing the built-in :func:`open` function which -operates much differently. - -.. index:: builtin: help - -The built-in :func:`dir` and :func:`help` functions are useful as interactive -aids for working with large modules like :mod:`os`:: - - >>> import os - >>> dir(os) - - >>> help(os) - - -For daily file and directory management tasks, the :mod:`shutil` module provides -a higher level interface that is easier to use:: - - >>> import shutil - >>> shutil.copyfile('data.db', 'archive.db') - 'archive.db' - >>> shutil.move('/build/executables', 'installdir') - 'installdir' - - -.. _tut-file-wildcards: - -File Wildcards -============== - -The :mod:`glob` module provides a function for making file lists from directory -wildcard searches:: - - >>> import glob - >>> glob.glob('*.py') - ['primes.py', 'random.py', 'quote.py'] - - -.. _tut-command-line-arguments: - -Command Line Arguments -====================== - -Common utility scripts often need to process command line arguments. These -arguments are stored in the :mod:`sys` module's *argv* attribute as a list. For -instance the following output results from running ``python demo.py one two -three`` at the command line:: - - >>> import sys - >>> print(sys.argv) - ['demo.py', 'one', 'two', 'three'] - -The :mod:`getopt` module processes *sys.argv* using the conventions of the Unix -:func:`getopt` function. More powerful and flexible command line processing is -provided by the :mod:`argparse` module. - - -.. _tut-stderr: - -Error Output Redirection and Program Termination -================================================ - -The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. -The latter is useful for emitting warnings and error messages to make them -visible even when *stdout* has been redirected:: - - >>> sys.stderr.write('Warning, log file not found starting a new one\n') - Warning, log file not found starting a new one - -The most direct way to terminate a script is to use ``sys.exit()``. - - -.. _tut-string-pattern-matching: - -String Pattern Matching -======================= - -The :mod:`re` module provides regular expression tools for advanced string -processing. For complex matching and manipulation, regular expressions offer -succinct, optimized solutions:: - - >>> import re - >>> re.findall(r'\bf[a-z]*', 'which foot or hand fell fastest') - ['foot', 'fell', 'fastest'] - >>> re.sub(r'(\b[a-z]+) \1', r'\1', 'cat in the the hat') - 'cat in the hat' - -When only simple capabilities are needed, string methods are preferred because -they are easier to read and debug:: - - >>> 'tea for too'.replace('too', 'two') - 'tea for two' - - -.. _tut-mathematics: - -Mathematics -=========== - -The :mod:`math` module gives access to the underlying C library functions for -floating point math:: - - >>> import math - >>> math.cos(math.pi / 4) - 0.70710678118654757 - >>> math.log(1024, 2) - 10.0 - -The :mod:`random` module provides tools for making random selections:: - - >>> import random - >>> random.choice(['apple', 'pear', 'banana']) - 'apple' - >>> random.sample(range(100), 10) # sampling without replacement - [30, 83, 16, 4, 8, 81, 41, 50, 18, 33] - >>> random.random() # random float - 0.17970987693706186 - >>> random.randrange(6) # random integer chosen from range(6) - 4 - -The :mod:`statistics` module calculates basic statistical properties -(the mean, median, variance, etc.) of numeric data:: - - >>> import statistics - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> statistics.mean(data) - 1.6071428571428572 - >>> statistics.median(data) - 1.25 - >>> statistics.variance(data) - 1.3720238095238095 - -The SciPy project has many other modules for numerical -computations. - -.. _tut-internet-access: - -Internet Access -=============== - -There are a number of modules for accessing the internet and processing internet -protocols. Two of the simplest are :mod:`urllib.request` for retrieving data -from URLs and :mod:`smtplib` for sending mail:: - - >>> from urllib.request import urlopen - >>> with urlopen('http://tycho.usno.navy.mil/cgi-bin/timer.pl') as response: - ... for line in response: - ... line = line.decode('utf-8') # Decoding the binary data to text. - ... if 'EST' in line or 'EDT' in line: # look for Eastern Time - ... print(line) - -
    Nov. 25, 09:43:32 PM EST - - >>> import smtplib - >>> server = smtplib.SMTP('localhost') - >>> server.sendmail('soothsayer@example.org', 'jcaesar@example.org', - ... """To: jcaesar@example.org - ... From: soothsayer@example.org - ... - ... Beware the Ides of March. - ... """) - >>> server.quit() - -(Note that the second example needs a mailserver running on localhost.) - - -.. _tut-dates-and-times: - -Dates and Times -=============== - -The :mod:`datetime` module supplies classes for manipulating dates and times in -both simple and complex ways. While date and time arithmetic is supported, the -focus of the implementation is on efficient member extraction for output -formatting and manipulation. The module also supports objects that are timezone -aware. :: - - >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() - >>> now - datetime.date(2003, 12, 2) - >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") - '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' - - >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) - >>> age = now - birthday - >>> age.days - 14368 - - -.. _tut-data-compression: - -Data Compression -================ - -Common data archiving and compression formats are directly supported by modules -including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and -:mod:`tarfile`. :: - - >>> import zlib - >>> s = b'witch which has which witches wrist watch' - >>> len(s) - 41 - >>> t = zlib.compress(s) - >>> len(t) - 37 - >>> zlib.decompress(t) - b'witch which has which witches wrist watch' - >>> zlib.crc32(s) - 226805979 - - -.. _tut-performance-measurement: - -Performance Measurement -======================= - -Some Python users develop a deep interest in knowing the relative performance of -different approaches to the same problem. Python provides a measurement tool -that answers those questions immediately. - -For example, it may be tempting to use the tuple packing and unpacking feature -instead of the traditional approach to swapping arguments. The :mod:`timeit` -module quickly demonstrates a modest performance advantage:: - - >>> from timeit import Timer - >>> Timer('t=a; a=b; b=t', 'a=1; b=2').timeit() - 0.57535828626024577 - >>> Timer('a,b = b,a', 'a=1; b=2').timeit() - 0.54962537085770791 - -In contrast to :mod:`timeit`'s fine level of granularity, the :mod:`profile` and -:mod:`pstats` modules provide tools for identifying time critical sections in -larger blocks of code. - - -.. _tut-quality-control: - -Quality Control -=============== - -One approach for developing high quality software is to write tests for each -function as it is developed and to run those tests frequently during the -development process. - -The :mod:`doctest` module provides a tool for scanning a module and validating -tests embedded in a program's docstrings. Test construction is as simple as -cutting-and-pasting a typical call along with its results into the docstring. -This improves the documentation by providing the user with an example and it -allows the doctest module to make sure the code remains true to the -documentation:: - - def average(values): - """Computes the arithmetic mean of a list of numbers. - - >>> print(average([20, 30, 70])) - 40.0 - """ - return sum(values) / len(values) - - import doctest - doctest.testmod() # automatically validate the embedded tests - -The :mod:`unittest` module is not as effortless as the :mod:`doctest` module, -but it allows a more comprehensive set of tests to be maintained in a separate -file:: - - import unittest - - class TestStatisticalFunctions(unittest.TestCase): - - def test_average(self): - self.assertEqual(average([20, 30, 70]), 40.0) - self.assertEqual(round(average([1, 5, 7]), 1), 4.3) - with self.assertRaises(ZeroDivisionError): - average([]) - with self.assertRaises(TypeError): - average(20, 30, 70) - - unittest.main() # Calling from the command line invokes all tests - - -.. _tut-batteries-included: - -Batteries Included -================== - -Python has a "batteries included" philosophy. This is best seen through the -sophisticated and robust capabilities of its larger packages. For example: - -* The :mod:`xmlrpc.client` and :mod:`xmlrpc.server` modules make implementing - remote procedure calls into an almost trivial task. Despite the modules - names, no direct knowledge or handling of XML is needed. - -* The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and - :mod:`poplib` which actually send and receive messages, the email package has - a complete toolset for building or decoding complex message structures - (including attachments) and for implementing internet encoding and header - protocols. - -* The :mod:`json` package provides robust support for parsing this - popular data interchange format. The :mod:`csv` module supports - direct reading and writing of files in Comma-Separated Value format, - commonly supported by databases and spreadsheets. XML processing is - supported by the :mod:`xml.etree.ElementTree`, :mod:`xml.dom` and - :mod:`xml.sax` packages. Together, these modules and packages - greatly simplify data interchange between Python applications and - other tools. - -* The :mod:`sqlite3` module is a wrapper for the SQLite database - library, providing a persistent database that can be updated and - accessed using slightly nonstandard SQL syntax. - -* Internationalization is supported by a number of modules including - :mod:`gettext`, :mod:`locale`, and the :mod:`codecs` package. diff --git a/third_party/python/Doc/tutorial/stdlib2.rst b/third_party/python/Doc/tutorial/stdlib2.rst deleted file mode 100644 index bf02c71b6..000000000 --- a/third_party/python/Doc/tutorial/stdlib2.rst +++ /dev/null @@ -1,405 +0,0 @@ -.. _tut-brieftourtwo: - -********************************************** -Brief Tour of the Standard Library --- Part II -********************************************** - -This second tour covers more advanced modules that support professional -programming needs. These modules rarely occur in small scripts. - - -.. _tut-output-formatting: - -Output Formatting -================= - -The :mod:`reprlib` module provides a version of :func:`repr` customized for -abbreviated displays of large or deeply nested containers:: - - >>> import reprlib - >>> reprlib.repr(set('supercalifragilisticexpialidocious')) - "{'a', 'c', 'd', 'e', 'f', 'g', ...}" - -The :mod:`pprint` module offers more sophisticated control over printing both -built-in and user defined objects in a way that is readable by the interpreter. -When the result is longer than one line, the "pretty printer" adds line breaks -and indentation to more clearly reveal data structure:: - - >>> import pprint - >>> t = [[[['black', 'cyan'], 'white', ['green', 'red']], [['magenta', - ... 'yellow'], 'blue']]] - ... - >>> pprint.pprint(t, width=30) - [[[['black', 'cyan'], - 'white', - ['green', 'red']], - [['magenta', 'yellow'], - 'blue']]] - -The :mod:`textwrap` module formats paragraphs of text to fit a given screen -width:: - - >>> import textwrap - >>> doc = """The wrap() method is just like fill() except that it returns - ... a list of strings instead of one big string with newlines to separate - ... the wrapped lines.""" - ... - >>> print(textwrap.fill(doc, width=40)) - The wrap() method is just like fill() - except that it returns a list of strings - instead of one big string with newlines - to separate the wrapped lines. - -The :mod:`locale` module accesses a database of culture specific data formats. -The grouping attribute of locale's format function provides a direct way of -formatting numbers with group separators:: - - >>> import locale - >>> locale.setlocale(locale.LC_ALL, 'English_United States.1252') - 'English_United States.1252' - >>> conv = locale.localeconv() # get a mapping of conventions - >>> x = 1234567.8 - >>> locale.format("%d", x, grouping=True) - '1,234,567' - >>> locale.format_string("%s%.*f", (conv['currency_symbol'], - ... conv['frac_digits'], x), grouping=True) - '$1,234,567.80' - - -.. _tut-templating: - -Templating -========== - -The :mod:`string` module includes a versatile :class:`~string.Template` class -with a simplified syntax suitable for editing by end-users. This allows users -to customize their applications without having to alter the application. - -The format uses placeholder names formed by ``$`` with valid Python identifiers -(alphanumeric characters and underscores). Surrounding the placeholder with -braces allows it to be followed by more alphanumeric letters with no intervening -spaces. Writing ``$$`` creates a single escaped ``$``:: - - >>> from string import Template - >>> t = Template('${village}folk send $$10 to $cause.') - >>> t.substitute(village='Nottingham', cause='the ditch fund') - 'Nottinghamfolk send $10 to the ditch fund.' - -The :meth:`~string.Template.substitute` method raises a :exc:`KeyError` when a -placeholder is not supplied in a dictionary or a keyword argument. For -mail-merge style applications, user supplied data may be incomplete and the -:meth:`~string.Template.safe_substitute` method may be more appropriate --- -it will leave placeholders unchanged if data is missing:: - - >>> t = Template('Return the $item to $owner.') - >>> d = dict(item='unladen swallow') - >>> t.substitute(d) - Traceback (most recent call last): - ... - KeyError: 'owner' - >>> t.safe_substitute(d) - 'Return the unladen swallow to $owner.' - -Template subclasses can specify a custom delimiter. For example, a batch -renaming utility for a photo browser may elect to use percent signs for -placeholders such as the current date, image sequence number, or file format:: - - >>> import time, os.path - >>> photofiles = ['img_1074.jpg', 'img_1076.jpg', 'img_1077.jpg'] - >>> class BatchRename(Template): - ... delimiter = '%' - >>> fmt = input('Enter rename style (%d-date %n-seqnum %f-format): ') - Enter rename style (%d-date %n-seqnum %f-format): Ashley_%n%f - - >>> t = BatchRename(fmt) - >>> date = time.strftime('%d%b%y') - >>> for i, filename in enumerate(photofiles): - ... base, ext = os.path.splitext(filename) - ... newname = t.substitute(d=date, n=i, f=ext) - ... print('{0} --> {1}'.format(filename, newname)) - - img_1074.jpg --> Ashley_0.jpg - img_1076.jpg --> Ashley_1.jpg - img_1077.jpg --> Ashley_2.jpg - -Another application for templating is separating program logic from the details -of multiple output formats. This makes it possible to substitute custom -templates for XML files, plain text reports, and HTML web reports. - - -.. _tut-binary-formats: - -Working with Binary Data Record Layouts -======================================= - -The :mod:`struct` module provides :func:`~struct.pack` and -:func:`~struct.unpack` functions for working with variable length binary -record formats. The following example shows -how to loop through header information in a ZIP file without using the -:mod:`zipfile` module. Pack codes ``"H"`` and ``"I"`` represent two and four -byte unsigned numbers respectively. The ``"<"`` indicates that they are -standard size and in little-endian byte order:: - - import struct - - with open('myfile.zip', 'rb') as f: - data = f.read() - - start = 0 - for i in range(3): # show the first 3 file headers - start += 14 - fields = struct.unpack('>> import weakref, gc - >>> class A: - ... def __init__(self, value): - ... self.value = value - ... def __repr__(self): - ... return str(self.value) - ... - >>> a = A(10) # create a reference - >>> d = weakref.WeakValueDictionary() - >>> d['primary'] = a # does not create a reference - >>> d['primary'] # fetch the object if it is still alive - 10 - >>> del a # remove the one reference - >>> gc.collect() # run garbage collection right away - 0 - >>> d['primary'] # entry was automatically removed - Traceback (most recent call last): - File "", line 1, in - d['primary'] # entry was automatically removed - File "C:/python36/lib/weakref.py", line 46, in __getitem__ - o = self.data[key]() - KeyError: 'primary' - - -.. _tut-list-tools: - -Tools for Working with Lists -============================ - -Many data structure needs can be met with the built-in list type. However, -sometimes there is a need for alternative implementations with different -performance trade-offs. - -The :mod:`array` module provides an :class:`~array.array()` object that is like -a list that stores only homogeneous data and stores it more compactly. The -following example shows an array of numbers stored as two byte unsigned binary -numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular -lists of Python int objects:: - - >>> from array import array - >>> a = array('H', [4000, 10, 700, 22222]) - >>> sum(a) - 26932 - >>> a[1:3] - array('H', [10, 700]) - -The :mod:`collections` module provides a :class:`~collections.deque()` object -that is like a list with faster appends and pops from the left side but slower -lookups in the middle. These objects are well suited for implementing queues -and breadth first tree searches:: - - >>> from collections import deque - >>> d = deque(["task1", "task2", "task3"]) - >>> d.append("task4") - >>> print("Handling", d.popleft()) - Handling task1 - -:: - - unsearched = deque([starting_node]) - def breadth_first_search(unsearched): - node = unsearched.popleft() - for m in gen_moves(node): - if is_goal(m): - return m - unsearched.append(m) - -In addition to alternative list implementations, the library also offers other -tools such as the :mod:`bisect` module with functions for manipulating sorted -lists:: - - >>> import bisect - >>> scores = [(100, 'perl'), (200, 'tcl'), (400, 'lua'), (500, 'python')] - >>> bisect.insort(scores, (300, 'ruby')) - >>> scores - [(100, 'perl'), (200, 'tcl'), (300, 'ruby'), (400, 'lua'), (500, 'python')] - -The :mod:`heapq` module provides functions for implementing heaps based on -regular lists. The lowest valued entry is always kept at position zero. This -is useful for applications which repeatedly access the smallest element but do -not want to run a full list sort:: - - >>> from heapq import heapify, heappop, heappush - >>> data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] - >>> heapify(data) # rearrange the list into heap order - >>> heappush(data, -5) # add a new entry - >>> [heappop(data) for i in range(3)] # fetch the three smallest entries - [-5, 0, 1] - - -.. _tut-decimal-fp: - -Decimal Floating Point Arithmetic -================================= - -The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` -implementation of binary floating point, the class is especially helpful for - -* financial applications and other uses which require exact decimal - representation, -* control over precision, -* control over rounding to meet legal or regulatory requirements, -* tracking of significant decimal places, or -* applications where the user expects the results to match calculations done by - hand. - -For example, calculating a 5% tax on a 70 cent phone charge gives different -results in decimal floating point and binary floating point. The difference -becomes significant if the results are rounded to the nearest cent:: - - >>> from decimal import * - >>> round(Decimal('0.70') * Decimal('1.05'), 2) - Decimal('0.74') - >>> round(.70 * 1.05, 2) - 0.73 - -The :class:`~decimal.Decimal` result keeps a trailing zero, automatically -inferring four place significance from multiplicands with two place -significance. Decimal reproduces mathematics as done by hand and avoids -issues that can arise when binary floating point cannot exactly represent -decimal quantities. - -Exact representation enables the :class:`~decimal.Decimal` class to perform -modulo calculations and equality tests that are unsuitable for binary floating -point:: - - >>> Decimal('1.00') % Decimal('.10') - Decimal('0.00') - >>> 1.00 % 0.10 - 0.09999999999999995 - - >>> sum([Decimal('0.1')]*10) == Decimal('1.0') - True - >>> sum([0.1]*10) == 1.0 - False - -The :mod:`decimal` module provides arithmetic with as much precision as needed:: - - >>> getcontext().prec = 36 - >>> Decimal(1) / Decimal(7) - Decimal('0.142857142857142857142857142857142857') - - diff --git a/third_party/python/Doc/tutorial/venv.rst b/third_party/python/Doc/tutorial/venv.rst deleted file mode 100644 index dc4136e42..000000000 --- a/third_party/python/Doc/tutorial/venv.rst +++ /dev/null @@ -1,210 +0,0 @@ - -.. _tut-venv: - -********************************* -Virtual Environments and Packages -********************************* - -Introduction -============ - -Python applications will often use packages and modules that don't -come as part of the standard library. Applications will sometimes -need a specific version of a library, because the application may -require that a particular bug has been fixed or the application may be -written using an obsolete version of the library's interface. - -This means it may not be possible for one Python installation to meet -the requirements of every application. If application A needs version -1.0 of a particular module but application B needs version 2.0, then -the requirements are in conflict and installing either version 1.0 or 2.0 -will leave one application unable to run. - -The solution for this problem is to create a :term:`virtual environment`, a -self-contained directory tree that contains a Python installation for a -particular version of Python, plus a number of additional packages. - -Different applications can then use different virtual environments. -To resolve the earlier example of conflicting requirements, -application A can have its own virtual environment with version 1.0 -installed while application B has another virtual environment with version 2.0. -If application B requires a library be upgraded to version 3.0, this will -not affect application A's environment. - - -Creating Virtual Environments -============================= - -The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. - -To create a virtual environment, decide upon a directory where you want to -place it, and run the :mod:`venv` module as a script with the directory path:: - - python3 -m venv tutorial-env - -This will create the ``tutorial-env`` directory if it doesn't exist, -and also create directories inside it containing a copy of the Python -interpreter, the standard library, and various supporting files. - -Once you've created a virtual environment, you may activate it. - -On Windows, run:: - - tutorial-env\Scripts\activate.bat - -On Unix or MacOS, run:: - - source tutorial-env/bin/activate - -(This script is written for the bash shell. If you use the -:program:`csh` or :program:`fish` shells, there are alternate -``activate.csh`` and ``activate.fish`` scripts you should use -instead.) - -Activating the virtual environment will change your shell's prompt to show what -virtual environment you're using, and modify the environment so that running -``python`` will get you that particular version and installation of Python. -For example: - -.. code-block:: bash - - $ source ~/envs/tutorial-env/bin/activate - (tutorial-env) $ python - Python 3.5.1 (default, May 6 2016, 10:59:36) - ... - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python35.zip', ..., - '~/envs/tutorial-env/lib/python3.5/site-packages'] - >>> - - -Managing Packages with pip -========================== - -You can install, upgrade, and remove packages using a program called -:program:`pip`. By default ``pip`` will install packages from the Python -Package Index, . You can browse the Python -Package Index by going to it in your web browser, or you can use ``pip``'s -limited search feature: - -.. code-block:: bash - - (tutorial-env) $ pip search astronomy - skyfield - Elegant astronomy for Python - gary - Galactic astronomy and gravitational dynamics. - novas - The United States Naval Observatory NOVAS astronomy library - astroobs - Provides astronomy ephemeris to plan telescope observations - PyAstronomy - A collection of astronomy related tools for Python. - ... - -``pip`` has a number of subcommands: "search", "install", "uninstall", -"freeze", etc. (Consult the :ref:`installing-index` guide for -complete documentation for ``pip``.) - -You can install the latest version of a package by specifying a package's name: - -.. code-block:: bash - - (tutorial-env) $ pip install novas - Collecting novas - Downloading novas-3.1.1.3.tar.gz (136kB) - Installing collected packages: novas - Running setup.py install for novas - Successfully installed novas-3.1.1.3 - -You can also install a specific version of a package by giving the -package name followed by ``==`` and the version number: - -.. code-block:: bash - - (tutorial-env) $ pip install requests==2.6.0 - Collecting requests==2.6.0 - Using cached requests-2.6.0-py2.py3-none-any.whl - Installing collected packages: requests - Successfully installed requests-2.6.0 - -If you re-run this command, ``pip`` will notice that the requested -version is already installed and do nothing. You can supply a -different version number to get that version, or you can run ``pip -install --upgrade`` to upgrade the package to the latest version: - -.. code-block:: bash - - (tutorial-env) $ pip install --upgrade requests - Collecting requests - Installing collected packages: requests - Found existing installation: requests 2.6.0 - Uninstalling requests-2.6.0: - Successfully uninstalled requests-2.6.0 - Successfully installed requests-2.7.0 - -``pip uninstall`` followed by one or more package names will remove the -packages from the virtual environment. - -``pip show`` will display information about a particular package: - -.. code-block:: bash - - (tutorial-env) $ pip show requests - --- - Metadata-Version: 2.0 - Name: requests - Version: 2.7.0 - Summary: Python HTTP for Humans. - Home-page: http://python-requests.org - Author: Kenneth Reitz - Author-email: me@kennethreitz.com - License: Apache 2.0 - Location: /Users/akuchling/envs/tutorial-env/lib/python3.4/site-packages - Requires: - -``pip list`` will display all of the packages installed in the virtual -environment: - -.. code-block:: bash - - (tutorial-env) $ pip list - novas (3.1.1.3) - numpy (1.9.2) - pip (7.0.3) - requests (2.7.0) - setuptools (16.0) - -``pip freeze`` will produce a similar list of the installed packages, -but the output uses the format that ``pip install`` expects. -A common convention is to put this list in a ``requirements.txt`` file: - -.. code-block:: bash - - (tutorial-env) $ pip freeze > requirements.txt - (tutorial-env) $ cat requirements.txt - novas==3.1.1.3 - numpy==1.9.2 - requests==2.7.0 - -The ``requirements.txt`` can then be committed to version control and -shipped as part of an application. Users can then install all the -necessary packages with ``install -r``: - -.. code-block:: bash - - (tutorial-env) $ pip install -r requirements.txt - Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) - ... - Collecting numpy==1.9.2 (from -r requirements.txt (line 2)) - ... - Collecting requests==2.7.0 (from -r requirements.txt (line 3)) - ... - Installing collected packages: novas, numpy, requests - Running setup.py install for novas - Successfully installed novas-3.1.1.3 numpy-1.9.2 requests-2.7.0 - -``pip`` has many more options. Consult the :ref:`installing-index` -guide for complete documentation for ``pip``. When you've written -a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. diff --git a/third_party/python/Doc/tutorial/whatnow.rst b/third_party/python/Doc/tutorial/whatnow.rst deleted file mode 100644 index d876d0740..000000000 --- a/third_party/python/Doc/tutorial/whatnow.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. _tut-whatnow: - -********* -What Now? -********* - -Reading this tutorial has probably reinforced your interest in using Python --- -you should be eager to apply Python to solving your real-world problems. Where -should you go to learn more? - -This tutorial is part of Python's documentation set. Some other documents in -the set are: - -* :ref:`library-index`: - - You should browse through this manual, which gives complete (though terse) - reference material about types, functions, and the modules in the standard - library. The standard Python distribution includes a *lot* of additional code. - There are modules to read Unix mailboxes, retrieve documents via HTTP, generate - random numbers, parse command-line options, write CGI programs, compress data, - and many other tasks. Skimming through the Library Reference will give you an - idea of what's available. - -* :ref:`installing-index` explains how to install additional modules written - by other Python users. - -* :ref:`reference-index`: A detailed explanation of Python's syntax and - semantics. It's heavy reading, but is useful as a complete guide to the - language itself. - -More Python resources: - -* https://www.python.org: The major Python Web site. It contains code, - documentation, and pointers to Python-related pages around the Web. This Web - site is mirrored in various places around the world, such as Europe, Japan, and - Australia; a mirror may be faster than the main site, depending on your - geographical location. - -* https://docs.python.org: Fast access to Python's documentation. - -* https://pypi.org: The Python Package Index, previously also nicknamed - the Cheese Shop, is an index of user-created Python modules that are available - for download. Once you begin releasing code, you can register it here so that - others can find it. - -* https://code.activestate.com/recipes/langs/python/: The Python Cookbook is a - sizable collection of code examples, larger modules, and useful scripts. - Particularly notable contributions are collected in a book also titled Python - Cookbook (O'Reilly & Associates, ISBN 0-596-00797-3.) - -* http://www.pyvideo.org collects links to Python-related videos from - conferences and user-group meetings. - -* https://scipy.org: The Scientific Python project includes modules for fast - array computations and manipulations plus a host of packages for such - things as linear algebra, Fourier transforms, non-linear solvers, - random number distributions, statistical analysis and the like. - -For Python-related questions and problem reports, you can post to the newsgroup -:newsgroup:`comp.lang.python`, or send them to the mailing list at -python-list@python.org. The newsgroup and mailing list are gatewayed, so -messages posted to one will automatically be forwarded to the other. There are -hundreds of postings a day, asking (and -answering) questions, suggesting new features, and announcing new modules. -Mailing list archives are available at https://mail.python.org/pipermail/. - -Before posting, be sure to check the list of -:ref:`Frequently Asked Questions ` (also called the FAQ). The -FAQ answers many of the questions that come up again and again, and may -already contain the solution for your problem. diff --git a/third_party/python/Doc/using/cmdline.rst b/third_party/python/Doc/using/cmdline.rst deleted file mode 100644 index d14793a10..000000000 --- a/third_party/python/Doc/using/cmdline.rst +++ /dev/null @@ -1,745 +0,0 @@ -.. highlightlang:: sh - -.. ATTENTION: You probably should update Misc/python.man, too, if you modify - this file. - -.. _using-on-general: - -Command line and environment -============================ - -The CPython interpreter scans the command line and the environment for various -settings. - -.. impl-detail:: - - Other implementations' command line schemes may differ. See - :ref:`implementations` for further resources. - - -.. _using-on-cmdline: - -Command line ------------- - -When invoking Python, you may specify any of these options:: - - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] - -The most common use case is, of course, a simple invocation of a script:: - - python myscript.py - - -.. _using-on-interface-options: - -Interface options -~~~~~~~~~~~~~~~~~ - -The interpreter interface resembles that of the UNIX shell, but provides some -additional methods of invocation: - -* When called with standard input connected to a tty device, it prompts for - commands and executes them until an EOF (an end-of-file character, you can - produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. -* When called with a file name argument or with a file as standard input, it - reads and executes a script from that file. -* When called with a directory name argument, it reads and executes an - appropriately named script from that directory. -* When called with ``-c command``, it executes the Python statement(s) given as - *command*. Here *command* may contain multiple statements separated by - newlines. Leading whitespace is significant in Python statements! -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. - -In non-interactive mode, the entire input is parsed before it is executed. - -An interface option terminates the list of options consumed by the interpreter, -all consecutive arguments will end up in :data:`sys.argv` -- note that the first -element, subscript zero (``sys.argv[0]``), is a string reflecting the program's -source. - -.. cmdoption:: -c - - Execute the Python code in *command*. *command* can be one or more - statements separated by newlines, with significant leading whitespace as in - normal module code. - - If this option is given, the first element of :data:`sys.argv` will be - ``"-c"`` and the current directory will be added to the start of - :data:`sys.path` (allowing modules in that directory to be imported as top - level modules). - - -.. cmdoption:: -m - - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. - - Since the argument is a *module* name, you must not give a file extension - (``.py``). The module name should be a valid absolute Python module name, but - the implementation may not always enforce this (e.g. it may allow you to - use a name that includes a hyphen). - - Package names (including namespace packages) are also permitted. When a - package name is supplied instead - of a normal module, the interpreter will execute ``.__main__`` as - the main module. This behaviour is deliberately similar to the handling - of directories and zipfiles that are passed to the interpreter as the - script argument. - - .. note:: - - This option cannot be used with built-in modules and extension modules - written in C, since they do not have Python module files. However, it - can still be used for precompiled modules, even if the original source - file is not available. - - If this option is given, the first element of :data:`sys.argv` will be the - full path to the module file (while the module file is being located, the - first element will be set to ``"-m"``). As with the :option:`-c` option, - the current directory will be added to the start of :data:`sys.path`. - - Many standard library modules contain code that is invoked on their execution - as a script. An example is the :mod:`timeit` module:: - - python -mtimeit -s 'setup here' 'benchmarked code here' - python -mtimeit -h # for details - - .. seealso:: - :func:`runpy.run_module` - Equivalent functionality directly available to Python code - - :pep:`338` -- Executing modules as scripts - - - .. versionchanged:: 3.1 - Supply the package name to run a ``__main__`` submodule. - - .. versionchanged:: 3.4 - namespace packages are also supported - - -.. describe:: - - - Read commands from standard input (:data:`sys.stdin`). If standard input is - a terminal, :option:`-i` is implied. - - If this option is given, the first element of :data:`sys.argv` will be - ``"-"`` and the current directory will be added to the start of - :data:`sys.path`. - - -.. describe:: `` and ````. - -- Issue #10817: Fix urlretrieve function to raise ContentTooShortError even - when reporthook is None. Patch by Jyrki Pulliainen. - -- Fix the xmlrpc.client user agent to return something similar to - urllib.request user agent: "Python-xmlrpc/3.3". - -- Issue #13293: Better error message when trying to marshal bytes using - xmlrpc.client. - -- Issue #13291: NameError in xmlrpc package. - -- Issue #13258: Use callable() built-in in the standard library. - -- Issue #13273: fix a bug that prevented HTMLParser to properly detect some - tags when strict=False. - -- Issue #11183: Add finer-grained exceptions to the ssl module, so that - you don't have to inspect the exception's attributes in the common case. - -- Issue #13216: Add cp65001 codec, the Windows UTF-8 (CP_UTF8). - -- Issue #13226: Add RTLD_xxx constants to the os module. These constants can be - used with sys.setdlopenflags(). - -- Issue #10278: Add clock_getres(), clock_gettime() and CLOCK_xxx constants to - the time module. time.clock_gettime(time.CLOCK_MONOTONIC) provides a - monotonic clock - -- Issue #10332: multiprocessing: fix a race condition when a Pool is closed - before all tasks have completed. - -- Issue #13255: wrong docstrings in array module. - -- Issue #8540: Remove deprecated Context._clamp attribute in Decimal module. - -- Issue #13235: Added DeprecationWarning to logging.warn() method and function. - -- Issue #9168: now smtpd is able to bind privileged port. - -- Issue #12529: fix cgi.parse_header issue on strings with double-quotes and - semicolons together. Patch by Ben Darnell and Petri Lehtinen. - -- Issue #13227: functools.lru_cache() now has an option to distinguish - calls with different argument types. - -- Issue #6090: zipfile raises a ValueError when a document with a timestamp - earlier than 1980 is provided. Patch contributed by Petri Lehtinen. - -- Issue #13150: sysconfig no longer parses the Makefile and config.h files - when imported, instead doing it at build time. This makes importing - sysconfig faster and reduces Python startup time by 20%. - -- Issue #12448: smtplib now flushes stdout while running ``python -m smtplib`` - in order to display the prompt correctly. - -- Issue #12454: The mailbox module is now using ASCII, instead of the locale - encoding, to read and write .mh_sequences files. - -- Issue #13194: zlib.compressobj().copy() and zlib.decompressobj().copy() are - now available on Windows. - -- Issue #1673007: urllib.request now supports HEAD request via new method argument. - Patch contributions by David Stanek, Patrick Westerhoff and Ezio Melotti. - -- Issue #12386: packaging does not fail anymore when writing the RESOURCES - file. - -- Issue #13158: Fix decoding and encoding of GNU tar specific base-256 number - fields in tarfile. - -- Issue #13025: mimetypes is now reading MIME types using the UTF-8 encoding, - instead of the locale encoding. - -- Issue #10653: On Windows, use strftime() instead of wcsftime() because - wcsftime() doesn't format time zone correctly. - -- Issue #13150: The tokenize module doesn't compile large regular expressions - at startup anymore. - -- Issue #11171: Fix distutils.sysconfig.get_makefile_filename when Python was - configured with different prefix and exec-prefix. - -- Issue #11254: Teach distutils and packaging to compile .pyc and .pyo files in - PEP 3147-compliant __pycache__ directories. - -- Issue #7367: Fix pkgutil.walk_paths to skip directories whose - contents cannot be read. - -- Issue #3163: The struct module gets new format characters 'n' and 'N' - supporting C integer types ``ssize_t`` and ``size_t``, respectively. - -- Issue #13099: Fix sqlite3.Cursor.lastrowid under a Turkish locale. - Reported and diagnosed by Thomas Kluyver. - -- Issue #13087: BufferedReader.seek() now always raises UnsupportedOperation - if the underlying raw stream is unseekable, even if the seek could be - satisfied using the internal buffer. Patch by John O'Connor. - -- Issue #7689: Allow pickling of dynamically created classes when their - metaclass is registered with copyreg. Patch by Nicolas M. Thiéry and Craig - Citro. - -- Issue #13034: When decoding some SSL certificates, the subjectAltName - extension could be unreported. - -- Issue #12306: Expose the runtime version of the zlib C library as a constant, - ZLIB_RUNTIME_VERSION, in the zlib module. Patch by Torsten Landschoff. - -- Issue #12959: Add collections.ChainMap to collections.__all__. - -- Issue #8933: distutils' PKG-INFO files and packaging's METADATA files will - now correctly report Metadata-Version: 1.1 instead of 1.0 if a Classifier or - Download-URL field is present. - -- Issue #12567: Add curses.unget_wch() function. Push a character so the next - get_wch() will return it. - -- Issue #9561: distutils and packaging now writes egg-info files using UTF-8, - instead of the locale encoding. - -- Issue #8286: The distutils command sdist will print a warning message instead - of crashing when an invalid path is given in the manifest template. - -- Issue #12841: tarfile unnecessarily checked the existence of numerical user - and group ids on extraction. If one of them did not exist the respective id - of the current user (i.e. root) was used for the file and ownership - information was lost. - -- Issue #12888: Fix a bug in HTMLParser.unescape that prevented it to escape - more than 128 entities. Patch by Peter Otten. - -- Issue #12878: Expose a __dict__ attribute on io.IOBase and its subclasses. - -- Issue #12494: On error, call(), check_call(), check_output() and - getstatusoutput() functions of the subprocess module now kill the process, - read its status (to avoid zombis) and close pipes. - -- Issue #12720: Expose low-level Linux extended file attribute functions in os. - -- Issue #10946: The distutils commands bdist_dumb, bdist_wininst and bdist_msi - now respect a --skip-build option given to bdist. The packaging commands - were fixed too. - -- Issue #12847: Fix a crash with negative PUT and LONG_BINPUT arguments in - the C pickle implementation. - -- Issue #11564: Avoid crashes when trying to pickle huge objects or containers - (more than 2**31 items). Instead, in most cases, an OverflowError is raised. - -- Issue #12287: Fix a stack corruption in ossaudiodev module when the FD is - greater than FD_SETSIZE. - -- Issue #12839: Fix crash in zlib module due to version mismatch. - Fix by Richard M. Tew. - -- Issue #9923: The mailcap module now correctly uses the platform path - separator for the MAILCAP environment variable on non-POSIX platforms. - -- Issue #12835: Follow up to #6560 that unconditionally prevents use of the - unencrypted sendmsg/recvmsg APIs on SSL wrapped sockets. Patch by David - Watson. - -- Issue #12803: SSLContext.load_cert_chain() now accepts a password argument - to be used if the private key is encrypted. Patch by Adam Simpkins. - -- Issue #11657: Fix sending file descriptors over 255 over a multiprocessing - Pipe. - -- Issue #12811: tabnanny.check() now promptly closes checked files. Patch by - Anthony Briggs. - -- Issue #6560: The sendmsg/recvmsg API is now exposed by the socket module - when provided by the underlying platform, supporting processing of - ancillary data in pure Python code. Patch by David Watson and Heiko Wundram. - -- Issue #12326: On Linux, sys.platform doesn't contain the major version - anymore. It is now always 'linux', instead of 'linux2' or 'linux3' depending - on the Linux version used to build Python. - -- Issue #12213: Fix a buffering bug with interleaved reads and writes that - could appear on BufferedRandom streams. - -- Issue #12778: Reduce memory consumption when JSON-encoding a large - container of many small objects. - -- Issue #12650: Fix a race condition where a subprocess.Popen could leak - resources (FD/zombie) when killed at the wrong time. - -- Issue #12744: Fix inefficient representation of integers between 2**31 and - 2**63 on systems with a 64-bit C "long". - -- Issue #12646: Add an 'eof' attribute to zlib.Decompress, to make it easier to - detect truncated input streams. - -- Issue #11513: Fix exception handling ``tarfile.TarFile.gzopen()`` when - the file cannot be opened. - -- Issue #12687: Fix a possible buffering bug when unpickling text mode - (protocol 0, mostly) pickles. - -- Issue #10087: Fix the html output format of the calendar module. - -- Issue #13121: add support for inplace math operators to collections.Counter. - -- Add support for unary plus and unary minus to collections.Counter. - -- Issue #12683: urlparse updated to include svn as schemes that uses relative - paths. (svn from 1.5 onwards support relative path). - -- Issue #12655: Expose functions from sched.h in the os module: sched_yield(), - sched_setscheduler(), sched_getscheduler(), sched_setparam(), - sched_get_min_priority(), sched_get_max_priority(), sched_rr_get_interval(), - sched_getaffinity(), sched_setaffinity(). - -- Add ThreadError to threading.__all__. - -- Issues #11104, #8688: Fix the behavior of distutils' sdist command with - manually-maintained MANIFEST files. - -- Issue #11281: smtplib.STMP gets source_address parameter, which adds the - ability to bind to specific source address on a machine with multiple - interfaces. Patch by Paulo Scardine. - -- Issue #12464: tempfile.TemporaryDirectory.cleanup() should not follow - symlinks: fix it. Patch by Petri Lehtinen. - -- Issue #8887: "pydoc somebuiltin.somemethod" (or help('somebuiltin.somemethod') - in Python code) now finds the doc of the method. - -- Issue #10968: Remove indirection in threading. The public names (Event, - Condition, etc.) used to be factory functions returning instances of hidden - classes (_Event, _Condition, etc.), because (if Guido recalls correctly) this - code pre-dates the ability to subclass extension types. It is now possible - to inherit from these classes, without having to import the private - underscored names like multiprocessing did. - -- Issue #9723: Add shlex.quote functions, to escape filenames and command - lines. - -- Issue #12603: Fix pydoc.synopsis() on files with non-negative st_mtime. - -- Issue #12514: Use try/finally to assure the timeit module restores garbage - collections when it is done. - -- Issue #12607: In subprocess, fix issue where if stdin, stdout or stderr is - given as a low fd, it gets overwritten. - -- Issue #12576: Fix urlopen behavior on sites which do not send (or obfuscates) - ``Connection: close`` header. - -- Issue #12560: Build libpython.so on OpenBSD. Patch by Stefan Sperling. - -- Issue #1813: Fix codec lookup under Turkish locales. - -- Issue #12591: Improve support of "universal newlines" in the subprocess - module: the piped streams can now be properly read from or written to. - -- Issue #12591: Allow io.TextIOWrapper to work with raw IO objects (without - a read1() method), and add a *write_through* parameter to mandate - unbuffered writes. - -- Issue #10883: Fix socket leaks in urllib.request when using FTP. - -- Issue #12592: Make Python build on OpenBSD 5 (and future major releases). - -- Issue #12372: POSIX semaphores are broken on AIX: don't use them. - -- Issue #12551: Provide a get_channel_binding() method on SSL sockets so as - to get channel binding data for the current SSL session (only the - "tls-unique" channel binding is implemented). This allows the implementation - of certain authentication mechanisms such as SCRAM-SHA-1-PLUS. Patch by - Jacek Konieczny. - -- Issue #665194: email.utils now has format_datetime and parsedate_to_datetime - functions, allowing for round tripping of RFC2822 format dates. - -- Issue #12571: Add a plat-linux3 directory mirroring the plat-linux2 - directory, so that "import DLFCN" and other similar imports work on - Linux 3.0. - -- Issue #7484: smtplib no longer puts <> around addresses in VRFY and EXPN - commands; they aren't required and in fact postfix doesn't support that form. - -- Issue #12273: Remove ast.__version__. AST changes can be accounted for by - checking sys.version_info or sys._mercurial. - -- Silence spurious "broken pipe" tracebacks when shutting down a - ProcessPoolExecutor. - -- Fix potential resource leaks in concurrent.futures.ProcessPoolExecutor - by joining all queues and processes when shutdown() is called. - -- Issue #11603: Fix a crash when __str__ is rebound as __repr__. Patch by - Andreas Stührk. - -- Issue #11321: Fix a crash with multiple imports of the _pickle module when - embedding Python. Patch by Andreas Stührk. - -- Issue #6755: Add get_wch() method to curses.window class. Patch by Iñigo - Serna. - -- Add cgi.closelog() function to close the log file. - -- Issue #12502: asyncore: fix polling loop with AF_UNIX sockets. - -- Issue #4376: ctypes now supports nested structures in an endian different than - the parent structure. Patch by Vlad Riscutia. - -- Raise ValueError when attempting to set the _CHUNK_SIZE attribute of a - TextIOWrapper to a huge value, not TypeError. - -- Issue #12504: Close file handles in a timely manner in packaging.database. - This fixes a bug with the remove (uninstall) feature on Windows. - -- Issues #12169 and #10510: Factor out code used by various packaging commands - to make HTTP POST requests, and make sure it uses CRLF. - -- Issue #12016: Multibyte CJK decoders now resynchronize faster. They only - ignore the first byte of an invalid byte sequence. For example, - b'\xff\n'.decode('gb2312', 'replace') gives '\ufffd\n' instead of '\ufffd'. - -- Issue #12459: time.sleep() now raises a ValueError if the sleep length is - negative, instead of an infinite sleep on Windows or raising an IOError on - Linux for example, to have the same behaviour on all platforms. - -- Issue #12451: pydoc: html_getfile() now uses tokenize.open() to support - Python scripts using an encoding different than UTF-8 (read the coding cookie - of the script). - -- Issue #12493: subprocess: Popen.communicate() now also handles EINTR errors - if the process has only one pipe. - -- Issue #12467: warnings: fix a race condition if a warning is emitted at - shutdown, if globals()['__file__'] is None. - -- Issue #12451: pydoc: importfile() now opens the Python script in binary mode, - instead of text mode using the locale encoding, to avoid encoding issues. - -- Issue #12451: runpy: run_path() now opens the Python script in binary mode, - instead of text mode using the locale encoding, to support other encodings - than UTF-8 (scripts using the coding cookie). - -- Issue #12451: xml.dom.pulldom: parse() now opens files in binary mode instead - of the text mode (using the locale encoding) to avoid encoding issues. - -- Issue #12147: Adjust the new-in-3.2 smtplib.send_message method for better - conformance to the RFCs: correctly handle Sender and Resent- headers. - -- Issue #12352: Fix a deadlock in multiprocessing.Heap when a block is freed by - the garbage collector while the Heap lock is held. - -- Issue #12462: time.sleep() now immediately calls the (Python) signal handler - if it is interrupted by a signal, instead of having to wait until the next - instruction. - -- Issue #12442: new shutil.disk_usage function, providing total, used and free - disk space statistics. - -- Issue #12451: The XInclude default loader of xml.etree now decodes files from - UTF-8 instead of the locale encoding if the encoding is not specified. It now - also opens XML files for the parser in binary mode instead of the text mode - to avoid encoding issues. - -- Issue #12451: doctest.debug_script() doesn't create a temporary file - anymore to avoid encoding issues. - -- Issue #12451: pydoc.synopsis() now reads the encoding cookie if available, - to read the Python script from the right encoding. - -- Issue #12451: distutils now opens the setup script in binary mode to read the - encoding cookie, instead of opening it in UTF-8. - -- Issue #9516: On Mac OS X, change Distutils to no longer globally attempt to - check or set the MACOSX_DEPLOYMENT_TARGET environment variable for the - interpreter process. This could cause failures in non-Distutils subprocesses - and was unreliable since tests or user programs could modify the interpreter - environment after Distutils set it. Instead, have Distutils set the - deployment target only in the environment of each build subprocess. It is - still possible to globally override the default by setting - MACOSX_DEPLOYMENT_TARGET before launching the interpreter; its value must be - greater or equal to the default value, the value with which the interpreter - was built. Also, implement the same handling in packaging. - -- Issue #12422: In the copy module, don't store objects that are their own copy - in the memo dict. - -- Issue #12303: Add sigwaitinfo() and sigtimedwait() to the signal module. - -- Issue #12404: Remove C89 incompatible code from mmap module. Patch by Akira - Kitada. - -- Issue #1874: email now detects and reports as a defect the presence of - any CTE other than 7bit, 8bit, or binary on a multipart. - -- Issue #12383: Fix subprocess module with env={}: don't copy the environment - variables, start with an empty environment. - -- Issue #11637: Fix support for importing packaging setup hooks from the - project directory. - -- Issue #6771: Moved the curses.wrapper function from the single-function - wrapper module into __init__, eliminating the module. Since __init__ was - already importing the function to curses.wrapper, there is no API change. - -- Issue #11584: email.header.decode_header no longer fails if the header - passed to it is a Header object, and Header/make_header no longer fail - if given binary unknown-8bit input. - -- Issue #11700: mailbox proxy object close methods can now be called multiple - times without error. - -- Issue #11767: Correct file descriptor leak in mailbox's __getitem__ method. - -- Issue #12133: AbstractHTTPHandler.do_open() of urllib.request closes the HTTP - connection if its getresponse() method fails with a socket error. Patch - written by Ezio Melotti. - -- Issue #12240: Allow multiple setup hooks in packaging's setup.cfg files. - Original patch by Erik Bray. - -- Issue #9284: Allow inspect.findsource() to find the source of doctest - functions. - -- Issue #11595: Fix assorted bugs in packaging.util.cfg_to_args, a - compatibility helper for the distutils-packaging transition. Original patch - by Erik Bray. - -- Issue #12287: In ossaudiodev, check that the device isn't closed in several - methods. - -- Issue #12009: Fixed regression in netrc file comment handling. - -- Issue #12246: Warn and fail when trying to install a third-party project from - an uninstalled Python (built in a source checkout). Original patch by - Tshepang Lekhonkhobe. - -- Issue #10694: zipfile now ignores garbage at the end of a zipfile. - -- Issue #12283: Fixed regression in smtplib quoting of leading dots in DATA. - -- Issue #10424: Argparse now includes the names of the missing required - arguments in the missing arguments error message. - -- Issue #12168: SysLogHandler now allows NUL termination to be controlled using - a new 'append_nul' attribute on the handler. - -- Issue #11583: Speed up os.path.isdir on Windows by using GetFileAttributes - instead of os.stat. - -- Issue #12021: Make mmap's read() method argument optional. Patch by Petri - Lehtinen. - -- Issue #9205: concurrent.futures.ProcessPoolExecutor now detects killed - children and raises BrokenProcessPool in such a situation. Previously it - would reliably freeze/deadlock. - -- Issue #12040: Expose a new attribute ``sentinel`` on instances of - ``multiprocessing.Process``. Also, fix Process.join() to not use polling - anymore, when given a timeout. - -- Issue #11893: Remove obsolete internal wrapper class ``SSLFakeFile`` in the - smtplib module. Patch by Catalin Iacob. - -- Issue #12080: Fix a Decimal.power() case that took an unreasonably long time - to compute. - -- Issue #12221: Remove __version__ attributes from pyexpat, pickle, tarfile, - pydoc, tkinter, and xml.parsers.expat. This were useless version constants - left over from the Mercurial transition - -- Named tuples now work correctly with vars(). - -- Issue #12085: Fix an attribute error in subprocess.Popen destructor if the - constructor has failed, e.g. because of an undeclared keyword argument. Patch - written by Oleg Oshmyan. - -- Issue #12028: Make threading._get_ident() public, rename it to - threading.get_ident() and document it. This function was already used using - _thread.get_ident(). - -- Issue #12171: IncrementalEncoder.reset() of CJK codecs (multibytecodec) calls - encreset() instead of decreset(). - -- Issue #12218: Removed wsgiref.egg-info. - -- Issue #12196: Add pipe2() to the os module. - -- Issue #985064: Make plistlib more resilient to faulty input plists. - Patch by Mher Movsisyan. - -- Issue #1625: BZ2File and bz2.decompress() now support multi-stream files. - Initial patch by Nir Aides. - -- Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. - -- Issue #12175: FileIO.readall() now only reads the file position and size - once. - -- Issue #12175: RawIOBase.readall() now returns None if read() returns None. - -- Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError - if the file is closed. - -- Issue #11109: New service_action method for BaseServer, used by ForkingMixin - class for cleanup. Initial Patch by Justin Warkentin. - -- Issue #12045: Avoid duplicate execution of command in - ctypes.util._get_soname(). Patch by Sijin Joseph. - -- Issue #10818: Remove the Tk GUI and the serve() function of the pydoc module, - pydoc -g has been deprecated in Python 3.2 and it has a new enhanced web - server. - -- Issue #1441530: In imaplib, read the data in one chunk to speed up large - reads and simplify code. - -- Issue #12070: Fix the Makefile parser of the sysconfig module to handle - correctly references to "bogus variable" (e.g. "prefix=$/opt/python"). - -- Issue #12100: Don't reset incremental encoders of CJK codecs at each call to - their encode() method anymore, but continue to call the reset() method if the - final argument is True. - -- Issue #12049: Add RAND_bytes() and RAND_pseudo_bytes() functions to the ssl - module. - -- Issue #6501: os.device_encoding() returns None on Windows if the application - has no console. - -- Issue #12105: Add O_CLOEXEC to the os module. - -- Issue #12079: Decimal('Infinity').fma(Decimal('0'), (3.91224318126786e+19+0j)) - now raises TypeError (reflecting the invalid type of the 3rd argument) rather - than Decimal.InvalidOperation. - -- Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore - to be able to unload the module. - -- Add the packaging module, an improved fork of distutils (also known as - distutils2). - -- Issue #12065: connect_ex() on an SSL socket now returns the original errno - when the socket's timeout expires (it used to return None). - -- Issue #8809: The SMTP_SSL constructor and SMTP.starttls() now support - passing a ``context`` argument pointing to an ssl.SSLContext instance. - Patch by Kasun Herath. - -- Issue #9516: Issue #9516: avoid errors in sysconfig when MACOSX_DEPLOYMENT_TARGET - is set in shell. - -- Issue #8650: Make zlib module 64-bit clean. compress(), decompress() and - their incremental counterparts now raise OverflowError if given an input - larger than 4GB, instead of silently truncating the input and returning - an incorrect result. - -- Issue #12050: zlib.decompressobj().decompress() now clears the unconsumed_tail - attribute when called without a max_length argument. - -- Issue #12062: Fix a flushing bug when doing a certain type of I/O sequence - on a file opened in read+write mode (namely: reading, seeking a bit forward, - writing, then seeking before the previous write but still within buffered - data, and writing again). - -- Issue #9971: Write an optimized implementation of BufferedReader.readinto(). - Patch by John O'Connor. - -- Issue #11799: urllib.request Authentication Handlers will raise a ValueError - when presented with an unsupported Authentication Scheme. Patch contributed - by Yuval Greenfield. - -- Issue #10419, #6011: build_scripts command of distutils handles correctly - non-ASCII path (path to the Python executable). Open and write the script in - binary mode, but ensure that the shebang is decodable from UTF-8 and from the - encoding of the script. - -- Issue #8498: In socket.accept(), allow specifying 0 as a backlog value in - order to accept exactly one connection. Patch by Daniel Evers. - -- Issue #12011: signal.signal() and signal.siginterrupt() raise an OSError, - instead of a RuntimeError: OSError has an errno attribute. - -- Issue #3709: add a flush_headers method to BaseHTTPRequestHandler, which - manages the sending of headers to output stream and flushing the internal - headers buffer. Patch contribution by Andrew Schaaf - -- Issue #11743: Rewrite multiprocessing connection classes in pure Python. - -- Issue #11164: Stop trying to use _xmlplus in the xml module. - -- Issue #11888: Add log2 function to math module. Patch written by Mark - Dickinson. - -- Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional. - -- Issue #8407: The signal handler writes the signal number as a single byte - instead of a nul byte into the wakeup file descriptor. So it is possible to - wait more than one signal and know which signals were raised. - -- Issue #8407: Add pthread_kill(), sigpending() and sigwait() functions to the - signal module. - -- Issue #11927: SMTP_SSL now uses port 465 by default as documented. Patch - by Kasun Herath. - -- Issue #12002: ftplib's abort() method raises TypeError. - -- Issue #11916: Add a number of MacOSX specific definitions to the errno module. - Patch by Pierre Carrier. - -- Issue #11999: fixed sporadic sync failure mailbox.Maildir due to its trying to - detect mtime changes by comparing to the system clock instead of to the - previous value of the mtime. - -- Issue #11072: added MLSD command (RFC-3659) support to ftplib. - -- Issue #8808: The IMAP4_SSL constructor now allows passing an SSLContext - parameter to control parameters of the secure channel. Patch by Sijin - Joseph. - -- ntpath.samefile failed to notice that "a.txt" and "A.TXT" refer to the same - file on Windows XP. As noticed in issue #10684. - -- Issue #12000: When a SSL certificate has a subjectAltName without any - dNSName entry, ssl.match_hostname() should use the subject's commonName. - Patch by Nicolas Bareil. - -- Issue #10775: assertRaises, assertRaisesRegex, assertWarns, and - assertWarnsRegex now accept a keyword argument 'msg' when used as context - managers. Initial patch by Winston Ewert. - -- Issue #10684: shutil.move used to delete a folder on case insensitive - filesystems when the source and destination name where the same except - for the case. - -- Issue #11647: objects created using contextlib.contextmanager now support - more than one call to the function when used as a decorator. Initial patch - by Ysj Ray. - -- Issue #11930: Removed deprecated time.accept2dyear variable. - Removed year >= 1000 restriction from datetime.strftime. - -- logging: don't define QueueListener if Python has no thread support. - -- functools.cmp_to_key() now works with collections.Hashable(). - -- Issue #11277: mmap.mmap() calls fcntl(fd, F_FULLFSYNC) on Mac OS X to get - around a mmap bug with sparse files. Patch written by Steffen Daode Nurpmeso. - -- Issue #8407: Add signal.pthread_sigmask() function to fetch and/or change the - signal mask of the calling thread. - -- Issue #11858: configparser.ExtendedInterpolation expected lower-case section - names. - -- Issue #11324: ConfigParser(interpolation=None) now works correctly. - -- Issue #11811: ssl.get_server_certificate() is now IPv6-compatible. Patch - by Charles-François Natali. - -- Issue #11763: don't use difflib in TestCase.assertMultiLineEqual if the - strings are too long. - -- Issue #11236: getpass.getpass responds to ctrl-c or ctrl-z on terminal. - -- Issue #11856: Speed up parsing of JSON numbers. - -- Issue #11005: threading.RLock()._release_save() raises a RuntimeError if the - lock was not acquired. - -- Issue #11258: Speed up ctypes.util.find_library() under Linux by a factor - of 5 to 10. Initial patch by Jonas H. - -- Issue #11382: Trivial system calls, such as dup() or pipe(), needn't - release the GIL. Patch by Charles-François Natali. - -- Issue #11223: Add threading._info() function providing informations about - the thread implementation. - -- Issue #11731: simplify/enhance email parser/generator API by introducing - policy objects. - -- Issue #11768: The signal handler of the signal module only calls - Py_AddPendingCall() for the first signal to fix a deadlock on reentrant or - parallel calls. PyErr_SetInterrupt() writes also into the wake up file. - -- Issue #11492: fix several issues with header folding in the email package. - -- Issue #11852: Add missing imports and update tests. - -- Issue #11875: collections.OrderedDict's __reduce__ was temporarily - mutating the object instead of just working on a copy. - -- Issue #11467: Fix urlparse behavior when handling urls which contains scheme - specific part only digits. Patch by Santoso Wijaya. - -- collections.Counter().copy() now works correctly for subclasses. - -- Issue #11474: Fix the bug with url2pathname() handling of '/C|/' on Windows. - Patch by Santoso Wijaya. - -- Issue #11684: complete email.parser bytes API by adding BytesHeaderParser. - -- The bz2 module now handles 4GiB+ input buffers correctly. - -- Issue #9233: Fix json.loads('{}') to return a dict (instead of a list), when - _json is not available. - -- Issue #11830: Remove unnecessary introspection code in the decimal module. - -- Issue #11703: urllib2.geturl() does not return correct url when the original - url contains #fragment. - -- Issue #10019: Fixed regression in json module where an indent of 0 stopped - adding newlines and acted instead like 'None'. - -- Issue #11186: pydoc ignores a module if its name contains a surrogate - character in the index of modules. - -- Issue #11815: Use a light-weight SimpleQueue for the result queue in - concurrent.futures.ProcessPoolExecutor. - -- Issue #5162: Treat services like frozen executables to allow child spawning - from multiprocessing.forking on Windows. - -- logging.basicConfig now supports an optional 'handlers' argument taking an - iterable of handlers to be added to the root logger. Additional parameter - checks were also added to basicConfig. - -- Issue #11814: Fix likely typo in multiprocessing.Pool._terminate(). - -- Issue #11747: Fix range formatting in difflib.context_diff() and - difflib.unified_diff(). - -- Issue #8428: Fix a race condition in multiprocessing.Pool when terminating - worker processes: new processes would be spawned while the pool is being - shut down. Patch by Charles-François Natali. - -- Issue #2650: re.escape() no longer escapes the '_'. - -- Issue #11757: select.select() now raises ValueError when a negative timeout - is passed (previously, a select.error with EINVAL would be raised). Patch - by Charles-François Natali. - -- Issue #7311: fix html.parser to accept non-ASCII attribute values. - -- Issue #11605: email.parser.BytesFeedParser was incorrectly converting - multipart subparts with an 8-bit CTE into unicode instead of preserving the - bytes. - -- Issue #1690608: email.util.formataddr is now RFC 2047 aware: it now has a - charset parameter that defaults to utf-8 and is used as the charset for RFC - 2047 encoding when the realname contains non-ASCII characters. - -- Issue #10963: Ensure that subprocess.communicate() never raises EPIPE. - -- Issue #10791: Implement missing method GzipFile.read1(), allowing GzipFile - to be wrapped in a TextIOWrapper. Patch by Nadeem Vawda. - -- Issue #11707: Added a fast C version of functools.cmp_to_key(). - Patch by Filip Gruszczyński. - -- Issue #11688: Add sqlite3.Connection.set_trace_callback(). Patch by - Torsten Landschoff. - -- Issue #11746: Fix SSLContext.load_cert_chain() to accept elliptic curve - private keys. - -- Issue #5863: Rewrite BZ2File in pure Python, and allow it to accept - file-like objects using a new ``fileobj`` constructor argument. Patch by - Nadeem Vawda. - -- unittest.TestCase.assertSameElements has been removed. - -- sys.getfilesystemencoding() raises a RuntimeError if initfsencoding() was not - called yet: detect bootstrap (startup) issues earlier. - -- Issue #11393: Add the new faulthandler module. - -- Issue #11618: Fix the timeout logic in threading.Lock.acquire() under Windows. - -- Removed the 'strict' argument to email.parser.Parser, which has been - deprecated since Python 2.4. - -- Issue #11256: Fix inspect.getcallargs on functions that take only keyword - arguments. - -- Issue #11696: Fix ID generation in msilib. - -- itertools.accumulate now supports an optional *func* argument for - a user-supplied binary function. - -- Issue #11692: Remove unnecessary demo functions in subprocess module. - -- Issue #9696: Fix exception incorrectly raised by xdrlib.Packer.pack_int when - trying to pack a negative (in-range) integer. - -- Issue #11675: multiprocessing.[Raw]Array objects created from an integer size - are now zeroed on creation. This matches the behaviour specified by the - documentation. - -- Issue #7639: Fix short file name generation in bdist_msi - -- Issue #11635: Don't use polling in worker threads and processes launched by - concurrent.futures. - -- Issue #5845: Automatically read readline configuration to enable completion - in interactive mode. - -- Issue #6811: Allow importlib to change a code object's co_filename attribute - to match the path to where the source code currently is, not where the code - object originally came from. - -- Issue #8754: Have importlib use the repr of a module name in error messages. - -- Issue #11591: Prevent "import site" from modifying sys.path when python - was started with -S. - -- collections.namedtuple() now adds a _source attribute to the generated - class. This make the source more accessible than the outdated - "verbose" option which prints to stdout but doesn't make the source - string available. - -- Issue #11371: Mark getopt error messages as localizable. Patch by Filip - Gruszczyński. - -- Issue #11333: Add __slots__ to collections ABCs. - -- Issue #11628: cmp_to_key generated class should use __slots__. - -- Issue #11666: let help() display named tuple attributes and methods - that start with a leading underscore. - -- Issue #11662: Make urllib and urllib2 ignore redirections if the - scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). - -- Issue #5537: Fix time2isoz() and time2netscape() functions of - httplib.cookiejar for expiration year greater than 2038 on 32-bit systems. - -- Issue #4391: Use proper gettext plural forms in optparse. - -- Issue #11127: Raise a TypeError when trying to pickle a socket object. - -- Issue #11563: ``Connection: close`` header is sent by requests using URLOpener - class which helps in closing of sockets after connection is over. Patch - contributions by Jeff McNeil and Nadeem Vawda. - -- Issue #11459: A ``bufsize`` value of 0 in subprocess.Popen() really creates - unbuffered pipes, such that select() works properly on them. - -- Issue #5421: Fix misleading error message when one of socket.sendto()'s - arguments has the wrong type. Patch by Nikita Vetoshkin. - -- Issue #10812: Add some extra posix functions to the os module. - -- Issue #10979: unittest stdout buffering now works with class and module - setup and teardown. - -- Issue #11243: fix the parameter querying methods of Message to work if - the headers contain un-encoded non-ASCII data. - -- Issue #11401: fix handling of headers with no value; this fixes a regression - relative to Python2 and the result is now the same as it was in Python2. - -- Issue #9298: base64 bodies weren't being folded to line lengths less than 78, - which was a regression relative to Python2. Unlike Python2, the last line - of the folded body now ends with a carriage return. - -- Issue #11560: shutil.unpack_archive now correctly handles the format - parameter. Patch by Evan Dandrea. - -- Issue #5870: Add `subprocess.DEVNULL` constant. - -- Issue #11133: fix two cases where inspect.getattr_static can trigger code - execution. Patch by Andreas Stührk. - -- Issue #11569: use absolute path to the sysctl command in multiprocessing to - ensure that it will be found regardless of the shell PATH. This ensures - that multiprocessing.cpu_count works on default installs of MacOSX. - -- Issue #11501: distutils.archive_utils.make_zipfile no longer fails if zlib is - not installed. Instead, the zipfile.ZIP_STORED compression is used to create - the ZipFile. Patch by Natalia B. Bidart. - -- Issue #11289: `smtp.SMTP` class is now a context manager so it can be used - in a `with` statement. Contributed by Giampaolo Rodola. - -- Issue #11554: Fixed support for Japanese codecs; previously the body output - encoding was not done if euc-jp or shift-jis was specified as the charset. - -- Issue #11407: `TestCase.run` returns the result object used or created. - Contributed by Janathan Hartley. - -- Issue #11500: Fixed a bug in the OS X proxy bypass code for fully qualified - IP addresses in the proxy exception list. - -- Issue #11491: dbm.error is no longer raised when dbm.open is called with - the "n" as the flag argument and the file exists. The behavior matches - the documentation and general logic. - -- Issue #1162477: Postel Principle adjustment to email date parsing: handle the - fact that some non-compliant MUAs use '.' instead of ':' in time specs. - -- Issue #11131: Fix sign of zero in decimal.Decimal plus and minus - operations when the rounding mode is ROUND_FLOOR. - -- Issue #9935: Speed up pickling of instances of user-defined classes. - -- Issue #5622: Fix curses.wrapper to raise correct exception if curses - initialization fails. - -- Issue #11408: In threading.Lock.acquire(), only call gettimeofday() when - really necessary. Patch by Charles-François Natali. - -- Issue #11391: Writing to a mmap object created with - ``mmap.PROT_READ|mmap.PROT_EXEC`` would segfault instead of raising a - TypeError. Patch by Charles-François Natali. - -- Issue #9795: add context management protocol support for nntplib.NNTP class. - -- Issue #11306: mailbox in certain cases adapts to an inability to open - certain files in read-write mode. Previously it detected this by - checking for EACCES, now it also checks for EROFS. - -- Issue #11265: asyncore now correctly handles EPIPE, EBADF and EAGAIN errors - on accept(), send() and recv(). - -- Issue #11377: Deprecate platform.popen() and reimplement it with os.popen(). - -- Issue #8513: On UNIX, subprocess supports bytes command string. - -- Issue #10866: Add socket.sethostname(). Initial patch by Ross Lagerwall. - -- Issue #11140: Lock.release() now raises a RuntimeError when attempting - to release an unacquired lock, as claimed in the threading documentation. - The _thread.error exception is now an alias of RuntimeError. Patch by - Filip Gruszczyński. Patch for _dummy_thread by Aymeric Augustin. - -- Issue #8594: ftplib now provides a source_address parameter to specify which - (address, port) to bind to before connecting. - -- Issue #11326: Add the missing connect_ex() implementation for SSL sockets, - and make it work for non-blocking connects. - -- Issue #11297: Add collections.ChainMap(). - -- Issue #10755: Add the posix.flistdir() function. Patch by Ross Lagerwall. - -- Issue #4761: Add the ``*at()`` family of functions (openat(), etc.) to the - posix module. Patch by Ross Lagerwall. - -- Issue #7322: Trying to read from a socket's file-like object after a timeout - occurred now raises an error instead of silently losing data. - -- Issue #11291: poplib.POP no longer suppresses errors on quit(). - -- Issue #11177: asyncore's create_socket() arguments can now be omitted. - -- Issue #6064: Add a ``daemon`` keyword argument to the threading.Thread - and multiprocessing.Process constructors in order to override the - default behaviour of inheriting the daemonic property from the current - thread/process. - -- Issue #10956: Buffered I/O classes retry reading or writing after a signal - has arrived and the handler returned successfully. - -- Issue #10784: New os.getpriority() and os.setpriority() functions. - -- Issue #11114: Fix catastrophic performance of tell() on text files (up - to 1000x faster in some cases). It is still one to two order of magnitudes - slower than binary tell(). - -- Issue #10882: Add os.sendfile function. - -- Issue #10868: Allow usage of the register method of an ABC as a class - decorator. - -- Issue #11224: Fixed a regression in tarfile that affected the file-like - objects returned by TarFile.extractfile() regarding performance, memory - consumption and failures with the stream interface. - -- Issue #10924: Adding salt and Modular Crypt Format to crypt library. - Moved old C wrapper to _crypt, and added a Python wrapper with - enhanced salt generation and simpler API for password generation. - -- Issue #11074: Make 'tokenize' so it can be reloaded. - -- Issue #11085: Moved collections abstract base classes into a separate - module called collections.abc, following the pattern used by importlib.abc. - For backwards compatibility, the names are imported into the collections - module. - -- Issue #4681: Allow mmap() to work on file sizes and offsets larger than - 4GB, even on 32-bit builds. Initial patch by Ross Lagerwall, adapted for - 32-bit Windows. - -- Issue #11169: compileall module uses repr() to format filenames and paths to - escape surrogate characters and show spaces. - -- Issue #11089: Fix performance issue limiting the use of ConfigParser() - with large config files. - -- Issue #10276: Fix the results of zlib.crc32() and zlib.adler32() on buffers - larger than 4GB. Patch by Nadeem Vawda. - -- Issue #11388: Added a clear() method to MutableSequence - -- Issue #11174: Add argparse.MetavarTypeHelpFormatter, which uses type names - for the names of optional and positional arguments in help messages. - -- Issue #9348: Raise an early error if argparse nargs and metavar don't match. - -- Issue #9026: Fix order of argparse sub-commands in help messages. - -- Issue #9347: Fix formatting for tuples in argparse type= error messages. - -- Issue #12191: Added shutil.chown() to change user and/or group owner of a - given path also specifying their names. - -- Issue #13988: The _elementtree accelerator is used whenever available. - Now xml.etree.cElementTree becomes a deprecated alias to ElementTree. - -Build ------ - -- Issue #6807: Run msisupport.mak earlier. - -- Issue #10580: Minor grammar change in Windows installer. - -- Issue #13326: Clean __pycache__ directories correctly on OpenBSD. - -- PEP 393: the configure option --with-wide-unicode is removed. - -- Issue #12852: Set _XOPEN_SOURCE to 700, instead of 600, to get POSIX 2008 - functions on OpenBSD (e.g. fdopendir). - -- Issue #11863: Remove support for legacy systems deprecated in Python 3.2 - (following PEP 11). These systems are systems using Mach C Threads, - SunOS lightweight processes, GNU pth threads and IRIX threads. - -- Issue #8746: Correct faulty configure checks so that os.chflags() and - os.lchflags() are once again built on systems that support these - functions (BSD and OS X). Also add new stat file flags for OS X - (UF_HIDDEN and UF_COMPRESSED). - -- Issue #10645: Installing Python no longer creates a - Python-X.Y.Z-pyX.Y.egg-info file in the lib-dynload directory. - -- Do not accidentally include the directory containing sqlite.h twice when - building sqlite3. - -- Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds, - ensure "make install" creates symlinks in --prefix bin for the "-32" - files in the framework bin directory like the installer does. - -- Issue #11347: Use --no-as-needed when linking libpython3.so. - -- Issue #11411: Fix 'make DESTDIR=' with a relative destination. - -- Issue #11268: Prevent Mac OS X Installer failure if Documentation - package had previously been installed. - -- Issue #11495: OSF support is eliminated. It was deprecated in Python 3.2. - -IDLE ----- - -- Issue #14409: IDLE now properly executes commands in the Shell window - when it cannot read the normal config files on startup and - has to use the built-in default key bindings. - There was previously a bug in one of the defaults. - -- IDLE can be launched as python -m idlelib - -- Issue #3573: IDLE hangs when passing invalid command line args - (directory(ies) instead of file(s)) (Patch by Guilherme Polo) - -- Issue #14200: IDLE shell crash on printing non-BMP unicode character. - -- Issue #5219: Prevent event handler cascade in IDLE. - -- Issue #964437: Make IDLE help window non-modal. - Patch by Guilherme Polo and Roger Serwy. - -- Issue #13933: IDLE auto-complete did not work with some imported - module, like hashlib. (Patch by Roger Serwy) - -- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. - Original patches by Marco Scataglini and Roger Serwy. - -- Issue #4625: If IDLE cannot write to its recent file or breakpoint files, - display a message popup and continue rather than crash. Original patch by - Roger Serwy. - -- Issue #8641: Update IDLE 3 syntax coloring to recognize b".." and not u"..". - Patch by Tal Einat. - -- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. - (Patch by Roger Serwy) - -- Issue #9871: Prevent IDLE 3 crash when given byte stings - with invalid hex escape sequences, like b'\x0'. - (Original patch by Claudiu Popa.) - -- Issue #12636: IDLE reads the coding cookie when executing a Python script. - -- Issue #12540: Prevent zombie IDLE processes on Windows due to changes - in os.kill(). - -- Issue #12590: IDLE editor window now always displays the first line - when opening a long file. With Tk 8.5, the first line was hidden. - -- Issue #11088: don't crash when using F5 to run a script in IDLE on MacOSX - with Tk 8.5. - -- Issue #1028: Tk returns invalid Unicode null in %A: UnicodeDecodeError. - With Tk < 8.5 _tkinter.c:PythonCmd() raised UnicodeDecodeError, caused - IDLE to exit. Converted to valid Unicode null in PythonCmd(). - -- Issue #11718: IDLE's open module dialog couldn't find the __init__.py - file in a package. - -Tools/Demos ------------ - -- Issue #14053: patchcheck.py ("make patchcheck") now works with MQ patches. - Patch by Francisco Martín Brugué. - -- Issue #13930: 2to3 is now able to write its converted output files to another - directory tree as well as copying unchanged files and altering the file - suffix. See its new -o, -W and --add-suffix options. This makes it more - useful in many automated code translation workflows. - -- Issue #13628: python-gdb.py is now able to retrieve more frames in the Python - traceback if Python is optimized. - -- Issue #11996: libpython (gdb), replace "py-bt" command by "py-bt-full" and - add a smarter "py-bt" command printing a classic Python traceback. - -- Issue #11179: Make ccbench work under Python 3.1 and 2.7 again. - -- Issue #10639: reindent.py no longer converts newlines and will raise - an error if attempting to convert a file with mixed newlines. - "--newline" option added to specify new line character. - -Extension Modules ------------------ - -- Issue #16847: Fixed improper use of _PyUnicode_CheckConsistency() in - non-pydebug builds. Several extension modules now compile cleanly when - assert()s are enabled in standard builds (-DDEBUG flag). - -- Issue #13840: The error message produced by ctypes.create_string_buffer - when given a Unicode string has been fixed. - -- Issue #9975: socket: Fix incorrect use of flowinfo and scope_id. Patch by - Vilmos Nebehaj. - -- Issue #7777: socket: Add Reliable Datagram Sockets (PF_RDS) support. - -- Issue #13159: FileIO and BZ2Compressor/BZ2Decompressor now use a linear-time - buffer growth strategy instead of a quadratic-time one. - -- Issue #10141: socket: Add SocketCAN (PF_CAN) support. Initial patch by - Matthias Fuchs, updated by Tiago Gonçalves. - -- Issue #13070: Fix a crash when a TextIOWrapper caught in a reference cycle - would be finalized after the reference to its underlying BufferedRWPair's - writer got cleared by the GC. - -- Issue #12881: ctypes: Fix segfault with large structure field names. - -- Issue #13058: ossaudiodev: fix a file descriptor leak on error. Patch by - Thomas Jarosch. - -- Issue #13013: ctypes: Fix a reference leak in PyCArrayType_from_ctype. - Thanks to Suman Saha for finding the bug and providing a patch. - -- Issue #13022: Fix: _multiprocessing.recvfd() doesn't check that - file descriptor was actually received. - -- Issue #1172711: Add 'long long' support to the array module. - Initial patch by Oren Tirosh and Hirokazu Yamamoto. - -- Issue #12483: ctypes: Fix a crash when the destruction of a callback - object triggers the garbage collector. - -- Issue #12950: Fix passing file descriptors in multiprocessing, under - OpenIndiana/Illumos. - -- Issue #12764: Fix a crash in ctypes when the name of a Structure field is not - a string. - -- Issue #11241: subclasses of ctypes.Array can now be subclassed. - -- Issue #9651: Fix a crash when ctypes.create_string_buffer(0) was passed to - some functions like file.write(). - -- Issue #10309: Define _GNU_SOURCE so that mremap() gets the proper - signature. Without this, architectures where sizeof void* != sizeof int are - broken. Patch given by Hallvard B Furuseth. - -- Issue #12051: Fix segfault in json.dumps() while encoding highly-nested - objects using the C accelerations. - -- Issue #12017: Fix segfault in json.loads() while decoding highly-nested - objects using the C accelerations. - -- Issue #1838: Prevent segfault in ctypes, when _as_parameter_ on a class is set - to an instance of the class. - -Tests ------ - -- Issue #13125: Silence spurious test_lib2to3 output when in non-verbose mode. - Patch by Mikhail Novikov. - -- Issue #13447: Add a test file to host regression tests for bugs in the - scripts found in the Tools directory. - -- Issue #10881: Fix test_site failure with OS X framework builds. - -- Issue #13901: Prevent test_distutils failures on OS X with --enable-shared. - -- Issue #13862: Fix spurious failure in test_zlib due to runtime/compile time - minor versions not matching. - -- Issue #12804: Fix test_socket and test_urllib2net failures when running tests - on a system without internet access. - -- Issue #13726: Fix the ambiguous -S flag in regrtest. It is -o/--slow for slow - tests. - -- Issue #11659: Fix ResourceWarning in test_subprocess introduced by #11459. - Patch by Ben Hayden. - -- Issue #11577: fix ResourceWarning triggered by improved binhex test coverage - -- Issue #11509: Significantly increase test coverage of fileinput. - Patch by Denver Coneybeare at PyCon 2011 Sprints. - -- Issue #11689: Fix a variable scoping error in an sqlite3 test - -- Issue #13786: Remove unimplemented 'trace' long option from regrtest.py. - -- Issue #13725: Fix regrtest to recognize the documented -d flag. - Patch by Erno Tukia. - -- Issue #13304: Skip test case if user site-packages disabled (-s or - PYTHONNOUSERSITE). (Patch by Carl Meyer) - -- Issue #5661: Add a test for ECONNRESET/EPIPE handling to test_asyncore. Patch - by Xavier de Gaye. - -- Issue #13218: Fix test_ssl failures on Debian/Ubuntu. - -- Re-enable lib2to3's test_parser.py tests, though with an expected failure - (see issue 13125). - -- Issue #12656: Add tests for IPv6 and Unix sockets to test_asyncore. - -- Issue #6484: Add unit tests for mailcap module (patch by Gregory Nofi) - -- Issue #11651: Improve the Makefile test targets to run more of the test suite - more quickly. The --multiprocess option is now enabled by default, reducing - the amount of time needed to run the tests. "make test" and "make quicktest" - now include some resource-intensive tests, but no longer run the test suite - twice to check for bugs in .pyc generation. Tools/scripts/run_test.py provides - an easy platform-independent way to run test suite with sensible defaults. - -- Issue #12331: The test suite for the packaging module can now run from an - installed Python. - -- Issue #12331: The test suite for lib2to3 can now run from an installed - Python. - -- Issue #12626: In regrtest, allow filtering tests using a glob filter - with the ``-m`` (or ``--match``) option. This works with all test cases - using the unittest module. This is useful with long test suites - such as test_io or test_subprocess. - -- Issue #12624: It is now possible to fail after the first failure when - running in verbose mode (``-v`` or ``-W``), by using the ``--failfast`` - (or ``-G``) option to regrtest. This is useful with long test suites - such as test_io or test_subprocess. - -- Issue #12587: Correct faulty test file and reference in test_tokenize. - (Patch by Robert Xiao) - -- Issue #12573: Add resource checks for dangling Thread and Process objects. - -- Issue #12549: Correct test_platform to not fail when OS X returns 'x86_64' - as the processor type on some Mac systems. - -- Skip network tests when getaddrinfo() returns EAI_AGAIN, meaning a temporary - failure in name resolution. - -- Issue #11812: Solve transient socket failure to connect to 'localhost' - in test_telnetlib.py. - -- Solved a potential deadlock in test_telnetlib.py. Related to issue #11812. - -- Avoid failing in test_robotparser when mueblesmoraleda.com is flaky and - an overzealous DNS service (e.g. OpenDNS) redirects to a placeholder - Web site. - -- Avoid failing in test_urllibnet.test_bad_address when some overzealous - DNS service (e.g. OpenDNS) resolves a non-existent domain name. The test - is now skipped instead. - -- Issue #12440: When testing whether some bits in SSLContext.options can be - reset, check the version of the OpenSSL headers Python was compiled against, - rather than the runtime version of the OpenSSL library. - -- Issue #11512: Add a test suite for the cgitb module. Patch by Robbie Clemons. - -- Issue #12497: Install test/data to prevent failures of the various codecmaps - tests. - -- Issue #12496: Install test/capath directory to prevent test_connect_capath - testcase failure in test_ssl. - -- Issue #12469: Run wakeup and pending signal tests in a subprocess to run the - test in a fresh process with only one thread and to not change signal - handling of the parent process. - -- Issue #8716: Avoid crashes caused by Aqua Tk on OSX when attempting to run - test_tk or test_ttk_guionly under a username that is not currently logged - in to the console windowserver (as may be the case under buildbot or ssh). - -- Issue #12407: Explicitly skip test_capi.EmbeddingTest under Windows. - -- Issue #12400: regrtest -W doesn't rerun the tests twice anymore, but captures - the output and displays it on failure instead. regrtest -v doesn't print the - error twice anymore if there is only one error. - -- Issue #12141: Install copies of template C module file so that - test_build_ext of test_distutils and test_command_build_ext of - test_packaging are no longer silently skipped when - run outside of a build directory. - -- Issue #8746: Add additional tests for os.chflags() and os.lchflags(). - Patch by Garrett Cooper. - -- Issue #10736: Fix test_ttk test_widgets failures with Cocoa Tk 8.5.9 - 2.8 + on Mac OS X. (Patch by Ronald Oussoren) - -- Issue #12057: Add tests for ISO 2022 codecs (iso2022_jp, iso2022_jp_2, - iso2022_kr). - -- Issue #12096: Fix a race condition in test_threading.test_waitfor(). Patch - written by Charles-François Natali. - -- Issue #11614: import __hello__ prints "Hello World!". Patch written by - Andreas Stührk. - -- Issue #5723: Improve json tests to be executed with and without accelerations. - -- Issue #12041: Make test_wait3 more robust. - -- Issue #11873: Change regex in test_compileall to fix occasional failures when - the randomly generated temporary path happened to match the regex. - -- Issue #11958: Fix FTP tests for IPv6, bind to "::1" instead of "localhost". - Patch written by Charles-Francois Natali. - -- Issue #8407, #11859: Fix tests of test_io using threads and an alarm: use - pthread_sigmask() to ensure that the SIGALRM signal is received by the main - thread. - -- Issue #11811: Factor out detection of IPv6 support on the current host - and make it available as ``test.support.IPV6_ENABLED``. Patch by - Charles-François Natali. - -- Issue #10914: Add a minimal embedding test to test_capi. - -- Issue #11223: Skip test_lock_acquire_interruption() and - test_rlock_acquire_interruption() of test_threadsignals if a thread lock is - implemented using a POSIX mutex and a POSIX condition variable. A POSIX - condition variable cannot be interrupted by a signal (e.g. on Linux, the - futex system call is restarted). - -- Issue #11790: Fix sporadic failures in test_multiprocessing.WithProcessesTestCondition. - -- Fix possible "file already exists" error when running the tests in parallel. - -- Issue #11719: Fix message about unexpected test_msilib skip on non-Windows - platforms. Patch by Nadeem Vawda. - -- Issue #11727: Add a --timeout option to regrtest: if a test takes more than - TIMEOUT seconds, dumps the traceback of all threads and exits. - -- Issue #11653: fix -W with -j in regrtest. - -- The email test suite now lives in the Lib/test/test_email package. The test - harness code has also been modernized to allow use of new unittest features. - -- regrtest now discovers test packages as well as test modules. - -- Issue #11577: improve test coverage of binhex.py. Patch by Arkady Koplyarov. - -- New test_crashers added to exercise the scripts in the Lib/test/crashers - directory and confirm they fail as expected - -- Issue #11578: added test for the timeit module. Patch by Michael Henry. - -- Issue #11503: improve test coverage of posixpath.py. Patch by Evan Dandrea. - -- Issue #11505: improves test coverage of string.py, increases granularity of - string.Formatter tests. Initial patch by Alicia Arlen. - -- Issue #11548: Improve test coverage of the shutil module. Patch by - Evan Dandrea. - -- Issue #11554: Reactivated test_email_codecs. - -- Issue #11505: improves test coverage of string.py. Patch by Alicia - Arlen - -- Issue #11490: test_subprocess.test_leaking_fds_on_error no longer gives a - false positive if the last directory in the path is inaccessible. - -- Issue #11223: Fix test_threadsignals to fail, not hang, when the - non-semaphore implementation of locks is used under POSIX. - -- Issue #10911: Add tests on CGI with non-ASCII characters. Patch written by - Pierre Quentel. - -- Issue #9931: Fix hangs in GUI tests under Windows in certain conditions. - Patch by Hirokazu Yamamoto. - -- Issue #10512: Properly close sockets under test.test_cgi. - -- Issue #10992: Make tests pass under coverage. - -- Issue #10826: Prevent sporadic failure in test_subprocess on Solaris due - to open door files. - -- Issue #10990: Prevent tests from clobbering a set trace function. - -C-API ------ - -- Issue #13452: PyUnicode_EncodeDecimal() doesn't support error handlers - different than "strict" anymore. The caller was unable to compute the - size of the output buffer: it depends on the error handler. - -- Issue #13560: Add PyUnicode_DecodeLocale(), PyUnicode_DecodeLocaleAndSize() - and PyUnicode_EncodeLocale() functions to the C API to decode/encode from/to - the current locale encoding. - -- Issue #10831: PyUnicode_FromFormat() supports %li, %lli and %zi formats. - -- Issue #11246: Fix PyUnicode_FromFormat("%V") to decode the byte string from - UTF-8 (with replace error handler) instead of ISO-8859-1 (in strict mode). - Patch written by Ray Allen. - -- Issue #10830: Fix PyUnicode_FromFormatV("%c") for non-BMP characters on - narrow build. - -- Add PyObject_GenericGetDict and PyObject_GeneriSetDict. They are generic - implementations for the getter and setter of a ``__dict__`` descriptor of C - types. - -- Issue #13727: Add 3 macros to access PyDateTime_Delta members: - PyDateTime_DELTA_GET_DAYS, PyDateTime_DELTA_GET_SECONDS, - PyDateTime_DELTA_GET_MICROSECONDS. - -- Issue #10542: Add 4 macros to work with surrogates: Py_UNICODE_IS_SURROGATE, - Py_UNICODE_IS_HIGH_SURROGATE, Py_UNICODE_IS_LOW_SURROGATE, - Py_UNICODE_JOIN_SURROGATES. - -- Issue #12724: Add Py_RETURN_NOTIMPLEMENTED macro for returning NotImplemented. - -- PY_PATCHLEVEL_REVISION has been removed, since it's meaningless with - Mercurial. - -- Issue #12173: The first argument of PyImport_ImportModuleLevel is now `const - char *` instead of `char *`. - -- Issue #12380: PyArg_ParseTuple now accepts a bytearray for the 'c' format. - -Documentation -------------- - -- Issue #13989: Document that GzipFile does not support text mode, and give a - more helpful error message when opened with an invalid mode string. - -- Issue #13921: Undocument and clean up sqlite3.OptimizedUnicode, - which is obsolete in Python 3.x. It's now aliased to str for - backwards compatibility. - -- Issue #12102: Document that buffered files must be flushed before being used - with mmap. Patch by Steffen Daode Nurpmeso. - -- Issue #8982: Improve the documentation for the argparse Namespace object. - -- Issue #9343: Document that argparse parent parsers must be configured before - their children. - -- Issue #13498: Clarify docs of os.makedirs()'s exist_ok argument. Done with - great native-speaker help from R. David Murray. - -- Issues #13491 and #13995: Fix many errors in sqlite3 documentation. - Initial patch for #13491 by Johannes Vogel. - -- Issue #13402: Document absoluteness of sys.executable. - -- Issue #13883: PYTHONCASEOK also works on OS X. - -- Issue #9021: Add an introduction to the copy module documentation. - -- Issue #6005: Examples in the socket library documentation use sendall, where - relevant, instead send method. - -- Issue #12798: Updated the mimetypes documentation. - -- Issue #12949: Document the kwonlyargcount argument for the PyCode_New - C API function. - -- Issue #13513: Fix io.IOBase documentation to correctly link to the - io.IOBase.readline method instead of the readline module. - -- Issue #13237: Reorganise subprocess documentation to emphasise convenience - functions and the most commonly needed arguments to Popen. - -- Issue #13141: Demonstrate recommended style for socketserver examples. - -- Issue #11818: Fix tempfile examples for Python 3. - - -What's New in Python 3.2? -========================= - -*Release date: 20-Feb-2011* - -Core and Builtins ------------------ - -- Issue #11249: Fix potential crashes when using the limited API. - -Build ------ - -- Issue #11222: Fix non-framework shared library build on Mac OS X. - -- Issue #11184: Fix large-file support on AIX. - -- Issue #941346: Fix broken shared library build on AIX. - -Documentation -------------- - -- Issue #10709: Add updated AIX notes in Misc/README.AIX. - - -What's New in Python 3.2 Release Candidate 3? -============================================= - -*Release date: 13-Feb-2011* - -Core and Builtins ------------------ - -- Issue #11134: Add missing fields to typeslots.h. - -- Issue #11135: Remove redundant doc field from PyType_Spec. - -- Issue #11067: Add PyType_GetFlags, to support PyUnicode_Check in the limited - ABI. - -- Issue #11118: Fix bogus export of None in python3.dll. - -Library -------- - -- Issue #11116: any error during addition of a message to a mailbox now causes a - rollback, instead of leaving the mailbox partially modified. - -- Issue #11132: Fix passing of "optimize" parameter when recursing in - compileall.compile_dir(). - -- Issue #11110: Fix a potential decref of a NULL in sqlite3. - -- Issue #8275: Fix passing of callback arguments with ctypes under Win64. Patch - by Stan Mihai. - -Build ------ - -- Issue #11079: The /Applications/Python x.x folder created by the Mac OS X - installers now includes a link to the installed documentation and no longer - includes an Extras directory. The Tools directory is now installed in the - framework under share/doc. - -- Issue #11121: Fix building with --enable-shared. - -Tests ------ - -- Issue #10971: test_zipimport_support is once again compatible with the refleak - hunter feature of test.regrtest. - - -What's New in Python 3.2 Release Candidate 2? -============================================= - -*Release date: 30-Jan-2011* - -Core and Builtins ------------------ - -- Issue #10451: memoryview objects could allow mutating a readable buffer. - Initial patch by Ross Lagerwall. - -Library -------- - -- Issue #9124: mailbox now accepts binary input and reads and writes mailbox - files in binary mode, using the email package's binary support to parse - arbitrary email messages. StringIO and text file input is deprecated, - and string input fails early if non-ASCII characters are used, where - previously it would fail when the email was processed in a later step. - -- Issue #10845: Mitigate the incompatibility between the multiprocessing - module on Windows and the use of package, zipfile or directory execution - by special casing main modules that actually *are* called __main__.py. - -- Issue #11045: Protect logging call against None argument. - -- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save - commands. - -- Issue #11053: Fix IDLE "Syntax Error" windows to behave as in 2.x, - preventing a confusing hung appearance on OS X with the windows - obscured. - -- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the - menu accelerators for Open Module, Go to Line, and New Indent Width. - The accelerators still work but no longer appear in the menu items. - -- Issue #10989: Fix a crash on SSLContext.load_verify_locations(None, True). - -- Issue #11020: Command-line pyclbr was broken because of missing 2-to-3 - conversion. - -- Issue #11019: Fixed BytesGenerator so that it correctly handles a Message - with a None body. - -- Issue #11014: Make 'filter' argument in tarfile.Tarfile.add() into a - keyword-only argument. The preceding positional argument was deprecated, - so it made no sense to add filter as a positional argument. - -- Issue #11004: Repaired edge case in deque.count(). - -- Issue #10974: IDLE no longer crashes if its recent files list includes files - with non-ASCII characters in their path names. - -- Have hashlib.algorithms_available and hashlib.algorithms_guaranteed both - return sets instead of one returning a tuple and the other a frozenset. - -- Issue #10987: Fix the recursion limit handling in the _pickle module. - -- Issue #10983: Fix several bugs making tunnel requests in http.client. - -- Issue #10955: zipimport uses ASCII encoding instead of cp437 to decode - filenames, at bootstrap, if the codec registry is not ready yet. It is still - possible to have non-ASCII filenames using the Unicode flag (UTF-8 encoding) - for all file entries in the ZIP file. - -- Issue #10949: Improved robustness of rotating file handlers. - -- Issue #10955: Fix a potential crash when trying to mmap() a file past its - length. Initial patch by Ross Lagerwall. - -- Issue #10898: Allow compiling the posix module when the C library defines - a symbol named FSTAT. - -- Issue #10980: the HTTP server now encodes headers with iso-8859-1 (latin1) - encoding. This is the preferred encoding of PEP 3333 and the base encoding - of HTTP 1.1. - -- To match the behaviour of HTTP server, the HTTP client library now also - encodes headers with iso-8859-1 (latin1) encoding. It was already doing - that for incoming headers which makes this behaviour now consistent in - both incoming and outgoing direction. - -- Issue #9509: argparse now properly handles IOErrors raised by - argparse.FileType. - -- Issue #10961: The new pydoc server now better handles exceptions raised - during request handling. - -- Issue #10680: Fix mutually exclusive arguments for argument groups in - argparse. - -Build ------ - -- Issue #11054: Allow Mac OS X installer builds to again work on 10.5 with - the system-provided Python. - - -What's New in Python 3.2 Release Candidate 1 -============================================ - -*Release date: 16-Jan-2011* - -Core and Builtins ------------------ - -- Issue #10889: range indexing and slicing now works correctly on ranges with - a length that exceeds sys.maxsize. - -- Issue #10892: Don't segfault when trying to delete __abstractmethods__ from a - class. - -- Issue #8020: Avoid a crash where the small objects allocator would read - non-Python managed memory while it is being modified by another thread. Patch - by Matt Bandy. - -- Issue #10841: On Windows, set the binary mode on stdin, stdout, stderr and all - io.FileIO objects (to not translate newlines, \r\n <=> \n). The Python parser - translates newlines (\r\n => \n). - -- Remove buffer API from stable ABI for now, see #10181. - -- Issue #8651: PyArg_Parse*() functions raise an OverflowError if the file - doesn't have PY_SSIZE_T_CLEAN define and the size doesn't fit in an int - (length bigger than 2^31-1 bytes). - -- Issue #9015, #9611: FileIO.readinto(), FileIO.write(), os.write() and - stdprinter.write() clamp the length to INT_MAX on Windows. - -- Issue #8278: On Windows and with a NTFS filesystem, os.stat() and os.utime() - can now handle dates after 2038. - -- Issue #10780: PyErr_SetFromWindowsErrWithFilename() and - PyErr_SetExcFromWindowsErrWithFilename() decode the filename from the - filesystem encoding instead of UTF-8. - -- Issue #10779: PyErr_WarnExplicit() decodes the filename from the filesystem - encoding instead of UTF-8. - -- Add sys.flags attribute for the new -q command-line option. - -- Issue #11506: Trying to assign to a bytes literal should result in a - SyntaxError. - -Library -------- - -- Issue #10916: mmap should not segfault when a file is mapped using 0 as length - and a non-zero offset, and an attempt to read past the end of file is made - (IndexError is raised instead). Patch by Ross Lagerwall. - -- Issue #10154, #10090: change the normalization of UTF-8 to "UTF-8" instead - of "UTF8" in the locale module as the latter is not supported MacOSX and OpenBSD. - -- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather - than the currently problematic Apple-supplied one, when running with the - 64-/32-bit installer variant. - -- Issue #4953: cgi.FieldStorage and cgi.parse() parse the request as bytes, not - as unicode, and accept binary files. Add encoding and errors attributes to - cgi.FieldStorage. Patch written by Pierre Quentel (with many inputs by Glenn - Linderman). - -- Add encoding and errors arguments to urllib.parse_qs() and urllib.parse_qsl(). - -- Issue #10899: No function type annotations in the standard library. Removed - function type annotations from _pyio.py. - -- Issue #10875: Update Regular Expression HOWTO; patch by 'SilentGhost'. - -- Issue #10872: The repr() of TextIOWrapper objects now includes the mode - if available. - -- Issue #10869: Fixed bug where ast.increment_lineno modified the root node - twice. - -- Issue #5871: email.header.Header.encode now raises an error if any - continuation line in the formatted value has no leading white space and looks - like a header. Since Generator uses Header to format all headers, this check - is made for all headers in any serialized message at serialization time. This - provides protection against header injection attacks. - -- Issue #10859: Make ``contextlib.GeneratorContextManager`` officially - private by renaming it to ``_GeneratorContextManager``. - -- Issue #10042: Fixed the total_ordering decorator to handle cross-type - comparisons that could lead to infinite recursion. - -- Issue #10686: the email package now :rfc:`2047`\ -encodes headers with - non-ASCII bytes (parsed by a BytesParser) when doing conversion to 7bit-clean - presentation, instead of replacing them with ?s. - -- email.header.Header was incorrectly encoding folding whitespace when - rfc2047-encoding header values with embedded newlines, leaving them without - folding whitespace. It now uses the continuation_ws, as it does for - continuation lines that it creates itself. - -- Issue #1777412, #10827: Changed the rules for 2-digit years. The - time.asctime(), time.ctime() and time.strftime() functions will now format - any year when ``time.accept2dyear`` is False and will accept years >= 1000 - otherwise. ``time.mktime`` and ``time.strftime`` now accept full range - supported by the OS. With Visual Studio or on Solaris, the year is limited to - the range [1; 9999]. Conversion of 2-digit years to 4-digit is deprecated. - -- Issue #7858: Raise an error properly when os.utime() fails under Windows - on an existing file. - -- Issue #3839: wsgiref should not override a Content-Length header set by - the application. Initial patch by Clovis Fabricio. - -- Issue #10492: bdb.Bdb.run() only traces the execution of the code, not the - compilation (if the input is a string). - -- Issue #7995: When calling accept() on a socket with a timeout, the returned - socket is now always blocking, regardless of the operating system. - -- Issue #10756: atexit normalizes the exception before displaying it. Patch by - Andreas Stührk. - -- Issue #10790: email.header.Header.append's charset logic now works correctly - for charsets whose output codec is different from its input codec. - -- Issue #10819: SocketIO.name property returns -1 when its closed, instead of - raising a ValueError, to fix repr(). - -- Issue #8650: zlib.compress() and zlib.decompress() raise an OverflowError if - the input buffer length doesn't fit into an unsigned int (length bigger than - 2^32-1 bytes). - -- Issue #6643: Reinitialize locks held within the threading module after fork to - avoid a potential rare deadlock or crash on some platforms. - -- Issue #10806, issue #9905: Fix subprocess pipes when some of the standard file - descriptors (0, 1, 2) are closed in the parent process. Initial patch by Ross - Lagerwall. - -- `unittest.TestCase` can be instantiated without a method name; for simpler - exploration from the interactive interpreter. - -- Issue #10798: Reject supporting concurrent.futures if the system has too - few POSIX semaphores. - -- Issue #10807: Remove base64, bz2, hex, quopri, rot13, uu and zlib codecs from - the codec aliases. They are still accessible via codecs.lookup(). - -- Issue #10801: In zipfile, support different encodings for the header and the - filenames. - -- Issue #6285: IDLE no longer crashes on missing help file; patch by Scott - David Daniels. - -- Fix collections.OrderedDict.setdefault() so that it works in subclasses that - define __missing__(). - -- Issue #10786: unittest.TextTestRunner default stream no longer bound at import - time. `sys.stderr` now looked up at instantiation time. Fix contributed by - Mark Roddy. - -- Issue #10753: Characters ';', '=' and ',' in the PATH_INFO environment variable - won't be quoted when the URI is constructed by the wsgiref.util's request_uri - method. According to RFC 3986, these characters can be a part of params in - PATH component of URI and need not be quoted. - -- Issue #10738: Fix webbrowser.Opera.raise_opts. - -- Issue #9824: SimpleCookie now encodes , and ; in values to cater to how - browsers actually parse cookies. - -- Issue #9333: os.symlink now available regardless of user privileges. The - function now raises OSError on Windows >=6.0 when the user is unable to create - symbolic links. XP and 2003 still raise NotImplementedError. - -- Issue #10783: struct.pack() no longer implicitly encodes unicode to UTF-8. - -- Issue #10730: Add SVG mime types to mimetypes module. - -- Issue #10768: Make the Tkinter ScrolledText widget work again. - -- Issue #10777: Fix "dictionary changed size during iteration" bug in - ElementTree register_namespace(). - -- Issue #10626: test_logging now preserves logger disabled states. - -- Issue #10774: test_logging now removes temp files created during tests. - -- Issue #5258/#10642: if site.py encounters a .pth file that generates an error, - it now prints the filename, line number, and traceback to stderr and skips - the rest of that individual file, instead of stopping processing entirely. - -- Issue #10763: subprocess.communicate() closes stdout and stderr if both are - pipes (bug specific to Windows). - -- Issue #1693546: fix email.message RFC 2231 parameter encoding to be in better - compliance (no "s around encoded values). - -- Improved the diff message in the unittest module's assertCountEqual(). - -- Issue #1155362: email.utils.parsedate_tz now handles a missing space before - the '-' of a timezone field as well as before a '+'. - -- Issue #4871: The zipfile module now gives a more useful error message if - an attempt is made to use a string to specify the archive password. - -- Issue #10750: The ``raw`` attribute of buffered IO objects is now read-only. - -- Deprecated assertDictContainsSubset() in the unittest module. - -C-API ------ - -- PyObject_CallMethod now passes along any underlying AttributeError from - PyObject_GetAttr, instead of replacing it with something less informative - -- Issue #10913: Deprecate misleading functions PyEval_AcquireLock() and - PyEval_ReleaseLock(). The thread-state aware APIs should be used instead. - -- Issue #10333: Remove ancient GC API, which has been deprecated since Python - 2.2. - -Build ------ - -- Issue #10843: Update third-party library versions used in OS X 32-bit - installer builds: bzip2 1.0.6, readline 6.1.2, SQLite 3.7.4 (with FTS3/FTS4 - and RTREE enabled), and ncursesw 5.5 (wide-char support enabled). - -- Issue #10820: Fix OS X framework installs to support version-specific - scripts (#10679). - -- Issue #7716: Under Solaris, don't assume existence of /usr/xpg4/bin/grep in - the configure script but use $GREP instead. Patch by Fabian Groffen. - -- Issue #10475: Don't hardcode compilers for LDSHARED/LDCXXSHARED on NetBSD - and DragonFly BSD. Patch by Nicolas Joly. - -- Issue #10679: The "idle", "pydoc" and "2to3" scripts are now installed with - a version-specific suffix on "make altinstall". - -- Issue #10655: Fix the build on PowerPC on Linux with GCC when building with - timestamp profiling (--with-tsc): the preprocessor test for the PowerPC - support now looks for "__powerpc__" as well as "__ppc__": the latter seems to - only be present on OS X; the former is the correct one for Linux with GCC. - -- Issue #1099: Fix the build on MacOSX when building a framework with pydebug - using GCC 4.0. - -Tools/Demos ------------ - -- Issue #10843: Install the Tools directory on OS X in the applications Extras - (/Applications/Python 3.n/Extras/) where the Demo directory had previous been - installed. - -- Issue #7962: The Demo directory is gone. Most of the old and unmaintained - demos have been removed, others integrated in documentation or a new - Tools/demo subdirectory. - -- Issue #10502: Addition of the unittestgui tool. Originally by Steve Purcell. - Updated for test discovery by Mark Roddy and Python 3 compatibility by Brian - Curtin. - -Tests ------ - -- Issue #11910: Fix test_heapq to skip the C tests when _heapq is missing. - -- Fix test_startfile to wait for child process to terminate before finishing. - -- Issue #10822: Fix test_posix:test_getgroups failure under Solaris. Patch - by Ross Lagerwall. - -- Make the --coverage flag work for test.regrtest. - -- Issue #1677694: Refactor and improve test_timeout. Original patch by - Björn Lindqvist. - -- Issue #5485: Add tests for the UseForeignDTD method of expat parser objects. - Patch by Jean-Paul Calderone and Sandro Tosi. - -- Issue #6293: Have regrtest.py echo back sys.flags. This is done by default in - whole runs and enabled selectively using ``--header`` when running an explicit - list of tests. Original patch by Collin Winter. - - -What's New in Python 3.2 Beta 2? -================================ - -*Release date: 19-Dec-2010* - -Core and Builtins ------------------ - -- Issue #8844: Regular and recursive lock acquisitions can now be interrupted - by signals on platforms using pthreads. Patch by Reid Kleckner. - -- Issue #4236: PyModule_Create2 now checks the import machinery directly - rather than the Py_IsInitialized flag, avoiding a Fatal Python - error in certain circumstances when an import is done in __del__. - -- Issue #5587: add a repr to dict_proxy objects. Patch by David Stanek and - Daniel Urban. - -Library -------- - -- Issue #3243: Support iterable bodies in httplib. Patch Contributions by - Xuanji Li and Chris AtLee. - -- Issue #10611: SystemExit exception will no longer kill a unittest run. - -- Issue #9857: It is now possible to skip a test in a setUp, tearDown or clean - up function. - -- Issue #10573: use actual/expected consistently in unittest methods. - The order of the args of assertCountEqual is also changed. - -- Issue #9286: email.utils.parseaddr no longer concatenates blank-separated - words in the local part of email addresses, thereby preserving the input. - -- Issue #6791: Limit header line length (to 65535 bytes) in http.client - and http.server, to avoid denial of services from the other party. - -- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. - -- Issue #9907: Fix tab handling on OSX when using editline by calling - rl_initialize first, then setting our custom defaults, then reading .editrc. - -- Issue #4188: Avoid creating dummy thread objects when logging operations - from the threading module (with the internal verbose flag activated). - -- Issue #10711: Remove HTTP 0.9 support from http.client. The ``strict`` - parameter to HTTPConnection and friends is deprecated. - -- Issue #9721: Fix the behavior of urljoin when the relative url starts with a - ';' character. Patch by Wes Chow. - -- Issue #10714: Limit length of incoming request in http.server to 65536 bytes - for security reasons. Initial patch by Ross Lagerwall. - -- Issue #9558: Fix distutils.command.build_ext with VS 8.0. - -- Issue #10667: Fast path for collections.Counter(). - -- Issue #10695: passing the port as a string value to telnetlib no longer - causes debug mode to fail. - -- Issue #1078919: add_header now automatically RFC2231 encodes parameters - that contain non-ascii values. - -- Issue #10188 (partial resolution): tempfile.TemporaryDirectory emits - a warning on sys.stderr rather than throwing a misleading exception - if cleanup fails due to nulling out of modules during shutdown. - Also avoids an AttributeError when mkdtemp call fails and issues - a ResourceWarning on implicit cleanup via __del__. - -- Issue #10107: Warn about unsaved files in IDLE on OSX. - -- Issue #7213: subprocess.Popen's default for close_fds has been changed. - It is now True in most cases other than on Windows when input, output or - error handles are provided. - -- Issue #6559: subprocess.Popen has a new pass_fds parameter (actually - added in 3.2beta1) to allow specifying a specific list of file descriptors - to keep open in the child process. - -- Issue #1731717: Fixed the problem where subprocess.wait() could cause an - OSError exception when The OS had been told to ignore SIGCLD in our process - or otherwise not wait for exiting child processes. - -Tests ------ - -- Issue #775964: test_grp now skips YP/NIS entries instead of failing when - encountering them. - -Tools/Demos ------------ - -- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and - Cocoa AquaTk. - -- Issue #10710: ``Misc/setuid-prog.c`` is removed from the source tree. - -- Issue #10706: Remove outdated script runtests.sh. Either ``make test`` - or ``python -m test`` should be used instead. - -Build ------ - -- The Windows build now uses Tcl/Tk 8.5.9 and sqlite3 3.7.4. - -- Issue #9234: argparse supports alias names for subparsers. - - -What's New in Python 3.2 Beta 1? -================================ - -*Release date: 05-Dec-2010* - -Core and Builtins ------------------ - -- Issue #10630: Return dict views from the dict proxy keys()/values()/items() - methods. - -- Issue #10596: Fix float.__mod__ to have the same behaviour as float.__divmod__ - with respect to signed zeros. -4.0 % 4.0 should be 0.0, not -0.0. - -- Issue #1772833: Add the -q command-line option to suppress copyright and - version output in interactive mode. - -- Provide an *optimize* parameter in the built-in compile() function. - -- Fixed several corner case issues on Windows in os.stat/os.lstat related to - reparse points. - -- PEP 384 (Defining a Stable ABI) is implemented. - -- Issue #2690: Range objects support negative indices and slicing. - -- Issue #9915: Speed up sorting with a key. - -- Issue #8685: Speed up set difference ``a - b`` when source set ``a`` is much - larger than operand ``b``. Patch by Andrew Bennetts. - -- Issue #10518: Bring back the callable() builtin. - -- Issue #7094: Added alternate formatting (specified by '#') to ``__format__`` - method of float, complex, and Decimal. This allows more precise control over - when decimal points are displayed. - -- Issue #10474: range.count() should return integers. - -- Issue #1574217: isinstance now catches only AttributeError, rather than - masking all errors. - -Library -------- - -- logging: added "handler of last resort". See http://bit.ly/last-resort-handler - -- test.support: Added TestHandler and Matcher classes for better support of - assertions about logging. - -- Issue #4391: Use proper plural forms in argparse. - -- Issue #10601: sys.displayhook uses 'backslashreplace' error handler on - UnicodeEncodeError. - -- Add the "display" and "undisplay" pdb commands. - -- Issue #7245: Add a SIGINT handler in pdb that allows breaking a program again - after a "continue" command. - -- Add the "interact" pdb command. - -- Issue #7905: Actually respect the keyencoding parameter to shelve.Shelf. - -- Issue #1569291: Speed up array.repeat(). - -- Provide an interface to set the optimization level of compilation in - py_compile, compileall and zipfile.PyZipFile. - -- Issue #7904: Changes to urllib.parse.urlsplit to handle schemes as defined by - RFC3986. Anything before :// is considered a scheme and is followed by an - authority (or netloc) and by '/' led path, which is optional. - -- Issue #6045: dbm.gnu databases now support get() and setdefault() methods. - -- Issue #10620: `python -m unittest` can accept file paths instead of module - names for running specific tests. - -- Issue #9424: Deprecate the `unittest.TestCase` methods `assertEquals`, - `assertNotEquals`, `assertAlmostEquals`, `assertNotAlmostEquals` and `assert_` - and replace them with the correct methods in the Python test suite. - -- Issue #10272: The ssl module now raises socket.timeout instead of a generic - SSLError on socket timeouts. - -- Issue #10528: Allow translators to reorder placeholders in localizable - messages from argparse. - -- Issue #10497: Fix incorrect use of gettext in argparse. - -- Issue #10478: Reentrant calls inside buffered IO objects (for example by - way of a signal handler) now raise a RuntimeError instead of freezing the - current process. - -- logging: Added getLogRecordFactory/setLogRecordFactory with docs and tests. - -- Issue #10549: Fix pydoc traceback when text-documenting certain classes. - -- Issue #2001: New HTML server with enhanced Web page features. Patch by Ron - Adam. - -- Issue #10360: In WeakSet, do not raise TypeErrors when testing for membership - of non-weakrefable objects. - -- Issue #940286: pydoc.Helper.help() ignores input/output init parameters. - -- Issue #1745035: Add a command size and data size limit to smtpd.py, to prevent - DoS attacks. Patch by Savio Sena. - -- Issue #4925: Add filename to error message when executable can't be found in - subprocess. - -- Issue #10391: Don't dereference invalid memory in error messages in the ast - module. - -- Issue #10027: st_nlink was not being set on Windows calls to os.stat or - os.lstat. Patch by Hirokazu Yamamoto. - -- Issue #9333: Expose os.symlink only when the SeCreateSymbolicLinkPrivilege is - held by the user's account, i.e., when the function can actually be used. - -- Issue #8879: Add os.link support for Windows. - -- Issue #7911: ``unittest.TestCase.longMessage`` defaults to True for improved - failure messages by default. Patch by Mark Roddy. - -- Issue #1486713: HTMLParser now has an optional tolerant mode where it tries to - guess at the correct parsing of invalid html. - -- Issue #10554: Add context management protocol support to subprocess.Popen objects. - -- Issue #8989: email.utils.make_msgid now has a domain parameter that can - override the domain name used in the generated msgid. - -- Issue #9299: Add exist_ok parameter to os.makedirs to suppress the 'File - exists' exception when a target directory already exists with the specified - mode. Patch by Ray Allen. - -- Issue #9573: os.fork() now works correctly when triggered as a side effect of - a module import. - -- Issue #10464: netrc now correctly handles lines with embedded '#' characters. - -- Added itertools.accumulate(). - -- Issue #4113: Added custom ``__repr__`` method to ``functools.partial``. - Original patch by Daniel Urban. - -- Issue #10273: Rename `assertRegexpMatches` and `assertRaisesRegexp` to - `assertRegex` and `assertRaisesRegex`. - -- Issue #10535: Enable silenced warnings in unittest by default. - -- Issue #9873: The URL parsing functions in urllib.parse now accept ASCII byte - sequences as input in addition to character strings. - -- Issue #10586: The statistics API for the new functools.lru_cache has been - changed to a single cache_info() method returning a named tuple. - -- Issue #10323: itertools.islice() now consumes the minimum number of inputs - before stopping. Formerly, the final state of the underlying iterator was - undefined. - -- Issue #10565: The collections.Iterator ABC now checks for both __iter__ and - __next__. - -- Issue #10242: Fixed implementation of unittest.ItemsEqual and gave it a new - more informative name, unittest.CountEqual. - -- Issue #10561: In pdb, clear the breakpoints by the breakpoint number. - -- Issue #2986: difflib.SequenceMatcher gets a new parameter, autojunk, which can - be set to False to turn off the previously undocumented 'popularity' - heuristic. Patch by Terry Reedy and Eli Bendersky. - -- Issue #10534: in difflib, expose bjunk and bpopular sets; deprecate - undocumented and now redundant isbjunk and isbpopular methods. - -- Issue #9846: zipfile is now correctly closing underlying file objects. - -- Issue #10459: Update CJK character names to Unicode 6.0. - -- Issue #4493: urllib.request adds '/' in front of path components which does not - start with '/. Common behavior exhibited by browsers and other clients. - -- Issue #6378: idle.bat now runs with the appropriate Python version rather than - the system default. Patch by Sridhar Ratnakumar. - -- Issue #10470: 'python -m unittest' will now run test discovery by default, - when no extra arguments have been provided. - -- Issue #3709: BaseHTTPRequestHandler will buffer the headers and write to - output stream only when end_headers is invoked. This is a speedup and an - internal optimization. Patch by Andrew Shaaf. - -- Issue #10220: Added inspect.getgeneratorstate. Initial patch by Rodolpho - Eckhardt. - -- Issue #10453: compileall now uses argparse instead of getopt, and thus - provides clean output when called with '-h'. - -- Issue #8078: Add constants for higher baud rates in the termios module. Patch - by Rodolpho Eckhardt. - -- Issue #10407: Fix two NameErrors in distutils. - -- Issue #10371: Deprecated undocumented functions in the trace module. - -- Issue #10467: Fix BytesIO.readinto() after seeking into a position after the - end of the file. - -- configparser: 100% test coverage. - -- Issue #10499: configparser supports pluggable interpolation handlers. The - default classic interpolation handler is called BasicInterpolation. Another - interpolation handler added (ExtendedInterpolation) which supports the syntax - used by zc.buildout (e.g. interpolation between sections). - -- configparser: the SafeConfigParser class has been renamed to ConfigParser. - The legacy ConfigParser class has been removed but its interpolation mechanism - is still available as LegacyInterpolation. - -- configparser: Usage of RawConfigParser is now discouraged for new projects - in favor of ConfigParser(interpolation=None). - -- Issue #1682942: configparser supports alternative option/value delimiters. - -- Issue #5412: configparser supports mapping protocol access. - -- Issue #9411: configparser supports specifying encoding for read operations. - -- Issue #9421: configparser's getint(), getfloat() and getboolean() methods - accept vars and default arguments just like get() does. - -- Issue #9452: configparser supports reading from strings and dictionaries - (thanks to the mapping protocol API, the latter can be used to copy data - between parsers). - -- configparser: accepted INI file structure is now customizable, including - comment prefixes, name of the DEFAULT section, empty lines in multiline - values, and indentation. - -- Issue #10326: unittest.TestCase instances can be pickled. - -- Issue #9926: Wrapped TestSuite subclass does not get __call__ executed. - -- Issue #9920: Skip tests for cmath.atan and cmath.atanh applied to complex - zeros on systems where the log1p function fails to respect the sign of zero. - This fixes a test failure on AIX. - -- Issue #9732: Addition of getattr_static to the inspect module. - -- Issue #10446: Module documentation generated by pydoc now links to a - version-specific online reference manual. - -- Make the 'No module named' exception message from importlib consistent. - -- Issue #10443: Add the SSLContext.set_default_verify_paths() method. - -- Issue #10440: Support RUSAGE_THREAD as a constant in the resource module. - Patch by Robert Collins. - -- Issue #10429: IMAP.starttls() stored the capabilities as bytes objects, rather - than strings. - -C-API ------ - -- Issue #10557: Added a new API function, PyUnicode_TransformDecimalToASCII(), - which transforms non-ASCII decimal digits in a Unicode string to their ASCII - equivalents. - -- Issue #9518: Extend the PyModuleDef_HEAD_INIT macro to explicitly - zero-initialize all fields, fixing compiler warnings seen when building - extension modules with gcc with "-Wmissing-field-initializers" (implied by - "-W"). - -- Issue #10255: Fix reference leak in Py_InitializeEx(). Patch by Neil - Schemenauer. - -- structseq.h is now included in Python.h. - -- Loosen PyArg_ValidateKeywordArguments to allow dict subclasses. - -Tests ------ - -- regrtest.py once again ensures the test directory is removed from sys.path - when it is invoked directly as the __main__ module. - -- `python -m test` can be used to run the test suite as well as `python -m - test.regrtest`. - -- Do not fail test_socket when the IP address of the local hostname cannot be - looked up. - -- Issue #8886: Use context managers throughout test_zipfile. Patch by Eric - Carstensen. - -Build ------ - -- Issue #10325: Fix two issues in the fallback definitions for PY_ULLONG_MAX and - PY_LLONG_MAX that made them unsuitable for use in preprocessor conditionals. - -Documentation -------------- - -- Issue #10299: List the built-in functions in a table in functions.rst. - - -What's New in Python 3.2 Alpha 4? -================================= - -*Release date: 13-Nov-2010* - -Core and Builtins ------------------ - -- Issue #10372: Import the warnings module only after the IO library is - initialized, so as to avoid bootstrap issues with the '-W' option. - -- Issue #10293: Remove obsolete field in the PyMemoryView structure, unused - undocumented value PyBUF_SHADOW, and strangely-looking code in - PyMemoryView_GetContiguous. - -- Issue #6081: Add str.format_map(), similar to ``str.format(**mapping)``. - -- If FileIO.__init__ fails, close the file descriptor. - -- Issue #10221: dict.pop(k) now has a key error message that includes the - missing key (same message d[k] returns for missing keys). - -- Issue #5437: A preallocated MemoryError instance should not keep traceback - data (including local variables caught in the stack trace) alive infinitely. - -- Issue #10186: Fix the SyntaxError caret when the offset is equal to the length - of the offending line. - -- Issue #10089: Add support for arbitrary -X options on the command line. They - can be retrieved through a new attribute ``sys._xoptions``. - -- Issue #4388: On Mac OS X, decode command line arguments from UTF-8, instead of - the locale encoding. If the LANG (and LC_ALL and LC_CTYPE) environment - variable is not set, the locale encoding is ISO-8859-1, whereas most programs - (including Python) expect UTF-8. Python already uses UTF-8 for the filesystem - encoding and to encode command line arguments on this OS. - -- Issue #9713, #10114: Parser functions (e.g. PyParser_ASTFromFile) expect - filenames encoded to the filesystem encoding with the surrogateescape error - handler (to support undecodable bytes), instead of UTF-8 in strict mode. - -- Issue #9997: Don't let the name "top" have special significance in scope - resolution. - -- Issue #9862: Compensate for broken PIPE_BUF in AIX by hard coding its value as - the default 512 when compiling on AIX. - -- Use locale encoding instead of UTF-8 to encode and decode filenames if - Py_FileSystemDefaultEncoding is not set. - -- Issue #10095: fp_setreadl() doesn't reopen the file, instead reuse the file - descriptor. - -- Issue #9418: Moved private string methods ``_formatter_parser`` and - ``_formatter_field_name_split`` into a new ``_string`` module. - -- Issue #9992: Remove PYTHONFSENCODING environment variable. - -Library -------- - -- Issue #12943: python -m tokenize support has been added to tokenize. - -- Issue #10465: fix broken delegating of attributes by gzip._PaddedFile. - -- Issue #10356: Decimal.__hash__(-1) should return -2. - -- Issue #1553375: logging: Added stack_info kwarg to display stack information. - -- Issue #5111: IPv6 Host in the Header is wrapped inside [ ]. Patch by Chandru. - -- Fix Fraction.__hash__ so that Fraction.__hash__(-1) is -2. (See also issue - #10356.) - -- Issue #4471: Add the IMAP.starttls() method to enable encryption on standard - IMAP4 connections. Original patch by Lorenzo M. Catucci. - -- Issue #1466065: Add 'validate' option to base64.b64decode to raise an error if - there are non-base64 alphabet characters in the input. - -- Issue #10386: Add __all__ to token module; this simplifies importing in - tokenize module and prevents leaking of private names through ``import *``. - -- Issue #4471: Properly shutdown socket in IMAP.shutdown(). Patch by Lorenzo - M. Catucci. - -- Fix IMAP.login() to work properly. - -- Issue #9244: multiprocessing pool worker processes could terminate - unexpectedly if the return value of a task could not be pickled. Only the - ``repr`` of such errors are now sent back, wrapped in an - ``MaybeEncodingError`` exception. - -- Issue #9244: The ``apply_async()`` and ``map_async()`` methods of - ``multiprocessing.Pool`` now accepts an ``error_callback`` argument. This can - be a callback with the signature ``callback(exc)``, which will be called if - the target raises an exception. - -- Issue #10022: The dictionary returned by the ``getpeercert()`` method of SSL - sockets now has additional items such as ``issuer`` and ``notBefore``. - -- ``usenetrc`` is now false by default for NNTP objects. - -- Issue #1926: Add support for NNTP over SSL on port 563, as well as STARTTLS. - Patch by Andrew Vant. - -- Issue #10335: Add tokenize.open(), detect the file encoding using - tokenize.detect_encoding() and open it in read only mode. - -- Issue #10321: Add support for binary data to smtplib.SMTP.sendmail, and a new - method send_message to send an email.message.Message object. - -- Issue #6011: sysconfig and distutils.sysconfig use the surrogateescape error - handler to parse the Makefile file. Avoid a UnicodeDecodeError if the source - code directory name contains a non-ASCII character and the locale encoding is - ASCII. - -- Issue #10329: The trace module writes reports using the input Python script - encoding, instead of the locale encoding. Patch written by Alexander - Belopolsky. - -- Issue #10126: Fix distutils' test_build when Python was built with - --enable-shared. - -- Issue #9281: Prevent race condition with mkdir in distutils. Patch by - Arfrever. - -- Issue #10229: Fix caching error in gettext. - -- Issue #10252: Close file objects in a timely manner in distutils code and - tests. Patch by Brian Brazil, completed by Éric Araujo. - -- Issue #10180: Pickling file objects is now explicitly forbidden, since - unpickling them produced nonsensical results. - -- Issue #10311: The signal module now restores errno before returning from its - low-level signal handler. Patch by Hallvard B Furuseth. - -- Issue #10282: Add a ``nntp_implementation`` attribute to NNTP objects. - -- Issue #10283: Add a ``group_pattern`` argument to NNTP.list(). - -- Issue #10155: Add IISCGIHandler to wsgiref.handlers to support IIS CGI - environment better, and to correct unicode environment values for WSGI 1.0.1. - -- Issue #10281: nntplib now returns None for absent fields in the OVER/XOVER - response, instead of raising an exception. - -- wsgiref now implements and validates PEP 3333, rather than an experimental - extension of PEP 333. (Note: earlier versions of Python 3.x may have - incorrectly validated some non-compliant applications as WSGI compliant; if - your app validates with Python <3.2b1+, but not on this version, it is likely - the case that your app was not compliant.) - -- Issue #10280: NNTP.nntp_version should reflect the highest version advertised - by the server. - -- Issue #10184: Touch directories only once when extracting a tarfile. - -- Issue #10199: New package, ``turtledemo`` now contains selected demo scripts - that were formerly found under Demo/turtle. - -- Issue #10265: Close file objects explicitly in sunau. Patch by Brian Brazil. - -- Issue #10266: uu.decode didn't close in_file explicitly when it was given as a - filename. Patch by Brian Brazil. - -- Issue #10110: Queue objects didn't recognize full queues when the maxsize - parameter had been reduced. - -- Issue #10160: Speed up operator.attrgetter. Patch by Christos Georgiou. - -- logging: Added style option to basicConfig() to allow %, {} or $-formatting. - -- Issue #5729: json.dumps() now supports using a string such as '\t' for - pretty-printing multilevel objects. - -- Issue #10253: FileIO leaks a file descriptor when trying to open a file for - append that isn't seekable. Patch by Brian Brazil. - -- Support context management protocol for file-like objects returned by mailbox - ``get_file()`` methods. - -- Issue #10246: uu.encode didn't close file objects explicitly when filenames - were given to it. Patch by Brian Brazil. - -- Issue #10198: fix duplicate header written to wave files when writeframes() is - called without data. - -- Close file objects in modulefinder in a timely manner. - -- Close an io.TextIOWrapper object in email.parser in a timely manner. - -- Close a file object in distutils.sysconfig in a timely manner. - -- Close a file object in pkgutil in a timely manner. - -- Issue #10233: Close file objects in a timely manner in the tarfile module and - its test suite. - -- Issue #10093: ResourceWarnings are now issued when files and sockets are - deallocated without explicit closing. These warnings are silenced by default, - except in pydebug mode. - -- tarfile.py: Add support for all missing variants of the GNU sparse extensions - and create files with holes when extracting sparse members. - -- Issue #10218: Return timeout status from ``Condition.wait`` in threading. - -- Issue #7351: Add ``zipfile.BadZipFile`` spelling of the exception name and - deprecate the old name ``zipfile.BadZipfile``. - -- Issue #5027: The standard ``xml`` namespace is now understood by - xml.sax.saxutils.XMLGenerator as being bound to - http://www.w3.org/XML/1998/namespace. Patch by Troy J. Farrell. - -- Issue #5975: Add csv.unix_dialect class. - -- Issue #7761: telnetlib.interact failures on Windows fixed. - -- logging: Added style option to Formatter to allow %, {} or $-formatting. - -- Issue #5178: Added tempfile.TemporaryDirectory class that can be used as a - context manager. - -- Issue #1349106: Generator (and BytesGenerator) flatten method and Header - encode method now support a 'linesep' argument. - -- Issue #5639: Add a *server_hostname* argument to ``SSLContext.wrap_socket`` in - order to support the TLS SNI extension. ``HTTPSConnection`` and ``urlopen()`` - also use this argument, so that HTTPS virtual hosts are now supported. - -- Issue #10166: Avoid recursion in pstats Stats.add() for many stats items. - -- Issue #10163: Skip unreadable registry keys during mimetypes initialization. - -- logging: Made StreamHandler terminator configurable. - -- logging: Allowed filters to be just callables. - -- logging: Added tests for _logRecordClass changes. - -- Issue #10092: Properly reset locale in calendar.Locale*Calendar classes. - -- logging: Added _logRecordClass, getLogRecordClass, setLogRecordClass to - increase flexibility of LogRecord creation. - -- Issue #5117: Case normalization was needed on ntpath.relpath(). Also fixed - root directory issue on posixpath.relpath(). (Ported working fixes from - ntpath.) - -- Issue #1343: xml.sax.saxutils.XMLGenerator now has an option - short_empty_elements to direct it to use self-closing tags when appropriate. - -- Issue #9807 (part 1): Expose the ABI flags in sys.abiflags. Add --abiflags - switch to python-config for command line access. - -- Issue #6098: Don't claim DOM level 3 conformance in minidom. - -- Issue #5762: Fix AttributeError raised by ``xml.dom.minidom`` when an empty - XML namespace attribute is encountered. - -- Issue #2830: Add the ``html.escape()`` function, which quotes all problematic - characters by default. Deprecate ``cgi.escape()``. - -- Issue #9409: Fix the regex to match all kind of filenames, for interactive - debugging in doctests. - -- Issue #9183: ``datetime.timezone(datetime.timedelta(0))`` will now return the - same instance as ``datetime.timezone.utc``. - -- Issue #7523: Add SOCK_CLOEXEC and SOCK_NONBLOCK to the socket module, where - supported by the system. Patch by Nikita Vetoshkin. - -- Issue #10063: file:// scheme will stop accessing remote hosts via ftp - protocol. file:// urls had fallback to access remote hosts via ftp. This was - not correct, change is made to raise a URLError when a remote host is tried to - access via file:// scheme. - -- Issue #1710703: Write structures for an empty ZIP archive when a ZipFile is - created in modes 'a' or 'w' and then closed without adding any files. Raise - BadZipfile (rather than IOError) when opening small non-ZIP files. - -- Issue #10041: The signature of optional arguments in socket.makefile() didn't - match that of io.open(), and they also didn't get forwarded properly to - TextIOWrapper in text mode. Patch by Kai Zhu. - -- Issue #9003: http.client.HTTPSConnection, urllib.request.HTTPSHandler and - urllib.request.urlopen now take optional arguments to allow for server - certificate checking, as recommended in public uses of HTTPS. - -- Issue #6612: Fix site and sysconfig to catch os.getcwd() error, eg. if the - current directory was deleted. Patch written by W. Trevor King. - -- Issue #3873: Speed up unpickling from file objects that have a peek() method. - -- Issue #10075: Add a session_stats() method to SSLContext objects. - -- Issue #9948: Fixed problem of losing filename case information. - -Extension Modules ------------------ - -- Issue #5109: array.array constructor will now use fast code when - initial data is provided in an array object with correct type. - -- Issue #6317: Now winsound.PlaySound only accepts unicode. - -- Issue #6317: Now winsound.PlaySound can accept non ascii filename. - -- Issue #9377: Use Unicode API for gethostname on Windows. - -- Issue #10143: Update "os.pathconf" values. - -- Issue #6518: Support context management protocol for ossaudiodev types. - -- Issue #678250: Make mmap flush a noop on ACCESS_READ and ACCESS_COPY. - -- Issue #9054: Fix a crash occurring when using the pyexpat module with expat - version 2.0.1. - -- Issue #5355: Provide mappings from Expat error numbers to string descriptions - and backwards, in order to actually make it possible to analyze error codes - provided by ExpatError. - -- The Unicode database was updated to 6.0.0. - -C-API ------ - -- Issue #10288: The deprecated family of "char"-handling macros - (ISLOWER()/ISUPPER()/etc) have now been removed: use Py_ISLOWER() etc instead. - -- Issue #9778: Hash values are now always the size of pointers. A new Py_hash_t - type has been introduced. - -Tools/Demos ------------ - -- Issue #10117: Tools/scripts/reindent.py now accepts source files that use - encoding other than ASCII or UTF-8. Source encoding is preserved when - reindented code is written to a file. - -- Issue #7287: Demo/imputil/knee.py was removed. - -Tests ------ - -- Issue #3699: Fix test_bigaddrspace and extend it to test bytestrings as well - as unicode strings. Initial patch by Sandro Tosi. - -- Issue #10294: Remove dead code form test_unicode_file. - -- Issue #10123: Don't use non-ascii filenames in test_doctest tests. Add a new - test specific to unicode (non-ascii name and filename). - -Build ------ - -- Issue #10268: Add a --enable-loadable-sqlite-extensions option to configure. - -- Issue #8852: Allow the socket module to build on OpenSolaris. - -- Drop -OPT:Olimit compiler option. - -- Issue #10094: Use versioned .so files on GNU/kfreeBSD and the GNU Hurd. - -- Accept Oracle Berkeley DB 5.0 and 5.1 as backend for the dbm extension. - -- Issue #7473: avoid link errors when building a framework with a different set - of architectures than the one that is currently installed. - - -What's New in Python 3.2 Alpha 3? -================================= - -*Release date: 09-Oct-2010* - -Core and Builtins ------------------ - -- Issue #10068: Global objects which have reference cycles with their module's - dict are now cleared again. This causes issue #7140 to appear again. - -- Issue #9738: Document PyErr_SetString() and PyErr_SetFromErrnoWithFilename() - encodings. - -- ast.literal_eval() can now handle negative numbers. It is also a little more - liberal in what it accepts without compromising the safety of the evaluation. - For example, 3j+4 and 3+4+5 are both accepted. - -- Issue #10006: type.__abstractmethods__ now raises an AttributeError. As a - result metaclasses can now be ABCs (see #9533). - -- Issue #8670: ctypes.c_wchar supports non-BMP characters with 32 bits wchar_t. - -- Issue #8670: PyUnicode_AsWideChar() and PyUnicode_AsWideCharString() replace - UTF-16 surrogate pairs by single non-BMP characters for 16 bits Py_UNICODE and - 32 bits wchar_t (eg. Linux in narrow build). - -- Issue #10003: Allow handling of SIGBREAK on Windows. Fixes a regression - introduced by issue #9324. - -- Issue #9979: Create function PyUnicode_AsWideCharString(). - -- Issue #7397: Mention that importlib.import_module() is probably what someone - really wants to be using in __import__'s docstring. - -- Issue #8521: Allow CreateKeyEx, OpenKeyEx, and DeleteKeyEx functions of winreg - to use named arguments. - -- Issue #9930: Remove bogus subtype check that was causing (e.g.) - float.__rdiv__(2.0, 3) to return NotImplemented instead of the expected 1.5. - -- Issue #9808: Implement os.getlogin for Windows. Patch by Jon Anglin. - -- Issue #9901: Destroying the GIL in Py_Finalize() can fail if some other - threads are still running. Instead, reinitialize the GIL on a second call to - Py_Initialize(). - -- All SyntaxErrors now have a column offset and therefore a caret when the error - is printed. - -- Issue #9252: PyImport_Import no longer uses a fromlist hack to return the - module that was imported, but instead gets the module from sys.modules. - -- Issue #9213: The range type_items now provides index() and count() methods, to - conform to the Sequence ABC. Patch by Daniel Urban and Daniel Stutzbach. - -- Issue #7994: Issue a PendingDeprecationWarning if object.__format__ is called - with a non-empty format string. This is an effort to future-proof user - code. If a derived class does not currently implement __format__ but later - adds its own __format__, it would most likely break user code that had - supplied a format string. This will be changed to a DeprecationWaring in - Python 3.3 and it will be an error in Python 3.4. - -- Issue #9828: Destroy the GIL in Py_Finalize(), so that it gets properly - re-created on a subsequent call to Py_Initialize(). The problem (a crash) - wouldn't appear in 3.1 or 2.7 where the GIL's structure is more trivial. - -- Issue #9210: Configure option --with-wctype-functions was removed. Using the - functions from the libc caused the methods .upper() and lower() to become - locale aware and created subtly wrong results. - -- Issue #9738: PyUnicode_FromFormat() and PyErr_Format() raise an error on a - non-ASCII byte in the format string. - -- Issue #4617: Previously it was illegal to delete a name from the local - namespace if it occurs as a free variable in a nested block. This limitation - of the compiler has been lifted, and a new opcode introduced (DELETE_DEREF). - -- Issue #9804: ascii() now always represents unicode surrogate pairs as a single - ``\UXXXXXXXX``, regardless of whether the character is printable or not. - Also, the "backslashreplace" error handler now joins surrogate pairs into a - single character on UCS-2 builds. - -- Issue #9757: memoryview objects get a release() method to release the - underlying buffer (previously this was only done when deallocating the - memoryview), and gain support for the context management protocol. - -- Issue #9797: pystate.c wrongly assumed that zero couldn't be a valid - thread-local storage key. - -Library -------- - -- Issue #2236: distutils' mkpath ignored the mode parameter. - -- Fix typo in one sdist option (medata-check). - -- Issue #9199: Fix incorrect use of distutils.cmd.Command.announce. - -- Issue #1718574: Fix options that were supposed to accept arguments but did - not in build_clib. - -- Issue #9437: Fix building C extensions with non-default LDFLAGS. - -- Issue #4661: email can now parse bytes input and generate either converted - 7bit output or bytes output. Email version bumped to 5.1.0. - -- Issue #1589: Add ssl.match_hostname(), to help implement server identity - verification for higher-level protocols. - -- Issue #9759: GzipFile now raises ValueError when an operation is attempted - after the file is closed. Patch by Jeffrey Finkelstein. - -- Issue #9042: Fix interaction of custom translation classes and caching in - gettext. - -- Issue #6706: asyncore.dispatcher now provides a handle_accepted() method - returning a (sock, addr) pair which is called when a connection has been - established with a new remote endpoint. This is supposed to be used as a - replacement for old handle_accept() and avoids the user to call accept() - directly. - -- Issue #9065: tarfile no longer uses "root" as the default for the uname and - gname field. - -- Issue #8980: Fixed a failure in distutils.command check that was shadowed by - an environment that does not have docutils. Patch by Arfrever. - -- Issue #1050268: parseaddr now correctly quotes double quote and backslash - characters that appear inside quoted strings in email addresses. - -- Issue #10004: quoprimime no longer generates a traceback when confronted with - invalid characters after '=' in a Q-encoded word. - -- Issue #1491: BaseHTTPServer nows send a ``100 Continue`` response before - sending a 200 OK for the Expect: 100-continue request header. - -- Issue #9360: Cleanup and improvements to the nntplib module. The API now - conforms to the philosophy of bytes and unicode separation in Python 3. A - test suite has also been added. - -- Issue #9962: GzipFile now has the peek() method. - -- Issue #9090: When a socket with a timeout fails with EWOULDBLOCK or EAGAIN, - retry the select() loop instead of bailing out. This is because select() can - incorrectly report a socket as ready for reading (for example, if it received - some data with an invalid checksum). - -- Issue #3612: Added new types to ctypes.wintypes. (CHAR and pointers) - -- Issue #9950: Fix socket.sendall() crash or misbehaviour when a signal is - received. Now sendall() properly calls signal handlers if necessary, and - retries sending if these returned successfully, including on sockets with a - timeout. - -- Issue #9947: logging: Fixed locking bug in stopListening. - -- Issue #9945: logging: Fixed locking bugs in addHandler/removeHandler. - -- Issue #9936: Fixed executable lines' search in the trace module. - -- Issue #9790: Rework imports necessary for samefile and sameopenfile - in ntpath. - -- Issue #9928: Properly initialize the types exported by the bz2 module. - -- Issue #1675951: Allow GzipFile to work with unseekable file objects. Patch by - Florian Festi. - -- Logging: Added QueueListener class to facilitate logging usage for - performance-critical threads. - -- Issue #9916: Add some missing errno symbols. - -- Issue #9877: Expose sysconfig.get_makefile_filename() - -- logging: Added hasHandlers() method to Logger and LoggerAdapter. - -- Issue #9908: Fix os.stat() on bytes paths under Windows 7. - -- Issue #2643: msync() is not called anymore when deallocating an open mmap - object, only munmap(). - -- logging: Changed LoggerAdapter implementation internally, to make it easier to - subclass in a useful way. - -- logging: hasHandlers method was added to Logger, and isEnabledFor, - getEffectiveLevel, hasHandlers and setLevel were added to LoggerAdapter. - LoggerAdapter was introduced into the unit tests for logging. - -- Issue #1686: Fix string.Template when overriding the pattern attribute. - -- Issue #9854: SocketIO objects now observe the RawIOBase interface in - non-blocking mode: they return None when an operation would block (instead of - raising an exception). - -- Issue #1730136: Fix the comparison between a tk.font.Font and an object of - another kind. - -- Issue #9441: logging has better coverage for rotating file handlers. - -- Issue #9865: collections.OrderedDict now has a __sizeof__ method. - -- Issue #9854: The default read() implementation in io.RawIOBase now handles - non-blocking readinto() returning None correctly. - -- Issue #1552: socket.socketpair() now returns regular socket.socket objects - supporting the whole socket API (rather than the "raw" _socket.socket - objects). - -- Issue #9853: Fix the signature of SSLSocket.recvfrom() and SSLSocket.sendto() - to match the corresponding socket methods. - -- Issue #9840: Added a decorator to reprlib for wrapping __repr__ methods to make - them handle recursive calls within the same thread. - -- logging: Enhanced HTTPHandler with secure and credentials initializers. - -- Issue #767645: Set os.path.supports_unicode_filenames to True on Mac OS X. - -- Issue #9837: The read() method of ZipExtFile objects (as returned by - ZipFile.open()) could return more bytes than requested. - -- Issue #9826: OrderedDict.__repr__ can now handle self-referential values: - d['x'] = d. - -- Issue #9825: Using __del__ in the definition of collections.OrderedDict made - it possible for the user to create self-referencing ordered dictionaries which - become permanently uncollectable GC garbage. Reinstated the Python 3.1 - approach of using weakref proxies so that reference cycles never get created - in the first place. - -- Issue #9579, #9580: Fix os.confstr() for value longer than 255 bytes and - encode the value with filesystem encoding and surrogateescape (instead of - utf-8 in strict mode) . Patch written by David Watson. - -- Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING - environment variable to set the filesystem encoding at Python startup. - sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. - -- Issue #9410: Various optimizations to the pickle module, leading to speedups - up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; - initial patch by Alexandre Vassalotti. - -- The pprint module now supports printing OrderedDicts in their given order - (formerly, it would sort the keys). - -- Logging: Added QueueHandler class to facilitate logging usage with - multiprocessing. - -- Issue #9707: Rewritten reference implementation of threading.local which is - friendlier towards reference cycles. This change is not normally visible - since an optimized C implementation (_thread._local) is used instead. - -- Issue #6394: os.getppid() is now supported on Windows. Note that it will - still return the id of the parent process after it has exited. This process - id may even have been reused by another unrelated process. - -- Issue #9792: In case of connection failure, socket.create_connection() would - swallow the exception and raise a new one, making it impossible to fetch the - original errno, or to filter timeout errors. Now the original error is - re-raised. - -- Issue #9758: When fcntl.ioctl() was called with mutable_flag set to True, and - the passed buffer was exactly 1024 bytes long, the buffer wouldn't be updated - back after the system call. Original patch by Brian Brazil. - -- Updates to the random module: - - * Document which parts of the module are guaranteed to stay the same across - versions and which parts are subject to change. - - * Update the seed() method to use all of the bits in a string instead of just - the hash value. This makes better use of the seed value and assures the - seeding is platform independent. Issue #7889. - - * Improved the random()-->integer algorithm used in choice(), shuffle(), - sample(), randrange(), and randint(). Formerly, it used int(n*random()) - which has a slight bias whenever n is not a power of two. Issue #9025. - - * Improved documentation of arguments to randrange(). Issue #9379. - -- collections.OrderedDict now supports a new method for repositioning keys to - either end. - -- Issue #9754: Similarly to assertRaises and assertRaisesRegexp, unittest test - cases now also have assertWarns and assertWarnsRegexp methods to check that a - given warning type was triggered by the code under test. - -- Issue #5506: BytesIO objects now have a getbuffer() method exporting a view of - their contents without duplicating them. The view is both readable and - writable. - -- Issue #7566: Implement os.path.sameopenfile for Windows. - -- Issue #9293: I/O streams now raise ``io.UnsupportedOperation`` when an - unsupported operation is attempted (for example, writing to a file open only - for reading). - -- hashlib has two new constant attributes: algorithms_guaranteed and - algorithms_available that respectively list the names of hash algorithms - guaranteed to exist in all Python implementations and the names of hash - algorithms available in the current process. - -- A new package ``concurrent.futures`` as defined by PEP 3148. - -C-API ------ - -- Add PyErr_SyntaxLocationEx, which supports passing a column offset. - -- Issue #9834: Don't segfault in PySequence_GetSlice, PySequence_SetSlice, or - PySequence_DelSlice when the object doesn't have any mapping operations - defined. - -Tools/Demos ------------ - -- Issue #9188: The gdb extension now handles correctly narrow (UCS2) as well as - wide (UCS4) unicode builds for both the host interpreter (embedded inside gdb) - and the interpreter under test. - -Tests ------ - -- Issue #9308: Added tests for importing encoded modules that do not - depend on specific stdlib modules being encoded in a certain way. - -- Issue #1051: Add a script (Lib/test/make_ssl_certs.py) to generate the custom - certificate and private key files used by SSL-related certs. - -- Issue #9978: Wait until subprocess completes initialization. (Win32KillTests - in test_os) - -- Issue #7110: regrtest now sends test failure reports and single-failure - tracebacks to stderr rather than stdout. - -- Issue #9628: fix runtests.sh -x option so more than one test can be excluded. - -- Issue #9899: Fix test_tkinter.test_font on various platforms. Patch by Ned - Deily. - -- Issue #9894: Do not hardcode ENOENT in test_subprocess. - -- Issue #9315: Added tests for the trace module. Patch by Eli Bendersky. - -- Issue #9323: Make test.regrtest.__file__ absolute, this was not always the - case when running profile or trace, for example. - -- Issue #9568: Fix test_urllib2_localnet on OS X 10.3. - -Build ------ - -- Issue #10062: Allow building on platforms which do not have sem_timedwait. - -- Issue #10054: Some platforms provide uintptr_t in inttypes.h. Patch by Akira - Kitada. - -- Issue #10055: Make json C89-compliant in UCS4 mode. - -- Issue #9552: Avoid unnecessary rebuild of OpenSSL. (Windows) - -- Issue #1633863: Don't ignore $CC under AIX. - -- Issue #9810: Compile bzip2 source files in Python's project file directly. It - used to be built with bzip2's makefile. - -- Issue #9848: Stopping trying to build _weakref in setup.py as it is a built-in - module. - -- Issue #9806: python-config now has an ``--extension-suffix`` option that - outputs the suffix for dynamic libraries including the ABI version name - defined by PEP 3149. - -- Issue #941346: Improve the build process under AIX and allow Python to be - built as a shared library. Patch by Sébastien Sablé. - -- Issue #4026: Make the fcntl extension build under AIX. Patch by Sébastien - Sablé. - -- Issue #9701: The MacOSX installer can patch the shell profile to ensure that - the "bin" directory inside the framework is on the shell's search path. This - feature now also supports the ZSH shell. - - -What's New in Python 3.2 Alpha 2? -================================= - -*Release date: 05-Sep-2010* - -Core and Builtins ------------------ - -- Issue #9225: Remove the ROT_FOUR and DUP_TOPX opcode, the latter replaced by - the new (and simpler) DUP_TOP_TWO. Performance isn't changed, but our - bytecode is a bit simplified. Patch by Demur Rumed. - -- Issue #9766: Rename poorly named variables exposed by _warnings to prevent - confusion with the proper variables names from 'warnings' itself. - -- Issue #9212: dict_keys and dict_items now provide the isdisjoint() method, to - conform to the Set ABC. Patch by Daniel Urban. - -- Issue #9737: Fix a crash when trying to delete a slice or an item from a - memoryview object. - -- Issue #9549: sys.setdefaultencoding() and PyUnicode_SetDefaultEncoding() are - now removed, since their effect was inexistent in 3.x (the default encoding is - hardcoded to utf-8 and cannot be changed). - -- Issue #7415: PyUnicode_FromEncodedObject() now uses the new buffer API - properly. Patch by Stefan Behnel. - -- Issue #5553: The Py_LOCAL_INLINE macro now results in inlining on most - platforms. Previously, it inlined only when using Microsoft Visual C. - -- Issue #9712: Fix tokenize on identifiers that start with non-ascii names. - -- Issue #9688: __basicsize__ and __itemsize__ must be accessed as Py_ssize_t. - -- Issue #9684: Added a definition for SIZEOF_WCHAR_T to PC/pyconfig.h, to match - the pyconfig.h generated by configure on other systems. - -- Issue #9666: Only catch AttributeError in hasattr(). All other exceptions that - occur during attribute lookup are now propagated to the caller. - -- Issue #8622: Add PYTHONFSENCODING environment variable to override the - filesystem encoding. - -- Issue #5127: The C functions that access the Unicode Database now accept and - return characters from the full Unicode range, even on narrow unicode builds - (Py_UNICODE_TOLOWER, Py_UNICODE_ISDECIMAL, and others). A visible difference - in Python is that unicodedata.numeric() now returns the correct value for - large code points, and repr() may consider more characters as printable. - -- Issue #9425: Create PyModule_GetFilenameObject() function to get the filename - as a unicode object, instead of a byte string. Function needed to support - unencodable filenames. Deprecate PyModule_GetFilename() in favor on the new - function. - -- Issue #8063: Call _PyGILState_Init() earlier in Py_InitializeEx(). - -- Issue #9612: The set object is now 64-bit clean under Windows. - -- Issue #8202: sys.argv[0] is now set to '-m' instead of '-c' when searching for - the module file to be executed with the -m command line option. - -- Issue #9599: Create PySys_FormatStdout() and PySys_FormatStderr() functions to - write a message formatted by PyUnicode_FromFormatV() to sys.stdout and - sys.stderr. - -- Issue #9542: Create PyUnicode_FSDecoder() function, a ParseTuple converter: - decode bytes objects to unicode using PyUnicode_DecodeFSDefaultAndSize(); str - objects are output as-is. - -- Issue #9203: Computed gotos are now enabled by default on supported compilers - (which are detected by the configure script). They can still be disable - selectively by specifying --without-computed-gotos. - -- Issue #9425: Create PyErr_WarnFormat() function, similar to PyErr_WarnEx() but - use PyUnicode_FromFormatV() to format the warning message. - -- Issue #8530: Prevent stringlib fastsearch from reading beyond the front of an - array. - -- Issue #5319: Print an error if flushing stdout fails at interpreter shutdown. - -- Issue #9337: The str() of a float or complex number is now identical to its - repr(). - -- Issue #9416: Fix some issues with complex formatting where the output with no - type specifier failed to match the str output: - - - format(complex(-0.0, 2.0), '-') omitted the real part from the output, - - format(complex(0.0, 2.0), '-') included a sign and parentheses. - -Extension Modules ------------------ - -- Issue #8013: time.asctime and time.ctime no longer call system - asctime and ctime functions. The year range for time.asctime is now - 1900 through maxint. The range for time.ctime is the same as for - time.localtime. The string produced by these functions is longer - than 24 characters when year is greater than 9999. - -- Issue #6608: time.asctime is now checking struct tm fields its input - before passing it to the system asctime. Patch by MunSic Jeong. - -- Issue #8734: Avoid crash in msvcrt.get_osfhandle() when an invalid file - descriptor is provided. Patch by Pascal Chambon. - -- Issue #7736: Release the GIL around calls to opendir() and closedir() in the - posix module. Patch by Marcin Bachry. - -- Issue #4835: make PyLong_FromSocket_t() and PyLong_AsSocket_t() private to the - socket module, and fix the width of socket descriptors to be correctly - detected under 64-bit Windows. - -- Issue #1027206: Support IDNA in gethostbyname, gethostbyname_ex, getaddrinfo - and gethostbyaddr. getnameinfo is now restricted to numeric addresses as - input. - -- Issue #9214: Set operations on a KeysView or ItemsView in collections now - correctly return a set. Patch by Eli Bendersky. - -- Issue #5737: Add Solaris-specific mnemonics in the errno module. Patch by - Matthew Ahrens. - -- Restore GIL in nis_cat in case of error. Decode NIS data to fs encoding, using - the surrogate error handler. - -- Issue #665761: ``functools.reduce()`` will no longer mask exceptions other - than ``TypeError`` raised by the iterator argument. - -- Issue #9570: Use PEP 383 decoding in os.mknod and os.mkfifo. - -- Issue #6915: Under Windows, os.listdir() didn't release the Global Interpreter - Lock around all system calls. Original patch by Ryan Kelly. - -- Issue #8524: Add a detach() method to socket objects, so as to put the socket - into the closed state without closing the underlying file descriptor. - -- Issue #477863: Emit a ResourceWarning at shutdown if gc.garbage is not empty. - -- Issue #6869: Fix a refcount problem in the _ctypes extension. - -- Issue #5504: ctypes should now work with systems where mmap can't be - PROT_WRITE and PROT_EXEC. - -- Issue #9507: Named tuple repr will now automatically display the right name in - a tuple subclass. - -- Issue #9324: Add parameter validation to signal.signal on Windows in order to - prevent crashes. - -- Issue #9526: Remove some outdated (int) casts that were preventing the array - module from working correctly with arrays of more than 2**31 elements. - -- Fix memory leak in ssl._ssl._test_decode_cert. - -- Issue #8065: Fix memory leak in readline module (from failure to free the - result of history_get_history_state()). - -- Issue #9450: Fix memory leak in readline.replace_history_item and - readline.remove_history_item for readline version >= 5.0. - -- Issue #8105: Validate file descriptor passed to mmap.mmap on Windows. - -- Issue #8046: Add context management protocol support and .closed property to mmap - objects. - -Library -------- - -- Issue #7451: Improve decoding performance of JSON objects, and reduce the - memory consumption of said decoded objects when they use the same strings as - keys. - -- Issue #1100562: Fix deep-copying of objects derived from the list and dict - types. Patch by Michele Orrù and Björn Lindqvist. - -- Issue #9753: Fixed socket.dup, which did not always work correctly on Windows. - -- Issue #9421: Made the get methods consistently accept the vars and - default arguments on all parser classes. - -- Issue #7005: Fixed output of None values for RawConfigParser.write and - ConfigParser.write. - -- Issue #8990: array.fromstring() and array.tostring() get renamed to - frombytes() and tobytes(), respectively, to avoid confusion. Furthermore, - array.frombytes(), array.extend() as well as the array.array() constructor now - accept bytearray objects. Patch by Thomas Jollans. - -- Issue #808164: Fixed socket.close to avoid references to globals, to avoid - issues when socket.close is called from a __del__ method. - -- Issue #9706: ssl module provides a better error handling in various - circumstances. - -- Issue #1868: Eliminate subtle timing issues in thread-local objects by getting - rid of the cached copy of thread-local attribute dictionary. - -- Issue #1512791: In setframerate() in the wave module, non-integral frame rates - are rounded to the nearest integer. - -- Issue #8797: urllib2 does a retry for Basic Authentication failure instead of - falling into recursion. - -- Issue #1194222: email.utils.parsedate now returns RFC2822 compliant four - character years even if the message contains RFC822 two character years. - -- Issue #8750: Fixed MutableSet's methods to correctly handle reflexive - operations on its self, namely x -= x and x ^= x. - -- Issue #9129: smtpd.py is vulnerable to DoS attacks deriving from missing error - handling when accepting a new connection. - -- Issue #9601: ftplib now provides a workaround for non-compliant - implementations such as IIS shipped with Windows server 2003 returning invalid - response codes for MKD and PWD commands. - -- Issue #658749: asyncore's connect() method now correctly interprets winsock - errors. - -- Issue #9501: Fixed logging regressions in cleanup code. - -- Fix functools.total_ordering() to skip methods inherited from object. - -- Issue #9572: Importlib should not raise an exception if a directory it thought - it needed to create was done concurrently by another process. - -- Issue #9617: Signals received during a low-level write operation aren't - ignored by the buffered IO layer anymore. - -- Issue #843590: Make "macintosh" an alias to the "mac_roman" encoding. - -- Create os.fsdecode(): decode from the filesystem encoding with surrogateescape - error handler, or strict error handler on Windows. - -- Issue #3488: Provide convenient shorthand functions ``gzip.compress`` and - ``gzip.decompress``. Original patch by Anand B. Pillai. - -- Issue #8807: poplib.POP3_SSL class now accepts a context parameter, which is a - ssl.SSLContext object allowing bundling SSL configuration options, - certificates and private keys into a single (potentially long-lived) - structure. - -- Issue #8866: parameters passed to socket.getaddrinfo can now be specified as - single keyword arguments. - -- Address XXX comment in dis.py by having inspect.py prefer to reuse the dis.py - compiler flag values over defining its own. - -- Issue #9147: Added dis.code_info() which is similar to show_code() but returns - formatted code information in a string rather than displaying on screen. - -- Issue #9567: functools.update_wrapper now adds a __wrapped__ attribute - pointing to the original callable. - -- Issue #3445: functools.update_wrapper now tolerates missing attributes on - wrapped callables. - -- Issue #5867: Add abc.abstractclassmethod and abc.abstractstaticmethod. - -- Issue #9605: posix.getlogin() decodes the username with file filesystem - encoding and surrogateescape error handler. Patch written by David Watson. - -- Issue #9604: posix.initgroups() encodes the username using the fileystem - encoding and surrogateescape error handler. Patch written by David Watson. - -- Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name - using the filesystem encoding and surrogateescape error handler. Patch written - by David Watson. - -- Issue #7647: The posix module now has the ST_RDONLY and ST_NOSUID constants, - for use with the statvfs() function. Patch by Adam Jackson. - -- Issue #8688: MANIFEST files created by distutils now include a magic comment - indicating they are generated. Manually maintained MANIFESTs without this - marker will not be overwritten or removed. - -- Issue #7467: when reading a file from a ZIP archive, its CRC is checked and a - BadZipfile error is raised if it doesn't match (as used to be the case in - Python 2.5 and earlier). - -- Issue #9550: a BufferedReader could issue an additional read when the original - read request had been satisfied, which could block indefinitely when the - underlying raw IO channel was e.g. a socket. Report and original patch by - Jason V. Miller. - -- Issue #3757: thread-local objects now support cyclic garbage collection. - Thread-local objects involved in reference cycles will be deallocated timely - by the cyclic GC, even if the underlying thread is still running. - -- Issue #9452: Add read_file, read_string, and read_dict to the configparser - API; new source attribute to exceptions. - -- Issue #6231: Fix xml.etree.ElementInclude to include the tail of the current - node. - -- Issue #8047: Fix the xml.etree serializer to return bytes by default. Use - ``encoding="unicode"`` to generate a Unicode string. - -- Issue #8280: urllib2's Request method will remove fragments in the url. This - is how it is supposed to work, wget and curl do the same. Previous behavior - was wrong. - -- Issue #6683: For SMTP logins we now try all authentication methods advertised - by the server. Many servers are buggy and advertise authentication methods - they do not support in reality. - -- Issue #8814: function annotations (the ``__annotations__`` attribute) are now - included in the set of attributes copied by default by functools.wraps and - functools.update_wrapper. Patch by Terrence Cole. - -- Issue #2944: asyncore doesn't handle connection refused correctly. - -- Issue #4184: Private attributes on smtpd.SMTPChannel made public and deprecate - the private attributes. Add tests for smtpd module. - -- Issue #3196: email header decoding is now forgiving if an RFC2047 encoded word - encoded in base64 is lacking padding. - -- Issue #9444: Argparse now uses the first element of prefix_chars as the option - character for the added 'h/help' option if prefix_chars does not contain a - '-', instead of raising an error. - -- Issue #7372: Fix pstats regression when stripping paths from profile data - generated with the profile module. - -- Issue #9428: Fix running scripts with the profile/cProfile modules from the - command line. - -- Issue #7781: Fix restricting stats by entry counts in the pstats interactive - browser. - -- Issue #9209: Do not crash in the pstats interactive browser on invalid regular - expressions. - -- Update collections.OrderedDict to match the implementation in Py2.7 (based on - lists instead of weakly referenced Link objects). - -- Issue #8397: Raise an error when attempting to mix iteration and regular reads - on a BZ2File object, rather than returning incorrect results. - -- Issue #9448: Fix a leak of OS resources (mutexes or semaphores) when - re-initializing a buffered IO object by calling its ``__init__`` method. - -- Issue #1713: Fix os.path.ismount(), which returned true for symbolic links - across devices. - -- Issue #8826: Properly load old-style "expires" attribute in http.cookies. - -- Issue #1690103: Fix initial namespace for code run with trace.main(). - -- Issue #7395: Fix tracebacks in pstats interactive browser. - -- Issue #8230: Fix Lib/test/sortperf.py. - -- Issue #8620: when a cmd.Cmd() is fed input that reaches EOF without a final - newline, it no longer truncates the last character of the last command line. - -- Issue #5146: Handle UID THREAD command correctly in imaplib. - -- Issue #5147: Fix the header generated for cookie files written by - http.cookiejar.MozillaCookieJar. - -- Issue #8198: In pydoc, output all help text to the correct stream when - sys.stdout is reassigned. - -- Issue #7909: Do not touch paths with the special prefixes ``\\.\`` or ``\\?\`` - in ntpath.normpath(). - -- Issue #1286: Allow using fileinput.FileInput as a context manager. - -- Add lru_cache() decorator to the functools module. - -Tools/Demos ------------ - -- Fix ``Tools/scripts/checkpyc.py`` after PEP 3147. - -- Issue #8867: Fix ``Tools/scripts/serve.py`` to work with files containing - non-ASCII content. - -Tests ------ - -- Issue #9601: Provide a test case for ftplib.parse257. - -- Issue #8857: Provide a test case for socket.getaddrinfo. - -- Issue #7564: Skip test_ioctl if another process is attached to /dev/tty. - -- Issue #8433: Fix test_curses failure with newer versions of ncurses. - -- Issue #9496: Provide a test suite for the rlcompleter module. Patch by - Michele Orrù. - -- Issue #8687: provide a test suite for sched.py module. - -Build ------ - -- Issue #1303434: Generate ZIP file containing all PDBs. - -- Issue #9193: PEP 3149 is accepted. - -- Issue #3101: Helper functions _add_one_to_index_C() and _add_one_to_index_F() - become _Py_add_one_to_index_C() and _Py_add_one_to_index_F(), respectively. - -- Issue #9700: define HAVE_BROKEN_POSIX_SEMAPHORES under AIX 6.x. Patch by - Sébastien Sablé. - -- Don't run pgen twice when using make -j. - - -What's New in Python 3.2 Alpha 1? -================================= - -*Release date: 01-Aug-2010* - -Core and Builtins ------------------ - -- Issue #8991: convertbuffer() rejects discontigious buffers. - -- Issue #7616: Fix copying of overlapping memoryview slices with the Intel - compiler. - -- Issue #8413: structsequence now subclasses tuple. - -- Issue #8271: during the decoding of an invalid UTF-8 byte sequence, only the - start byte and the continuation byte(s) are now considered invalid, instead of - the number of bytes specified by the start byte. E.g.: - '\xf1\x80AB'.decode('utf-8', 'replace') now returns u'\ufffdAB' and replaces - with U+FFFD only the start byte ('\xf1') and the continuation byte ('\x80') - even if '\xf1' is the start byte of a 4-bytes sequence. Previous versions - returned a single u'\ufffd'. - -- Issue #9011: A negated imaginary literal (e.g., "-7j") now has real part -0.0 - rather than 0.0. So "-7j" is now exactly equivalent to "-(7j)". - -- Be more specific in error messages about positional arguments. - -- Issue #8949: "z" format of PyArg_Parse*() functions doesn't accept bytes - objects, as described in the documentation. - -- Issue #6543: Write the traceback in the terminal encoding instead of utf-8. - Fix the encoding of the modules filename. Patch written by Amaury Forgeot - d'Arc. - -- Issue #9011: Remove buggy and unnecessary (in 3.x) ST->AST compilation code - dealing with unary minus applied to a constant. The removed code was mutating - the ST, causing a second compilation to fail. - -- Issue #850997: mbcs encoding (Windows only) handles errors argument: strict - mode raises unicode errors. The encoder only supports "strict" and "replace" - error handlers, the decoder only supports "strict" and "ignore" error - handlers. Patch written by Mark Hammond. - -- Issue #8850: Remove "w" and "w#" formats from PyArg_Parse*() functions, use - "w*" format instead. Add tests for "w*" format. - -- Issue #8592: PyArg_Parse*() functions raise a TypeError for "y", "u" and "Z" - formats if the string contains a null byte/character. Write unit tests for - string formats. - -- Issue #7490: To facilitate sharing of doctests between 2.x and 3.x test - suites, the IGNORE_EXCEPTION_DETAIL directive now also ignores the module - location of the raised exception. - -- Issue #8969: On Windows, use mbcs codec in strict mode to encode and decode - filenames and enable os.fsencode(). - -- Issue #9058: Remove assertions about INT_MAX in UnicodeDecodeError. - -- Issue #8941: Decoding big endian UTF-32 data in UCS-2 builds could crash the - interpreter with characters outside the Basic Multilingual Plane (higher than - 0x10000). - -- Issue #8950: (See also issue #5080). Py_ArgParse*() functions now raise - TypeError instead of giving a DeprecationWarning when a float is parsed using - the 'L' code (for long long). (All other integer codes already raise - TypeError in this case.) - -- Issue #8922: Normalize the encoding name in PyUnicode_AsEncodedString() to - enable shortcuts for upper case encoding name. Add also a shortcut for - "iso-8859-1" in PyUnicode_AsEncodedString() and PyUnicode_Decode(). - -- Issue #8838: Remove codecs.charbuffer_encode() function. The buffer protocol - doesn't support "char buffer" anymore in Python 3. - -- Issue #8339: Remove "t#" format of PyArg_Parse*() functions, use "s#" or "s*" - instead. codecs.charbuffer_encode() now accepts modifiable buffer objects - like bytearray. - -- Issue #8837: Remove "O?" format of PyArg_Parse*() functions. The format is no - used anymore and it was never documented. - -- In str.format(), raise a ValueError when indexes to arguments are too large. - -- Issue #2844: Make int('42', n) consistently raise ValueError for invalid - integers n (including n = -909). - -- Issue #8188: Introduce a new scheme for computing hashes of numbers (instances - of int, float, complex, decimal.Decimal and fractions.Fraction) that makes it - easy to maintain the invariant that hash(x) == hash(y) whenever x and y have - equal value. - -- Issue #8748: Fix two issues with comparisons between complex and integer - objects. (1) The comparison could incorrectly return True in some cases - (2**53+1 == complex(2**53) == 2**53), breaking transitivity of equality. - (2) The comparison raised an OverflowError for large integers, leading to - unpredictable exceptions when combining integers and complex objects in sets - or dicts. - -- Issue #8766: Initialize _warnings module before importing the first module. - Fix a crash if an empty directory called "encodings" exists in sys.path. - -- Issue #8589: Decode PYTHONWARNINGS environment variable with the file system - encoding and surrogateescape error handler instead of the locale encoding to - be consistent with os.environ. Add PySys_AddWarnOptionUnicode() function. - -- PyObject_Dump() encodes unicode objects to utf8 with backslashreplace (instead - of strict) error handler to escape surrogates. - -- Issue #8715: Create PyUnicode_EncodeFSDefault() function: Encode a Unicode - object to Py_FileSystemDefaultEncoding with the "surrogateescape" error - handler, and return bytes. If Py_FileSystemDefaultEncoding is not set, fall - back to UTF-8. - -- Enable shortcuts for common encodings in PyUnicode_AsEncodedString() for any - error handler, not only the default error handler (strict). - -- Issue #8610: Load file system codec at startup, and display a fatal error on - failure. Set the file system encoding to utf-8 (instead of None) if getting - the locale encoding failed, or if nl_langinfo(CODESET) function is missing. - -- PyFile_FromFd() uses PyUnicode_DecodeFSDefault() instead of - PyUnicode_FromString() to support surrogates in the filename and use the right - encoding. - -- Issue #7507: Quote "!" in pipes.quote(); it is special to some shells. - -- PyUnicode_DecodeFSDefaultAndSize() uses surrogateescape error handler. - -- Issue #8419: Prevent the dict constructor from accepting non-string keyword - arguments. - -- Issue #8124: PySys_WriteStdout() and PySys_WriteStderr() don't execute - indirectly Python signal handlers anymore because mywrite() ignores exceptions - (KeyboardInterrupt). - -- Issue #8092: Fix PyUnicode_EncodeUTF8() to support error handler producing - unicode string (eg. backslashreplace). - -- Issue #8485: PyUnicode_FSConverter() doesn't accept byteearray objects - anymore, you have to convert your bytearray filenames to bytes. - -- Issue #7332: Remove the 16KB stack-based buffer in - PyMarshal_ReadLastObjectFromFile, which doesn't bring any noticeable benefit - compared to the dynamic memory allocation fallback. Patch by Charles-François - Natali. - -- Issue #8417: Raise an OverflowError when an integer larger than sys.maxsize is - passed to bytes or bytearray. - -- Issue #7301: Add environment variable $PYTHONWARNINGS. - -- Issue #8329: Don't return the same lists from select.select when no fds are - changed. - -- Issue #8259: 1L << (2**31) no longer produces an 'outrageous shift error' on - 64-bit machines. The shift count for either left or right shift is permitted - to be up to sys.maxsize. - -- Ensure that tokenization of identifiers is not affected by locale. - -- Issue #1222585: Added LDCXXSHARED for C++ support. Patch by Arfrever. - -- Raise a TypeError when trying to delete a T_STRING_INPLACE struct member. - -- Issue #8211: Save/restore CFLAGS around AC_PROG_CC in configure.in, in case it - is set. - -- Issue #8226: sys.setfilesystemencoding() raises a LookupError if the encoding - is unknown. - -- Issue #1583863: A str subclass can now override the __str__ method. - -- Issue #8014: Setting a T_UINT or T_PYSSIZET attribute of an object with - PyMemberDefs could produce an internal error; raise TypeError instead. - -- Issue #7845: Rich comparison methods on the complex type now return - NotImplemented rather than raising a TypeError when comparing with an - incompatible type; this allows user-defined classes to implement their own - comparisons with complex. - -- Issue #3137: Don't ignore errors at startup, especially a keyboard interrupt - (SIGINT). If an error occurs while importing the site module, the error is - printed and Python exits. Initialize the GIL before importing the site module. - -- Issue #7173: Generator finalization could invalidate sys.exc_info(). - -- Issue #7544: Preallocate thread memory before creating the thread to avoid a - fatal error in low memory condition. - -- Issue #7820: The parser tokenizer restores all bytes in the right if the BOM - check fails. - -- Handle errors from looking up __prepare__ correctly. - -- Issue #5939: Add additional runtime checking to ensure a valid capsule in - Modules/_ctypes/callproc.c. - -- Issue #7309: Fix unchecked attribute access when converting - UnicodeEncodeError, UnicodeDecodeError, and UnicodeTranslateError to strings. - -- Issue #6902: Fix problem with built-in types format incorrectly with 0 - padding. - -- Issue #7988: Fix default alignment to be right aligned for complex.__format__. - Now it matches other numeric types. - -- Issue #5988: Remove deprecated functions PyOS_ascii_formatd, - PyOS_ascii_strtod, and PyOS_ascii_atof. Use PyOS_double_to_string and - PyOS_string_to_double instead. See issue #5835 for the original deprecations. - -- Issue #7385: Fix a crash in `MemoryView_FromObject` when `PyObject_GetBuffer` - fails. Patch by Florent Xicluna. - -- Issue #7788: Fix an interpreter crash produced by deleting a list slice with - very large step value. - -- Issue #7766: Change sys.getwindowsversion() return value to a named tuple and - add the additional members returned in an OSVERSIONINFOEX structure. The new - members are service_pack_major, service_pack_minor, suite_mask, and - product_type. - -- Issue #7561: Operations on empty bytearrays (such as `int(bytearray())`) could - crash in many places because of the PyByteArray_AS_STRING() macro returning - NULL. The macro now returns a statically allocated empty string instead. - -- Issue #6690: Optimize the bytecode for expressions such as `x in {1, 2, 3}`, - where the right hand operand is a set of constants, by turning the set into a - frozenset and pre-building it as a constant. The comparison operation is made - against the constant instead of building a new set each time it is executed (a - similar optimization already existed which turned a list of constants into a - pre-built tuple). Patch and additional tests by Dave Malcolm. - -- Issue #7622: Improve the split(), rsplit(), splitlines() and replace() methods - of bytes, bytearray and unicode objects by using a common implementation based - on stringlib's fast search. Patch by Florent Xicluna. - -- Issue #7632: Fix various str -> float conversion bugs present in 2.7 alpha 2, - including: (1) a serious 'wrong output' bug that could occur for long (> 40 - digit) input strings, (2) a crash in dtoa.c that occurred in debug builds when - parsing certain long numeric strings corresponding to subnormal values, (3) a - memory leak for some values large enough to cause overflow, and (4) a number - of flaws that could lead to incorrectly rounded results. - -- The __complex__ method is now looked up on the class of instances to make it - consistent with other special methods. - -- Issue #7462: Implement the stringlib fast search algorithm for the `rfind`, - `rindex`, `rsplit` and `rpartition` methods. Patch by Florent Xicluna. - -- Issue #7604: Deleting an unset slotted attribute did not raise an - AttributeError. - -- Issue #7534: Fix handling of IEEE specials (infinities, nans, negative zero) - in ** operator. The behaviour now conforms to that described in C99 Annex F. - -- Issue #1811: improve accuracy and cross-platform consistency for true division - of integers: the result of a/b is now correctly rounded for ints a and b (at - least on IEEE 754 platforms), and in particular does not depend on the - internal representation of an int. - -- Issue #6834: replace the implementation for the 'python' and 'pythonw' - executables on OSX. - - These executables now work properly with the arch(1) command: ``arch -ppc - python`` will start a universal binary version of python in PPC mode (unlike - previous releases). - -- Issue #7466: Segmentation fault when the garbage collector is called in the - middle of populating a tuple. Patch by Florent Xicluna. - -- Issue #7419: setlocale() could crash the interpreter on Windows when called - with invalid values. - -- Issue #6077: On Windows, files opened with tempfile.TemporaryFile in "wt+" - mode would appear truncated on the first '0x1a' byte (aka. Ctrl+Z). - -- Issue #7085: Fix crash when importing some extensions in a thread on MacOSX - 10.6. - -- Issue #1757126: Fix the cyrillic-asian alias for the ptcp154 encoding. - -- Issue #6970: Remove redundant calls when comparing objects that don't - implement the relevant rich comparison methods. - -- Issue #7298: Fixes for range and reversed(range(...)). Iteration over - range(a, b, c) incorrectly gave an empty iterator when a, b and c fit in C - long but the length of the range did not. Also fix several cases where - reversed(range(a, b, c)) gave wrong results, and fix a refleak for - reversed(range(a, b, c)) with large arguments. - -- Issue #7244: itertools.izip_longest() no longer ignores exceptions raised - during the formation of an output tuple. - -- Issue #3297: On wide unicode builds, do not split unicode characters into - surrogates. - -- Remove length limitation when constructing a complex number from a string. - -- Issue #1087418: Boost performance of bitwise operations for longs. - -- Support for AtheOS has been completely removed from the code base. It was - disabled since Python 3.0. - -- Support for several legacy threading libraries has been disabled. These - libraries are: Mach C threads, SunOS LWP, GNU pth, Irix threads. Support code - will be entirely removed in 3.3. - -- Support for OSF* has been disabled. If nobody stands up, support will be - removed in 3.3. See . - -- Peephole constant folding had missed UNARY_POSITIVE. - -- Issue #1722344: threading._shutdown() is now called in Py_Finalize(), which - fixes the problem of some exceptions being thrown at shutdown when the - interpreter is killed. Patch by Adam Olsen. - -- Issue #7147: Remove support for compiling Python without complex number - support. - -- Issue #7120: logging: Removed import of multiprocessing which is causing crash - in GAE. - -- Issue #1754094: Improve the stack depth calculation in the compiler. There - should be no other effect than a small decrease in memory use. Patch by - Christopher Tur Lesniewski-Laas. - -- Issue #7065: Fix a crash in bytes.maketrans and bytearray.maketrans when using - byte values greater than 127. Patch by Derk Drukker. - -- Issue #1571184: The Unicode database contains properties for more characters. - The tables for code points representing numeric values, white spaces or line - breaks are now generated from the official Unicode Character Database files, - and include information from the Unihan.txt file. - -- Issue #7019: Raise ValueError when unmarshalling bad long data, instead of - producing internally inconsistent Python longs. - -- Issue #6990: Fix threading.local subclasses leaving old state around after a - reference cycle GC which could be recycled by new locals. - -- Issue #5460: Fix an ambiguity in the grammar. - -- Issue #1766304: Improve performance of membership tests on range objects. - -- Issue #6713: Improve performance of integer -> string conversions. - -- Issue #6846: Fix bug where bytearray.pop() returns negative integers. - -- Issue #6750: A text file opened with io.open() could duplicate its output when - writing from multiple threads at the same time. - -- Issue #6707: dir() on an uninitialized module caused a crash. - -- Issue #6540: Fixed crash for bytearray.translate() with invalid parameters. - -- Issue #6573: set.union() stopped processing inputs if an instance of self - occurred in the argument chain. - -- Issue #6070: On posix platforms import no longer copies the execute bit from - the .py file to the .pyc file if it is set. - -- Issue #1616979: Added the cp720 (Arabic DOS) encoding. - -- Issue #6428: Since Python 3.0, the __bool__ method must return a bool object, - and not an int. Fix the corresponding error message, and the documentation. - -- The deprecated PyCObject has been removed. - -- Issue #6347: Include inttypes.h as well as stdint.h in pyport.h. This fixes a - build failure on HP-UX: int32_t and uint32_t are defined in inttypes.h instead - of stdint.h on that platform. - -- Issue #6373: Fixed a SystemError when encoding with the latin-1 codec and the - 'surrogateescape' error handler, a string which contains unpaired surrogates. - -- Issue #4856: Remove checks for win NT. - -- Issue #6687: PyBytes_FromObject() no longer accepts an integer as its argument - to construct a null-initialized bytes object. - -- Issue #1023290: Add from_bytes() and to_bytes() methods to integers. These - methods allow the conversion of integers to bytes, and vice-versa. - -- Issue #7382: Fix bug in bytes.__getnewargs__ that prevented bytes instances - from being copied with copy.copy(), and bytes subclasses from being pickled - properly. - -- Code objects now support weak references. - -- Issue #7072: isspace(0xa0) is true on Mac OS X. - -- Issue #8084: PEP 370 now conforms to system conventions for framework builds - on MacOS X. That is, "python setup.py install --user" will install into - "~/Library/Python/2.7" instead of "~/.local". - -C-API ------ - -- Issue #2443: A new macro, `Py_VA_COPY`, copies the state of the - variable argument list. `Py_VA_COPY` is equivalent to C99 - `va_copy`, but available on all python platforms. - -- PySlice_GetIndicesEx now clips the step to [-PY_SSIZE_T_MAX, PY_SSIZE_T_MAX] - instead of [-PY_SSIZE_T_MAX-1, PY_SSIZE_T_MAX]. This makes it safe to do - "step = -step" when reversing a slice. - -- Issue #5753: A new C API function, `PySys_SetArgvEx`, allows embedders of the - interpreter to set sys.argv without also modifying sys.path. This helps fix - `CVE-2008-5983 - `_. - -- Add PyArg_ValidateKeywordArguments, which checks if all keyword arguments are - strings in an efficient manner. - -- Issue #8276: PyEval_CallObject() is now only available in macro form. The - function declaration, which was kept for backwards compatibility reasons, is - now removed (the macro was introduced in 1997!). - -- Issue #7767: New function PyLong_AsLongLongAndOverflow added, analogous to - PyLong_AsLongAndOverflow. - -- Make PyUnicode_CompareWithASCIIString return not equal if the Python string - has '\0' at the end. - -- Issue #5080: The argument parsing functions PyArg_ParseTuple, - PyArg_ParseTupleAndKeywords, PyArg_VaParse, PyArg_VaParseTupleAndKeywords and - PyArg_Parse now raise a DeprecationWarning for float arguments passed with the - 'L' format code. This will become a TypeError in a future version of Python, - to match the behaviour of the other integer format codes. - -- Issue #7033: Function ``PyErr_NewExceptionWithDoc()`` added. - -- Issue #7414: 'C' code wasn't being skipped properly (for keyword arguments) in - PyArg_ParseTupleAndKeywords. - -- Issue #7228: Add '%lld' and '%llu' support to PyString_FromFormat(V) and - PyErr_Format, on machines with HAVE_LONG_LONG defined. - -- Issue #6151: Made PyDescr_COMMON conform to standard C (like PyObject_HEAD in - PEP 3123). The PyDescr_TYPE and PyDescr_NAME macros should be used for - accessing the d_type and d_name members of structures using PyDescr_COMMON. - -- Issue #6405: Remove duplicate type declarations in descrobject.h. - -- The code flags for old __future__ features are now available again. - -- Issue #5954: Add a PyFrame_GetLineNumber() function to replace most uses of - PyCode_Addr2Line(). - -- Issue #5959: Add a PyCode_NewEmpty() function to create a new empty code - object at a specified file, function, and line number. - -- Issue #1419652: Change the first argument to PyImport_AppendInittab() to - ``const char *`` as the string is stored beyond the call. - -- Issue #2422: When compiled with the ``--with-valgrind`` option, the pymalloc - allocator will be automatically disabled when running under Valgrind. This - gives improved memory leak detection when running under Valgrind, while taking - advantage of pymalloc at other times. - -Library -------- - -- In pdb, when Ctrl-C is entered while defining commands for a breakpoint, the - old commands are restored. - -- For traceback debugging, the pdb listing now also shows the locations where - the exception was originally (re)raised, if it differs from the last line - executed (e.g. in case of finally clauses). - -- The pdb command "source" has been added. It displays the source code for a - given object, if possible. - -- The pdb command "longlist" has been added. It displays the whole source code - for the current function. - -- Issue #1503502: Make pdb.Pdb easier to subclass by putting message and error - output into methods. - -- Issue #809887: Make the output of pdb's breakpoint deletions more consistent; - emit a message when a breakpoint is enabled or disabled. - -- Issue #5294: Fix the behavior of pdb's "continue" command when called in the - top-level debugged frame. - -- Issue #5727: Restore the ability to use readline when calling into pdb in - doctests. - -- Issue #6719: In pdb, do not stop somewhere in the encodings machinery if the - source file to be debugged is in a non-builtin encoding. - -- Issue #8048: Prevent doctests from failing when sys.displayhook has been - reassigned. - -- Issue #8015: In pdb, do not crash when an empty line is entered as a - breakpoint command. - -- In pdb, allow giving a line number to the "until" command. - -- Issue #1437051: For pdb, allow "continue" and related commands in .pdbrc - files. Also, add a command-line option "-c" that runs a command as if given - in .pdbrc. - -- Issue #4179: In pdb, allow "list ." as a command to return to the currently - debugged line. - -- Issue #4108: In urllib.robotparser, if there are multiple ``User-agent: *`` - entries, consider the first one. - -- Issue #6630: Allow customizing regex flags when subclassing the - string.Template class. - -- Issue #9411: Allow specifying an encoding for config files in the configparser - module. - -- Issue #1682942: Improvements to configparser: support alternate delimiters, - alternate comment prefixes and empty lines in values. - -- Issue #9354: Provide getsockopt() in asyncore's file_wrapper. - -- Issue #8966: ctypes: Remove implicit bytes-unicode conversion. - -- Issue #9378: python -m pickle will now load and display the - first object in the pickle file. - -- Issue #4770: Restrict binascii module to accept only bytes (as specified). - And fix the email package to encode to ASCII instead of ``raw-unicode-escape`` - before ASCII-to-binary decoding. - -- Issue #9384: ``python -m tkinter`` will now display a simple demo applet. - -- The default size of the re module's compiled regular expression cache has been - increased from 100 to 500 and the cache replacement policy has changed from - simply clearing the entire cache on overflow to forgetting the least recently - used cached compiled regular expressions. This is a performance win for - applications that use a lot of regular expressions and limits the impact of - the performance hit anytime the cache is exceeded. - -- Issue #7113: Speed up loading in configparser. Patch by Łukasz Langa. - -- Issue #9032: XML-RPC client retries the request on EPIPE error. The EPIPE - error occurs when the server closes the socket and the client sends a big - XML-RPC request. - -- Issue #4629: getopt raises an error if an argument ends with "=", whereas - getopt doesn't accept a value (eg. --help= is rejected if getopt uses - ['help='] long options). - -- Issue #7989: Added pure python implementation of the `datetime` module. The C - module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based - on the original python prototype for the datetime module by Tim Peters with - minor modifications by the PyPy project. The test suite now tests `datetime` - module with and without `_datetime` acceleration using the same test cases. - -- Issue #7895: platform.mac_ver() no longer crashes after calling os.fork(). - -- Issue #9323: Fixed a bug in trace.py that resulted in losing the name of the - script being traced. Patch by Eli Bendersky. - -- Issue #9282: Fixed --listfuncs option of trace.py. Thanks Eli Bendersky for - the patch. - -- Issue #3704: http.cookiejar was not properly handling URLs with a / in the - parameters. - -- Issue #9268: ``pickletools.dis()`` now has an optional *annotate* argument - which controls printing of opcode descriptions in ``dis()`` output. - -- Issue #1555570: email no longer inserts extra blank lines when a \r\n combo - crosses an 8192 byte boundary. - -- Issue #9243: Fix sndhdr module and add unit tests, contributed by James Lee. - -- ``ast.literal_eval()`` now allows byte literals. - -- Issue #9137: Fix issue in MutableMapping.update, which incorrectly treated - keyword arguments called 'self' or 'other' specially. - -- ``ast.literal_eval()`` now allows set literals. - -- Issue #9164: Ensure that sysconfig handles duplicate -arch flags in CFLAGS. - -- Issue #7646: The fnmatch pattern cache no longer grows without bound. - -- Issue #9136: Fix 'dictionary changed size during iteration' RuntimeError - produced when profiling the decimal module. This was due to a dangerous - iteration over 'locals()' in Context.__init__. - -- Fix extreme speed issue in Decimal.pow when the base is an exact power of 10 - and the exponent is tiny (for example, ``Decimal(10) ** - Decimal('1e-999999999')``). - -- Issue #9186: Fix math.log1p(-1.0) to raise ValueError, not OverflowError. - -- Issue #9130: Fix validation of relative imports in parser module. - -- Issue #9128: Fix validation of class decorators in parser module. - -- Issue #9094: python -m pickletools will now disassemble pickle files listed in - the command line arguments. See output of python -m pickletools -h for more - details. - -- Issue #5468: urlencode to handle bytes type and other encodings in its query - parameter. Patch by Dan Mahn. - -- Issue #7673: Fix security vulnerability (CVE-2010-2089) in the audioop module, - ensure that the input string length is a multiple of the frame size. - -- Issue #6507: Accept source strings in dis.dis(). Original patch by Daniel - Urban. - -- Issue #7829: Clearly document that the dis module is exposing an - implementation detail that is not stable between Python VMs or releases. - -- Issue #6589: cleanup asyncore.socket_map in case smtpd.SMTPServer constructor - raises an exception. - -- Issue #9110: Addition of ContextDecorator to contextlib, for creating APIs - that act as both context managers and decorators. contextmanager changes to - use ContextDecorator. - -- Implement importlib.abc.SourceLoader and deprecate PyLoader and PyPycLoader - for removal in Python 3.4. - -- Issue #9064: pdb's "up" and "down" commands now accept an optional argument - giving the number of frames to go. - -- Issue #9018: os.path.normcase() now raises a TypeError if the argument is not - ``str`` or ``bytes``. - -- Issue #9075: In the ssl module, remove the setting of a ``debug`` flag on an - OpenSSL structure. - -- Issue #8682: The ssl module now temporary increments the reference count of a - socket object got through ``PyWeakref_GetObject``, so as to avoid possible - deallocation while the object is still being used. - -- Issue #1368368: FancyURLOpener class changed to throw an Exception on wrong - password instead of presenting an interactive prompt. Older behavior can be - obtained by passing retry=True to http_error_xxx methods of FancyURLOpener. - -- Issue #8720: Fix regression caused by fix for #4050 by making getsourcefile - smart enough to find source files in the linecache. - -- Issue #5610: feedparser no longer eats extra characters at the end of a body - part if the body part ends with a ``\r\n``. - -- Issue #8986: math.erfc was incorrectly raising OverflowError for values - between -27.3 and -30.0 on some platforms. - -- Issue #8784: Set tarfile default encoding to 'utf-8' on Windows. - -- Issue #8966: If a ctypes structure field is an array of c_char, convert its - value to bytes instead of str (as done for c_char and c_char_p). - -- Issue #8188: Comparisons between Decimal and Fraction objects are now - permitted, returning a result based on the exact numerical values of the - operands. This builds on issue #2531, which allowed Decimal-to-float - comparisons; all comparisons involving numeric types (bool, int, float, - complex, Decimal, Fraction) should now act as expected. - -- Issue #8897: Fix sunau module, use bytes to write the header. Patch written by - Thomas Jollans. - -- Issue #8899: time.struct_time now has class and attribute docstrings. - -- Issue #6470: Drop UNC prefix in FixTk. - -- Issue #4768: base64 encoded email body parts were incorrectly stored as binary - strings. They are now correctly converted to strings. - -- Issue #8833: tarfile created hard link entries with a size field != 0 by - mistake. - -- Charset.body_encode now correctly handles base64 encoding by encoding with the - output_charset before calling base64mime.encode. Passes the tests from 2.x - issue #1368247. - -- Issue #8845: sqlite3 Connection objects now have a read-only in_transaction - attribute that is True iff there are uncommitted changes. - -- Issue #1289118: datetime.timedelta objects can now be multiplied by float and - divided by float and int objects. Results are rounded to the nearest multiple - of timedelta.resolution with ties resolved using round-half-to-even method. - -- Issue #7150: Raise OverflowError if the result of adding or subtracting - timedelta from date or datetime falls outside of the MINYEAR:MAXYEAR range. - -- Issue #8806: add SSL contexts support to ftplib. - -- Issue #4769: Fix main() function of the base64 module, use sys.stdin.buffer - and sys.stdout.buffer (instead of sys.stdin and sys.stdout) to use the bytes - API. - -- Issue #8770: Now sysconfig displays information when it's called as a script. - Initial idea by Sridhar Ratnakumar. - -- Issue #6662: Fix parsing of malformatted charref (&#bad;), patch written by - Fredrik Håård. - -- Issue #8540: Decimal module: rename the Context._clamp attribute to - Context.clamp and make it public. This is useful in creating contexts that - correspond to the decimal interchange formats specified in IEEE 754. - -- Issue #6268: Fix seek() method of codecs.open(), don't read or write the BOM - twice after seek(0). Fix also reset() method of codecs, UTF-16, UTF-32 and - StreamWriter classes. - -- Issue #3798: sys.exit(message) writes the message to sys.stderr file, instead - of the C file stderr, to use stderr encoding and error handler. - -- Issue #8782: Add a trailing newline in linecache.updatecache to the last line - of files without one. - -- Issue #8729: Return NotImplemented from collections.Mapping.__eq__ when - comparing to a non-mapping. - -- Issue #8774: tabnanny uses the encoding cookie (#coding:...) to use the - correct encoding. - -- Issue #4870: Add an `options` attribute to SSL contexts, as well as several - ``OP_*`` constants to the `ssl` module. This allows selectively disabling - protocol versions, when used in combination with `PROTOCOL_SSLv23`. - -- Issue #8759: Fixed user paths in sysconfig for posix and os2 schemes. - -- Issue #8663: distutils.log emulates backslashreplace error handler. Fix - compilation in a non-ASCII directory if stdout encoding is ASCII (eg. if - stdout is not a TTY). - -- Issue #8513: os.get_exec_path() supports b'PATH' key and bytes value. - subprocess.Popen() and os._execvpe() support bytes program name. Add - os.supports_bytes_environ flag: True if the native OS type of the environment - is bytes (eg. False on Windows). - -- Issue #8633: tarfile is now able to read and write archives with "raw" binary - pax headers as described in POSIX.1-2008. - -- Issue #1285086: Speed up urllib.parse functions: quote, quote_from_bytes, - unquote, unquote_to_bytes. - -- Issue #8688: Distutils now recalculates MANIFEST every time. - -- Issue #8477: ssl.RAND_egd() and ssl._test_decode_cert() support str with - surrogates and bytes for the filename. - -- Issue #8550: Add first class ``SSLContext`` objects to the ssl module. - -- Issue #8681: Make the zlib module's error messages more informative when the - zlib itself doesn't give any detailed explanation. - -- The audioop module now supports sound fragments of length greater than 2**31 - bytes on 64-bit machines, and is PY_SSIZE_T_CLEAN. - -- Issue #4972: Add support for the context management protocol to the ftplib.FTP - class. - -- Issue #8664: In py_compile, create __pycache__ when the compiled path is - given. - -- Issue #8514: Add os.fsencode() function (Unix only): encode a string to bytes - for use in the file system, environment variables or the command line. - -- Issue #8571: Fix an internal error when compressing or decompressing a chunk - larger than 1GB with the zlib module's compressor and decompressor objects. - -- Issue #8603: Support bytes environmental variables on Unix: Add os.environb - mapping and os.getenvb() function. os.unsetenv() encodes str argument to the - file system encoding with the surrogateescape error handler (instead of - utf8/strict) and accepts bytes. posix.environ keys and values are now bytes. - -- Issue #8573: asyncore _strerror() function might throw ValueError. - -- Issue #8483: asyncore.dispatcher's __getattr__ method produced confusing error - messages when accessing undefined class attributes because of the cheap - inheritance with the underlying socket object. The cheap inheritance has been - deprecated. - -- Issue #4265: shutil.copyfile() was leaking file descriptors when disk fills. - Patch by Tres Seaver. - -- Issue #8390: tarfile uses surrogateescape as the default error handler - (instead of replace in read mode or strict in write mode). - -- Issue #7755: Use an unencumbered audio file for tests. - -- Issue #8621: uuid.uuid4() returned the same sequence of values in the parent - and any children created using ``os.fork`` on MacOS X 10.6. - -- Issue #8567: Fix precedence of signals in Decimal module: when a Decimal - operation raises multiple signals and more than one of those signals is - trapped, the specification determines the order in which the signals should be - handled. In many cases this order wasn't being followed, leading to the wrong - Python exception being raised. - -- Issue #7865: The close() method of ``io`` objects should not swallow - exceptions raised by the implicit flush(). Also qensure that calling close() - several times is supported. Patch by Pascal Chambon. - -- Issue #4687: Fix accuracy of garbage collection runtimes displayed with - gc.DEBUG_STATS. - -- Issue #8354: The siginterrupt setting is now preserved for all signals, not - just SIGCHLD. - -- Issue #7192: webbrowser.get("firefox") now works on Mac OS X, as does - webbrowser.get("safari"). - -- Issue #8464: tarfile no longer creates files with execute permissions set when - mode="w|" is used. - -- Issue #7834: Fix connect() of Bluetooth L2CAP sockets with recent versions of - the Linux kernel. Patch by Yaniv Aknin. - -- Issue #8295: Added shutil.unpack_archive. - -- Issue #6312: Fixed http HEAD request when the transfer encoding is chunked. - It should correctly return an empty response now. - -- Issue #8546: Reject None given as the buffering argument to _pyio.open. - -- Issue #8549: Fix compiling the _ssl extension under AIX. Patch by - Sridhar Ratnakumar. - -- Issue #6656: fix locale.format_string to handle escaped percents - and mappings. - -- Issue #2302: Fix a race condition in SocketServer.BaseServer.shutdown, where - the method could block indefinitely if called just before the event loop - started running. This also fixes the occasional freezes witnessed in - test_httpservers. - -- Issue #8524: When creating an SSL socket, the timeout value of the original - socket wasn't retained (instead, a socket with a positive timeout would be - turned into a non-blocking SSL socket). - -- Issue #5103: SSL handshake would ignore the socket timeout and block - indefinitely if the other end didn't respond. - -- The do_handshake() method of SSL objects now adjusts the blocking mode of the - SSL structure if necessary (as other methods already do). - -- Issue #8391: os.execvpe() and os.getenv() supports unicode with surrogates and - bytes strings for environment keys and values. - -- Issue #8467: Pure Python implementation of subprocess encodes the error - message using surrogatepass error handler to support surrogates in the - message. - -- Issue #8468: bz2.BZ2File() accepts str with surrogates and bytes filenames. - -- Issue #8451: Syslog module now uses basename(sys.argv[0]) instead of the - string "python" as the *ident*. openlog() arguments are all optional and - keywords. - -- Issue #8108: Fix the unwrap() method of SSL objects when the socket has a - non-infinite timeout. Also make that method friendlier with applications - wanting to continue using the socket in clear-text mode, by disabling - OpenSSL's internal readahead. Thanks to Darryl Miles for guidance. - -- Issue #8496: make mailcap.lookup() always return a list, rather than an - iterator. Patch by Gregory Nofi. - -- Issue #8195: Fix a crash in sqlite Connection.create_collation() if the - collation name contains a surrogate character. - -- Issue #8484: Load all ciphers and digest algorithms when initializing the _ssl - extension, such that verification of some SSL certificates doesn't fail - because of an "unknown algorithm". - -- Issue #6547: Added the ignore_dangling_symlinks option to shutil.copytree. - -- Issue #1540112: Now allowing the choice of a copy function in shutil.copytree. - -- Issue #4814: timeout parameter is now applied also for connections resulting - from PORT/EPRT commands. - -- Issue #8463: added missing reference to bztar in shutil's documentation. - -- Issue #7154: urllib.request can now detect the proxy settings on OSX 10.6 (as - long as the user didn't specify 'automatic proxy configuration'). - -- Issue #3817: ftplib.FTP.abort() method now considers 225 a valid response code - as stated in RFC-959 at chapter 5.4. - -- Issue #8394: _ctypes.dlopen() accepts bytes, bytearray and str with - surrogates. - -- Issue #850728: Add a *timeout* parameter to the `acquire()` method of - `threading.Semaphore` objects. Original patch by Torsten Landschoff. - -- Issue #8322: Add a *ciphers* argument to SSL sockets, so as to change the - available cipher list. Helps fix test_ssl with OpenSSL 1.0.0. - -- Issue #8393: subprocess accepts bytes, bytearray and str with surrogates for - the current working directory. - -- Issue #7606: XML-RPC traceback stored in X-traceback is now encoded to ASCII - using backslashreplace error handler. - -- Issue #8412: os.system() now accepts bytes, bytearray and str with surrogates. - -- Issue #2987: RFC2732 support for urlparse (IPv6 addresses). Patch by Tony - Locke and Hans Ulrich Niedermann. - -- Issue #5277: Fix quote counting when parsing RFC 2231 encoded parameters. - -- Issue #7316: The acquire() method of lock objects in the ``threading`` - module now takes an optional timeout argument in seconds. Timeout support - relies on the system threading library, so as to avoid a semi-busy wait loop. - -- Issue #8383: pickle and pickletools use surrogatepass error handler when - encoding unicode as utf8 to support lone surrogates and stay compatible with - Python 2.x and 3.x. - -- Issue #7585: difflib context and unified diffs now place a tab between - filename and date, conforming to the 'standards' they were originally designed - to follow. This improves compatibility with patch tools. - -- Issue #7472: Fixed typo in email.encoders module; messages using ISO-2022 - character sets will now consistently use a Content-Transfer-Encoding of 7bit - rather than sometimes being marked as 8bit. - -- Issue #8375: test_distutils now checks if the temporary directory are still - present before it cleans them. - -- Issue #8374: Update the internal alias table in the ``locale`` module to - cover recent locale changes and additions. - -- Issue #8321: Give access to OpenSSL version numbers from the `ssl` module, - using the new attributes `ssl.OPENSSL_VERSION`, `ssl.OPENSSL_VERSION_INFO` and - `ssl.OPENSSL_VERSION_NUMBER`. - -- Add functools.total_ordering() and functools.cmp_to_key(). - -- Issue #8257: The Decimal construct now accepts a float instance directly, - converting that float to a Decimal of equal value: - - >>> Decimal(1.1) - Decimal('1.100000000000000088817841970012523233890533447265625') - -- Issue #8294: The Fraction constructor now accepts Decimal and float instances - directly. - -- Issue #7279: Comparisons involving a Decimal signaling NaN now signal - InvalidOperation instead of returning False. (Comparisons involving a quiet - NaN are unchanged.) Also, Decimal quiet NaNs are now hashable; Decimal - signaling NaNs remain unhashable. - -- Issue #2531: Comparison operations between floats and Decimal instances now - return a result based on the numeric values of the operands; previously they - returned an arbitrary result based on the relative ordering of id(float) and - id(Decimal). See also issue #8188, which adds Decimal-to-Fraction - comparisons. - -- Added a subtract() method to collections.Counter(). - -- Issue #8233: When run as a script, py_compile.py optionally takes a single - argument `-` which tells it to read files to compile from stdin. Each line is - read on demand and the named file is compiled immediately. (Original patch by - Piotr Ożarowski). - -- Backwards incompatible change: Unicode codepoints line tabulation (0x0B) and - form feed (0x0C) are now considered linebreaks, as specified in Unicode - Standard Annex #14. See issue #7643. http://www.unicode.org/reports/tr14/ - -- Comparisons using one of <, <=, >, >= between a complex instance and a - Fractions instance now raise TypeError instead of returning True/False. This - makes Fraction <=> complex comparisons consistent with int <=> complex, float - <=> complex, and complex <=> complex comparisons. - -- Issue #8139: ossaudiodev didn't initialize its types properly, therefore some - methods (such as oss_mixer_device.fileno()) were not available. Initial patch - by Bertrand Janin. - -- Issue #8205: Remove the "Modules" directory from sys.path when Python is - running from the build directory (POSIX only). - -- Issue #7512: shutil.copystat() could raise an OSError when the filesystem - didn't support chflags() (for example ZFS under FreeBSD). The error is now - silenced. - -- Issue #7860: platform.uname now reports the correct 'machine' type when Python - is running in WOW64 mode on 64 bit Windows. - -- Issue #3890, #8222: Fix recv() and recv_into() on non-blocking SSL sockets. - Also, enable the SSL_MODE_AUTO_RETRY flag on SSL sockets, so that blocking - reads and writes are always retried by OpenSSL itself. - -- Issue #4282: Fix the main function of the profile module for a non-ASCII - script, open the file in binary mode and not in text mode with the default - (utf8) encoding. - -- Issue #8179: Fix macpath.realpath() on a non-existing path. - -- Issue #8024: Update the Unicode database to 5.2. - -- Issue #8168: py_compile now handles files with utf-8 BOMS. - -- ``tokenize.detect_encoding`` now returns ``'utf-8-sig'`` when a UTF-8 BOM is - detected. - -- Issue #6716/2: Backslash-replace error output in compilall. - -- Issue #4961: Inconsistent/wrong result of askyesno function in tkMessageBox - with Tcl/Tk-8.5. - -- Issue #8140: extend compileall to compile single files. Add -i option. - -- Issue #7356: ctypes.util: Make parsing of ldconfig output independent of the - locale. - -- The internals of the subprocess module on POSIX systems have been replaced by - an extension module (_posixsubprocess) so that the fork()+exec() can be done - safely without the possibility of deadlock in multithreaded applications. - -- subprocess.Popen now has restore_signals and start_new_session features. The - default of restore_signals=True is a new behavior compared to earlier Python - versions. This means that signals such as SIGPIPE are not ignored by default - in subprocesses launched by Python (Issue #1652). - -- Issue #6472: The xml.etree package is updated to ElementTree 1.3. The - cElementTree module is updated too. - -- Issue #7774: Set sys.executable to an empty string if argv[0] has been set to - a non existent program name and Python is unable to retrieve the real program - name. - -- Issue #7880: Fix sysconfig when the python executable is a symbolic link. - -- Issue #6509: fix re.sub to work properly when the pattern, the string, and the - replacement were all bytes. Patch by Antoine Pitrou. - -- The sqlite3 module was updated to pysqlite 2.6.0. This fixes several obscure - bugs and allows loading SQLite extensions from shared libraries. - -- Issue #1054943: Fix ``unicodedata.normalize('NFC', text)`` for the Public - Review Issue #29 (http://unicode.org/review/pr-29.html). - -- Issue #7494: fix a crash in _lsprof (cProfile) after clearing the profiler, - reset also the pointer to the current pointer context. - -- Issue #7232: Add support for the context management protocol to the TarFile - class. - -- Issue #7250: Fix info leak of os.environ across multi-run uses of - wsgiref.handlers.CGIHandler. - -- Issue #1729305: Fix doctest to handle encode error with "backslashreplace". - -- Issue #691291: codecs.open() should not convert end of lines on reading and - writing. - -- Issue #7869: logging: improved diagnostic for format-time errors. - -- Issue #7868: logging: added loggerClass attribute to Manager. - -- logging: Implemented PEP 391. - -- Issue #1537721: Add a writeheader() method to csv.DictWriter. - -- Issue #7959: ctypes callback functions are now registered correctly with the - cycle garbage collector. - -- Issue #5801: removed spurious empty lines in wsgiref. - -- Issue #6666: fix bug in trace.py that applied the list of directories to be - ignored only to the first file. Noted by Bogdan Opanchuk. - -- Issue #7597: curses.use_env() can now be called before initscr(). Noted by - Kan-Ru Chen. - -- Issue #7310: fix the __repr__ of os.environ to show the environment variables. - -- Issue #7970: email.Generator.flatten now correctly flattens message/rfc822 - messages parsed by email.Parser.HeaderParser. - -- Issue #7361: Importlib was not properly checking the number of bytes in - bytecode file when it was less than 8 bytes. - -- Issue #7633: In the decimal module, Context class methods (with the exception - of canonical and is_canonical) now accept instances of int and long wherever a - Decimal instance is accepted, and implicitly convert that argument to Decimal. - Previously only some arguments were converted. - -- Issue #7835: shelve should no longer produce mysterious warnings during - interpreter shutdown. - -- Issue #2746: Don't escape ampersands and angle brackets ("&", "<", ">") in XML - processing instructions and comments. These raw characters are allowed by the - XML specification, and are necessary when outputting e.g. PHP code in a - processing instruction. Patch by Neil Muller. - -- Issue #6233: ElementTree failed converting unicode characters to XML entities - when they could't be represented in the requested output encoding. Patch by - Jerry Chen. - -- Issue #6003: add an argument to ``zipfile.Zipfile.writestr`` to specify the - compression type. - -- Issue #4772: Raise a ValueError when an unknown Bluetooth protocol is - specified, rather than fall through to AF_PACKET (in the `socket` module). - Also, raise ValueError rather than TypeError when an unknown TIPC address type - is specified. Patch by Brian Curtin. - -- Issue #6939: Fix file I/O objects in the `io` module to keep the original file - position when calling `truncate()`. It would previously change the file - position to the given argument, which goes against the tradition of - ftruncate() and other truncation APIs. Patch by Pascal Chambon. - -- Issue #7610: Reworked implementation of the internal - ``zipfile.ZipExtFile`` class used to represent files stored inside an - archive. The new implementation is significantly faster and can be wrapped in - an ``io.BufferedReader`` object for more speedups. It also solves an - issue where interleaved calls to `read()` and `readline()` give wrong results. - Patch by Nir Aides. - -- Issue #6963: Added "maxtasksperchild" argument to multiprocessing.Pool, - allowing for a maximum number of tasks within the pool to be completed by the - worker before that worker is terminated, and a new one created to replace it. - -- Issue #7792: Registering non-classes to ABCs raised an obscure error. - -- Issue #7785: Don't accept bytes in FileIO.write(). - -- Removed the functions 'verify' and 'vereq' from Lib/test/support.py. - -- Issue #7773: Fix an UnboundLocalError in platform.linux_distribution() when - the release file is empty. - -- Issue #7561: Fix crashes when using bytearray objects with the posix - module. - -- Issue #1670765: Prevent email.generator.Generator from re-wrapping headers in - multipart/signed MIME parts, which fixes one of the sources of invalid - modifications to such parts by Generator. - -- Issue #7703: Add support for the new buffer API to `binascii.a2bhqx`. Patch - by Florent Xicluna, along with some additional tests. - -- Issue #7701: Fix crash in binascii.b2a_uu() in debug mode when given a 1-byte - argument. Patch by Victor Stinner. - -- Issue #3299: Fix possible crash in the _sre module when given bad argument - values in debug mode. Patch by Victor Stinner. - -- Issue #2846: Add support for gzip.GzipFile reading zero-padded files. Patch - by Brian Curtin. - -- Issue #7681: Use floor division in appropriate places in the wave module. - -- Issue #5372: Drop the reuse of .o files in Distutils' ccompiler (since - Extension extra options may change the output without changing the .c - file). Initial patch by Collin Winter. - -- Issue #7617: Make sure distutils.unixccompiler.UnixCCompiler recognizes gcc - when it has a fully qualified configuration prefix. Initial patch by Arfrever. - -- Issue #7105: Make WeakKeyDictionary and WeakValueDictionary robust against the - destruction of weakref'ed objects while iterating. - -- Issue #7455: Fix possible crash in cPickle on invalid input. Patch by Victor - Stinner. - -- Issue #1628205: Socket file objects returned by socket.socket.makefile() now - properly handles EINTR within the read, readline, write & flush methods. The - socket.sendall() method now properly handles interrupted system calls. - -- Issue #7471: Improve the performance of GzipFile's buffering mechanism, and - make it implement the `io.BufferedIOBase` ABC to allow for further speedups by - wrapping it in an `io.BufferedReader`. Patch by Nir Aides. - -- Issue #3972: http.client.HTTPConnection now accepts an optional source_address - parameter to allow specifying where your connections come from. - -- socket.create_connection now accepts an optional source_address parameter. - -- Issue #5511: now zipfile.ZipFile can be used as a context manager. Initial - patch by Brian Curtin. - -- Issue #7556: Make sure Distutils' msvc9compile reads and writes the MSVC XML - Manifest file in text mode so string patterns can be used in regular - expressions. - -- Issue #7552: Removed line feed in the base64 Authorization header in the - Distutils upload command to avoid an error when PyPI reads it. This occurs on - long passwords. Initial patch by JP St. Pierre. - -- Issue #7231: urllib2 cannot handle https with proxy requiring auth. Patch by - Tatsuhiro Tsujikawa. - -- Issue #4757: `zlib.compress` and other methods in the zlib module now raise a - TypeError when given an `str` object (rather than a `bytes`-like object). - Patch by Victor Stinner and Florent Xicluna. - -- Issue #7349: Make methods of file objects in the io module accept None as an - argument where file-like objects (ie StringIO and BytesIO) accept them to mean - the same as passing no argument. - -- Issue #7357: tarfile no longer suppresses fatal extraction errors by default. - -- Issue #5949: added check for correct lineends in input from IMAP server in - imaplib. - -- Add count() and reverse() methods to collections.deque(). - -- Fix variations of extending deques: d.extend(d) d.extendleft(d) d+=d - -- Issue #6986: Fix crash in the JSON C accelerator when called with the wrong - parameter types. Patch by Victor Stinner. - -- Issue #7457: added a read_pkg_file method to - distutils.dist.DistributionMetadata. - -- logging: Added optional `secure` parameter to SMTPHandler, to enable use of - TLS with authentication credentials. - -- Issue #1923: Fixed the removal of meaningful spaces when PKG-INFO is generated - in Distutils. Patch by Stephen Emslie. - -- Issue #4120: Drop reference to CRT from manifest when building extensions with - msvc9compiler. - -- Issue #7333: The `posix` module gains an `initgroups()` function providing - access to the initgroups(3) C library call on Unix systems which implement it. - Patch by Jean-Paul Calderone. - -- Issue #7408: Fixed distutils.tests.sdist so it doesn't check for group - ownership when the group is not forced, because the group may be different - from the user's group and inherit from its container when the test is run. - -- Issue #4486: When an exception has an explicit cause, do not print its - implicit context too. This affects the `traceback` module as well as built-in - exception printing. - -- Issue #1515: Enable use of deepcopy() with instance methods. Patch by Robert - Collins. - -- Issue #7403: logging: Fixed possible race condition in lock creation. - -- Issue #6845: Add restart support for binary upload in ftplib. The - `storbinary()` method of FTP and FTP_TLS objects gains an optional `rest` - argument. Patch by Pablo Mouzo. - -- Issue #5788: `datetime.timedelta` objects get a new `total_seconds()` method - returning the total number of seconds in the duration. Patch by Brian - Quinlan. - -- Issue #7133: SSL objects now support the new buffer API. - -- Issue #1488943: difflib.Differ() doesn't always add hints for tab characters. - -- Issue #6123: tarfile now opens empty archives correctly and consistently - raises ReadError on empty files. - -- Issue #7354: distutils.tests.test_msvc9compiler - dragfullwindows can be 2. - -- Issue #5037: Proxy the __bytes__ special method instead to __bytes__ instead - of __str__. - -- Issue #7341: Close the internal file object in the TarFile constructor in case - of an error. - -- Issue #7293: distutils.test_msvc9compiler is fixed to work on any fresh - Windows box. Help provided by David Bolen. - -- Issue #2054: ftplib now provides an FTP_TLS class to do secure FTP using TLS - or SSL. Patch by Giampaolo Rodola'. - -- Issue #7328: pydoc no longer corrupts sys.path when run with the '-m' switch. - -- Issue #4969: The mimetypes module now reads the MIME database from the - registry under Windows. Patch by Gabriel Genellina. - -- Issue #6816: runpy now provides a run_path function that allows Python code to - execute file paths that refer to source or compiled Python files as well as - zipfiles, directories and other valid sys.path entries that contain a - __main__.py file. This allows applications that run other Python scripts to - support the same flexibility as the CPython command line itself. - -- Issue #7318: multiprocessing now uses a timeout when it fails to establish a - connection with another process, rather than looping endlessly. The default - timeout is 20 seconds, which should be amply sufficient for local connections. - -- Issue #7197: Allow unittest.TextTestRunner objects to be pickled and - unpickled. This fixes crashes under Windows when trying to run - test_multiprocessing in verbose mode. - -- Issue #7893: ``unittest.TextTestResult`` is made public and a ``resultclass`` - argument added to the TextTestRunner constructor allowing a different result - class to be used without having to subclass. - -- Issue #7588: ``unittest.TextTestResult.getDescription`` now includes the test - name in failure reports even if the test has a docstring. - -- Issue #3001: Add a C implementation of recursive locks which is used by - default when instantiating a `threading.RLock` object. This makes recursive - locks as fast as regular non-recursive locks (previously, they were slower by - 10x to 15x). - -- Issue #7282: Fix a memory leak when an RLock was used in a thread other than - those started through `threading.Thread` (for example, using - `_thread.start_new_thread()`). - -- Issue #7187: Importlib would not silence the IOError raised when trying to - write new bytecode when it was made read-only. - -- Issue #7264: Fix a possible deadlock when deallocating thread-local objects - which are part of a reference cycle. - -- Issue #7211: Allow 64-bit values for the `ident` and `data` fields of kevent - objects on 64-bit systems. Patch by Michael Broghton. - -- Issue #6896: mailbox.Maildir now invalidates its internal cache each time a - modification is done through it. This fixes inconsistencies and test failures - on systems with slightly bogus mtime behaviour. - -- Issue #7246 & Issue #7208: getpass now properly flushes input before reading - from stdin so that existing input does not confuse it and lead to incorrect - entry or an IOError. It also properly flushes it afterwards to avoid the - terminal echoing the input afterwards on OSes such as Solaris. - -- Issue #7233: Fix a number of two-argument Decimal methods to make sure that - they accept an int or long as the second argument. Also fix buggy handling of - large arguments (those with coefficient longer than the current precision) in - shift and rotate. - -- Issue #4750: Store the basename of the original filename in the gzip FNAME - header as required by RFC 1952. - -- Issue #1180: Added a new global option to ignore ~/.pydistutils.cfg in - Distutils. - -- Issue #7218: Fix test_site for win32, the directory comparison was done with - an uppercase. - -- Issue #7205: Fix a possible deadlock when using a BZ2File object from - several threads at once. - -- Issue #7077: logging: SysLogHandler now treats Unicode as per RFC 5424. - -- Issue #7099: Decimal.is_normal now returns True for numbers with exponent - larger than emax. - -- Issue #7080: locale.strxfrm() raises a MemoryError on 64-bit non-Windows - platforms, and assorted locale fixes by Derk Drukker. - -- Issue #5833: Fix extra space character in readline completion with the GNU - readline library version 6.0. - -- Issue #6894: Fixed the issue urllib2 doesn't respect "no_proxy" environment. - -- Issue #7086: Added TCP support to SysLogHandler, and tidied up some - anachronisms in the code which were a relic of 1.5.2 compatibility. - -- Issue #7082: When falling back to the MIME 'name' parameter, the correct place - to look for it is the Content-Type header. - -- Make tokenize.detect_coding() normalize utf-8 and iso-8859-1 variants like the - builtin tokenizer. - -- Issue #7048: Force Decimal.logb to round its result when that result is too - large to fit in the current precision. - -- Issue #6236, #6348: Fix various failures in the I/O library under AIX and - other platforms, when using a non-gcc compiler. Patch by Derk Drukker. - -- Issue #4606: Passing 'None' if ctypes argtype is set to POINTER(...) does now - always result in NULL. - -- Issue #5042: Structure sub-subclass does now initialize correctly with base - class positional arguments. - -- Issue #6882: Import uuid creates zombies processes. - -- Issue #6635: Fix profiler printing usage message. - -- Issue #6856: Add a filter keyword argument to TarFile.add(). - -- Issue #6888: pdb's alias command was broken when no arguments were given. - -- Issue #6857: Default format() alignment should be '>' for Decimal instances. - -- Issue #6795: int(Decimal('nan')) now raises ValueError instead of returning - NaN or raising InvalidContext. Also, fix infinite recursion in - long(Decimal('nan')). - -- Issue #6850: Fix bug in Decimal._parse_format_specifier for formats with no - type specifier. - -- Issue #6239: ctypes.c_char_p return value must return bytes. - -- Issue #6838: Use a list to accumulate the value instead of repeatedly - concatenating strings in http.client's HTTPResponse._read_chunked providing a - significant speed increase when downloading large files servend with a - Transfer-Encoding of 'chunked'. - -- Trying to import a submodule from a module that is not a package, ImportError - should be raised, not AttributeError. - -- When the globals past to importlib.__import__() has __package__ set to None, - fall back to computing what __package__ should be instead of giving up. - -- Raise a TypeError when the name of a module to be imported for - importlib.__import__ is not a string (was raising an AttributeError before). - -- Allow the fromlist passed into importlib.__import__ to be any iterable. - -- Have importlib raise ImportError if None is found in sys.modules. - -- Issue #6054: Do not normalize stored pathnames in tarfile. - -- Issue #6794: Fix Decimal.compare_total and Decimal.compare_total_mag: NaN - payloads are now ordered by integer value rather than lexicographically. - -- Issue #1356969: Add missing info methods in tix.HList. - -- Issue #1522587: New constants and methods for the tix.Grid widget. - -- Issue #1250469: Fix the return value of tix.PanedWindow.panes. - -- Issue #1119673: Do not override tkinter.Text methods when creating a - ScrolledText. - -- Issue #6665: Fix fnmatch to properly match filenames with newlines in them. - -- Issue #1135: Add the XView and YView mix-ins to avoid duplicating the xview* - and yview* methods. - -- Issue #6629: Fix a data corruption issue in the new I/O library, which could - occur when writing to a BufferedRandom object (e.g. a file opened in "rb+" or - "wb+" mode) after having buffered a certain amount of data for reading. This - bug was not present in the pure Python implementation. - -- Issue #6622: Fix "local variable 'secret' referenced before assignment" bug in - POP3.apop. - -- Issue #2715: Remove remnants of Carbon.File from binhex module. - -- Issue #6595: The Decimal constructor now allows arbitrary Unicode decimal - digits in input, as recommended by the standard. Previously it was restricted - to accepting [0-9]. - -- Issue #6106: telnetlib.Telnet.process_rawq doesn't handle default WILL/WONT - DO/DONT correctly. - -- Issue #1424152: Fix for http.client, urllib.request to support SSL while - working through proxy. Original patch by Christopher Li, changes made by - Senthil Kumaran. - -- Add importlib.abc.ExecutionLoader to represent the PEP 302 protocol for - loaders that allow for modules to be executed. Both importlib.abc.PyLoader and - PyPycLoader inherit from this class and provide implementations in relation to - other methods required by the ABCs. - -- importlib.abc.PyLoader did not inherit from importlib.abc.ResourceLoader like - the documentation said it did even though the code in PyLoader relied on the - abstract method required by ResourceLoader. - -- Issue #6431: Make Fraction type return NotImplemented when it doesn't know how - to handle a comparison without loss of precision. Also add correct handling - of infinities and nans for comparisons with float. - -- Issue #6415: Fixed warnings.warn segfault on bad formatted string. - -- Issue #6358: The exit status of a command started with os.popen() was reported - differently than it did with python 2.x. - -- Issue #6323: The pdb debugger did not exit when running a script with a syntax - error. - -- Issue #3392: The subprocess communicate() method no longer fails in select() - when file descriptors are large; communicate() now uses poll() when possible. - -- Issue #6369: Fix an RLE decompression bug in the binhex module. - -- Issue #6344: Fixed a crash of mmap.read() when passed a negative argument. - -- The deprecated function string.maketrans has been removed. - -- Issue #4005: Fixed a crash of pydoc when there was a zip file present in - sys.path. - -- Issue #6218: io.StringIO and io.BytesIO instances are now picklable. - -- The os.get_exec_path() function to return the list of directories that will be - searched for an executable when launching a subprocess was added. - -- Issue #7481: When a threading.Thread failed to start it would leave the - instance stuck in initial state and present in threading.enumerate(). - -- Issue #1068268: The subprocess module now handles EINTR in internal os.waitpid - and os.read system calls where appropriate. - -- Issue #6729: Added ctypes.c_ssize_t to represent ssize_t. - -- Issue #6247: The argparse module has been added to the standard library. - -- Issue #8235: _socket: Add the constant ``SO_SETFIB``. SO_SETFIB is a socket - option available on FreeBSD 7.1 and newer. - -- Issue #9315: Fix for the trace module to record correct class name - for tracing methods. - -Extension Modules ------------------ - -- Issue #9959: Tweak formula used for computing math.log of an integer, - making it marginally more accurate for exact powers of 2. - -- Issue #9422: Fix memory leak when re-initializing a struct.Struct object. - -- Issue #7900: The getgroups(2) system call on MacOSX behaves rather oddly - compared to other unix systems. In particular, os.getgroups() does not reflect - any changes made using os.setgroups() but basically always returns the same - information as the id command. os.getgroups() can now return more than 16 - groups on MacOSX. - -- Issue #6095: Make directory argument to os.listdir optional. - -- Issue #9277: Fix bug in struct.pack for bools in standard mode (e.g., - struct.pack('>?')): if conversion to bool raised an exception then that - exception wasn't properly propagated on machines where char is unsigned. - -- Issue #5180: Fixed a bug that prevented loading 2.x pickles in 3.x python when - they contain instances of old-style classes. - -- Issue #9165: Add new functions math.isfinite and cmath.isfinite, to accompany - existing isinf and isnan functions. - -- Issue #1578269: Implement os.symlink for Windows 6.0+. Patch by Jason - R. Coombs. - -- In struct.pack, correctly propagate exceptions from computing the truth of an - object in the '?' format. - -- Issue #9000: datetime.timezone objects now have eval-friendly repr. - -- In the math module, correctly lookup __trunc__, __ceil__, and __floor__ as - special methods. - -- Issue #9005: Prevent utctimetuple() from producing year 0 or year 10,000. - Prior to this change, timezone adjustment in utctimetuple() could produce - tm_year value of 0 or 10,000. Now an OverflowError is raised in these edge - cases. - -- Issue #6641: The ``datetime.strptime`` method now supports the ``%z`` - directive. When the ``%z`` directive is present in the format string, an - aware ``datetime`` object is returned with ``tzinfo`` bound to a - ``datetime.timezone`` instance constructed from the parsed offset. If both - ``%z`` and ``%Z`` are present, the data in ``%Z`` field is used for timezone - name, but ``%Z`` data without ``%z`` is discarded. - -- Issue #5094: The ``datetime`` module now has a simple concrete class - implementing ``datetime.tzinfo`` interface. Instances of the new class, - ``datetime.timezone``, return fixed name and UTC offset from their - ``tzname(dt)`` and ``utcoffset(dt)`` methods. The ``dst(dt)`` method always - returns ``None``. A class attribute, ``utc`` contains an instance - representing the UTC timezone. Original patch by Rafe Kaplan. - -- Issue #8973: Add __all__ to struct module; this ensures that help(struct) - includes documentation for the struct.Struct class. - -- Issue #3129: Trailing digits in struct format string are no longer ignored. - For example, "1" or "ilib123" are now invalid formats and cause - ``struct.error`` to be raised. Patch by Caleb Deveraux. - -- Issue #7384: If the system readline library is linked against ncurses, the - curses module must be linked against ncurses as well. Otherwise it is not safe - to load both the readline and curses modules in an application. - -- Issue #2810: Fix cases where the Windows registry API returns ERROR_MORE_DATA, - requiring a re-try in order to get the complete result. - -- Issue #8692: Optimize math.factorial: replace the previous naive algorithm - with an improved 'binary-split' algorithm that uses fewer multiplications and - allows many of the multiplications to be performed using plain C integer - arithmetic instead of PyLong arithmetic. Also uses a lookup table for small - arguments. - -- Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing - overflow checks in the audioop module. - -- Issue #8644: The accuracy of td.total_seconds() has been improved (by - calculating with integer arithmetic instead of float arithmetic internally): - the result is now always correctly rounded, and is equivalent to ``td / - timedelta(seconds=1)``. - -- Issue #2706: Allow division of a timedelta by another timedelta: timedelta / - timedelta, timedelta % timedelta, timedelta // timedelta and divmod(timedelta, - timedelta) are all supported. - -- Issue #8314: Fix unsigned long long bug in libffi on Sparc v8. - -- Issue #8300: When passing a non-integer argument to struct.pack with any - integer format code, struct.pack first attempts to convert the non-integer - using its __index__ method. If that method is non-existent or raises - TypeError it goes on to try the __int__ method, as described below. - -- Issue #8142: Update libffi to the 3.0.9 release. - -- Issue #6949: Allow the _dbm extension to be built with db 4.8.x. - -- Issue #6544: Fix a reference leak in the kqueue implementation's error - handling. - -- Stop providing crtassem.h symbols when compiling with Visual Studio 2010, as - msvcr100.dll is not a platform assembly anymore. - -- Issue #6508: Add posix.{getresuid,getresgid,setresuid,setresgid}. - -- Issue #7078: Set struct.__doc__ from _struct.__doc__. - -- Issue #3366: Add erf, erfc, expm1, gamma, lgamma functions to math module. - -- Issue #6877: It is now possible to link the readline extension to the libedit - readline emulation on OSX 10.5 or later. - -- Issue #6848: Fix curses module build failure on OS X 10.6. - -- Fix a segfault that could be triggered by expat with specially formed input. - -- Issue #6561: '\d' in a regex now matches only characters with Unicode category - 'Nd' (Number, Decimal Digit). Previously it also matched characters with - category 'No'. - -- Issue #4509: Array objects are no longer modified after an operation failing - due to the resize restriction in-place when the object has exported buffers. - -- Issue #2389: Array objects are now pickled in a portable manner. - -- Expat: Fix DoS via XML document with malformed UTF-8 sequences - (CVE_2009_3560). - -- Issue #7242: On Solaris 9 and earlier calling os.fork() from within a thread - could raise an incorrect RuntimeError about not holding the import lock. The - import lock is now reinitialized after fork. - -- Issue #7999: os.setreuid() and os.setregid() would refuse to accept a -1 - parameter on some platforms such as OS X. - -- Build the ossaudio extension on GNU/kFreeBSD. - -- Issue #7347: winreg: Add CreateKeyEx and DeleteKeyEx, as well as fix a bug in - the return value of QueryReflectionKey. - -- Issue #7567: PyCurses_setupterm: Don't call ``setupterm`` twice. - -Build ------ - -- Use OpenSSL 1.0.0a on Windows. - -- Issue #9280: Make sharedinstall depend on sharedmods. - -- Issue #9189: Make a user-specified CFLAGS, CPPFLAGS, or LDFLAGS setting - override the configure and makefile defaults, without deleting options the - user didn't intend to override. Developers should no longer need to specify - OPT or EXTRA_CFLAGS, although those variables are still present for - backward-compatibility. - -- Issue #8854: Fix finding Visual Studio 2008 on Windows x64. - -- Issue #1759169, #8864: Drop _XOPEN_SOURCE on Solaris, define it for - multiprocessing only. - -- Issue #8625: Turn off optimization in --with-pydebug builds with gcc. - (Optimization was unintentionally turned on in gcc --with-pydebug builds as a - result of the issue #1628484 fix, combined with autoconf's strange choice of - default CFLAGS produced by AC_PROG_CC for gcc.) - -- Issue #3646: It is now easily possible to install a Python framework into your - home directory on MacOSX, see Mac/README for more information. - -- Issue #3928: os.mknod() now available in Solaris, also. - -- Issue #3326: Build Python without -fno-strict-aliasing when the gcc does not - give false warnings. - -- Issue #1628484: The Makefile doesn't ignore the CFLAGS environment variable - anymore. It also forwards the LDFLAGS settings to the linker when building a - shared library. - -- Issue #6716: Quote -x arguments of compileall in MSI installer. Exclude 2to3 - tests from compileall. - -- Issue #3920, #7903: Define _BSD_SOURCE on OpenBSD 4.4 through 4.9. - -- Issue #7632: When Py_USING_MEMORY_DEBUGGER is defined, disable the private - memory allocation scheme in dtoa.c and use PyMem_Malloc and PyMem_Free - instead. Also disable caching of powers of 5. - -- Issue #6491: Allow --with-dbmliborder to specify that no dbms will be built. - -- Issue #6943: Use pkg-config to find the libffi headers when the - --with-system-ffi flag is used. - -- Issue #7609: Add a --with-system-expat option that causes the system's expat - library to be used for the pyexpat module instead of the one included with - Python. - -- Issue #7589: Only build the nis module when the correct header files are - found. - -- Switch to OpenSSL 0.9.8l and sqlite 3.6.21 on Windows. - -- Issue #5792: Extend the short float repr support to x86 systems using - icc or suncc. - -- Issue #6603: Change READ_TIMESTAMP macro in ceval.c so that it compiles - correctly under gcc on x86-64. This fixes a reported problem with the - --with-tsc build on x86-64. - -- Issue #6802: Fix build issues on MacOSX 10.6. - -- Issue #6244: Allow detect_tkinter to look for Tcl/Tk 8.6. - -- Issue #4601: 'make install' did not set the appropriate permissions on - directories. - -- Issue #5390: Add uninstall icon independent of whether file extensions are - installed. - -- Issue #7541: When using ``python-config`` with a framework install the - compiler might use the wrong library. - -- python-config now supports multiple options on the same command line. - -- Issue #8509: Fix quoting in help strings and code snippets in configure.in. - -- Issue #8510: Update to autoconf2.65. - -Documentation -------------- - -- Issue #9817: Add expat COPYING file; add expat, libffi and expat licenses - to Doc/license.rst. - -- Issue #9524: Document that two CTRL* signals are meant for use only - with os.kill. - -- Issue #9255: Document that the 'test' package is meant for internal Python use - only. - -- A small WSGI server was added as Tools/scripts/serve.py, and is used to - implement a local documentation server via 'make serve' in the doc directory. - -- Updating `Using Python` documentation to include description of CPython's -J - and -X options. - -- Document that importing a module that has None in sys.modules triggers an - ImportError. - -- Issue #6556: Fixed the Distutils configuration files location explanation for - Windows. - -- Update python manual page (options -B, -O0, -s, environment variables - PYTHONDONTWRITEBYTECODE, PYTHONNOUSERSITE). - -- Issue #8909: Added the size of the bitmap used in the installer created by - distutils' bdist_wininst. Patch by Anatoly Techtonik. - -Tests ------ - -- Issue #9251: test_threaded_import didn't fail when run through regrtest if the - import lock was disabled. - -- Issue #8605: Skip test_gdb if Python is compiled with optimizations. - -- Issue #7449: Skip test_socketserver if threading support is disabled. - -- Issue #8672: Add a zlib test ensuring that an incomplete stream can be handled - by a decompressor object without errors (it returns incomplete uncompressed - data). - -- Issue #8533: regrtest uses backslashreplace error handler for stdout to avoid - UnicodeEncodeError (write non-ASCII character to stdout using ASCII encoding). - -- Issue #8576: Remove use of find_unused_port() in test_smtplib and - test_multiprocessing. Patch by Paul Moore. - -- Issue #7449: Fix many tests to support Python compiled without thread - support. Patches written by Jerry Seutter. - -- Issue #8108: test_ftplib's non-blocking SSL server now has proper handling of - SSL shutdowns. - -- Issues #8279, #8330, #8437, #8480, #8495: Fix test_gdb failures, patch written - by Dave Malcolm. - -- Issue #3864: Skip three test_signal tests on freebsd6 because they fail if any - thread was previously started, most likely due to a platform bug. - -- Issue #8193: Fix test_zlib failure with zlib 1.2.4. - -- Issue #8248: Add some tests for the bool type. Patch by Gregory Nofi. - -- Issue #8263: Now regrtest.py will report a failure if it receives a - KeyboardInterrupt (SIGINT). - -- Issue #8180 and #8207: Fix test_pep277 on OS X and add more tests for special - Unicode normalization cases. - -- Issue #7783: test.support.open_urlresource invalidates the outdated files from - the local cache. - -- Issue #7849: Now the utility ``check_warnings`` verifies if the warnings are - effectively raised. - -- The four path modules (genericpath, macpath, ntpath, posixpath) share a common - TestCase for some tests: test_genericpath.CommonTest. - -- Print platform information when running the whole test suite, or using the - --verbose flag. - -- Issue #767675: enable test_pep277 on POSIX platforms with Unicode-friendly - filesystem encoding. - -- Issue #6292: for the moment at least, the test suite runs cleanly if python is - run with the -OO flag. Tests requiring docstrings are skipped. - -- Issue #7712: test.support gained a new `temp_cwd` context manager which is now - also used by regrtest to run all the tests in a temporary directory. The - original CWD is saved in `support.SAVEDCWD`. Thanks to Florent Xicluna who - helped with the patch. - -- Issue #7924: Fix an intermittent 'XXX undetected error' failure in test_capi - (only seen so far on platforms where the curses module wasn't built), due to - an uncleared exception. - -- Issue #7728: test_timeout was changed to use support.bind_port instead of a - hard coded port. - -- Issue #7376: Instead of running a self-test (which was failing) when called - with no arguments, doctest.py now gives a usage message. - -- Issue #7396: fix regrtest -s, which was broken by the -j enhancement. - -- Issue #7498: test_multiprocessing now uses test.support.find_unused_port - instead of a hardcoded port number in test_rapid_restart. - -- Issue #7431: Use TESTFN in test_linecache instead of trying to create a file - in the Lib/test directory, which might be read-only for the user running the - tests. - -- Issue #7324: Add a sanity check to regrtest argument parsing to catch the case - of an option with no handler. - -- Issue #7312: Add a -F flag to run the selected tests in a loop until a test - fails. Can be combined with -j. - -- Issue #6551: test_zipimport could import and then destroy some modules of the - encodings package, which would make other tests fail further down the road - because the internally cached encoders and decoders would point to empty - global variables. - -- Issue #7295: Do not use a hardcoded file name in test_tarfile. - -- Issue #7270: Add some dedicated unit tests for multi-thread synchronization - primitives such as Lock, RLock, Condition, Event and Semaphore. - -- Issue #7248 (part 2): Use a unique temporary directory for importlib source - tests instead of tempfile.tempdir. This prevents the tests from sharing state - between concurrent executions on the same system. - -- Issue #7248: In importlib.test.source.util a try/finally block did not make - sure that some referenced objects actually were created in the block before - calling methods on the object. - -- Issue #7222: Make thread "reaping" more reliable so that reference - leak-chasing test runs give sensible results. The previous method of reaping - threads could return successfully while some Thread objects were still - referenced. This also introduces a new private function: - ``_thread._count()``. - -- Issue #7151: Fixed regrtest -j so that output to stderr from a test no longer - runs the risk of causing the worker thread to fail. - -- Issue #7055: test___all__ now greedily detects all modules which have an - __all__ attribute, rather than using a hardcoded and incomplete list. - -- Issue #7058: Added save/restore for things like sys.argv and cwd to - runtest_inner in regrtest, with warnings if the called test modifies them, and - a new section in the summary report at the end. - -- Issue #7042: Fix test_signal (test_itimer_virtual) failure on OS X 10.6. - -- Fixed tests in importlib.test.source.test_abc_loader that were masking the - proper exceptions that should be raised for missing or improper code object - bytecode. - -- Removed importlib's custom test discovery code and switched to - unittest.TestLoader.discover(). - -Tools/Demos ------------ - -- Issue #5464, #8974: Implement plural forms in msgfmt.py. - -- iobench (a file I/O benchmark) and ccbench (a concurrency benchmark) were - added to the `Tools/` directory. They were previously living in the sandbox. - - -What's New in Python 3.1? -========================= - -*Release date: 27-June-2009* - -Core and Builtins ------------------ - -- Issue #6334: Fix bug in range length calculation for ranges with - large arguments. - -- Issue #6329: Fixed iteration for memoryview objects (it was being blocked - because it wasn't recognized as a sequence). - -Library -------- - -- Issue #6126: Fixed pdb command-line usage. - -- Issue #6314: logging: performs extra checks on the "level" argument. - -- Issue #6274: Fixed possible file descriptors leak in subprocess.py - -- Accessing io.StringIO.buffer now raises an AttributeError instead of - io.UnsupportedOperation. - -- Issue #6271: mmap tried to close invalid file handle (-1) when anonymous. - (On Unix) - -- Issue #1202: zipfile module would cause a struct.error when attempting to - store files with a CRC32 > 2**31-1. - -Extension Modules ------------------ - -- Issue #5590: Remove unused global variable in pyexpat extension. - - -What's New in Python 3.1 Release Candidate 2? -============================================= - -*Release date: 13-June-2009* - -Core and Builtins ------------------ - -- Fixed SystemError triggered by "range([], 1, -1)". - -- Issue #5924: On Windows, a large PYTHONPATH environment variable - (more than 255 characters) would be completely ignored. - -- Issue #4547: When debugging a very large function, it was not always - possible to update the lineno attribute of the current frame. - -- Issue #5330: C functions called with keyword arguments were not reported by - the various profiling modules (profile, cProfile). Patch by Hagen Fürstenau. - -Library -------- - -- Issue #6438: Fixed distutils.cygwinccompiler.get_versions : the regular - expression string pattern was trying to match against a bytes returned by - Popen. Tested under win32 to build the py-postgresql project. - -- Issue #6258: Support AMD64 in bdist_msi. - -- Issue #6195: fixed doctest to no longer try to read 'source' data from - binary files. - -- Issue #5262: Fixed bug in next rollover time computation in - TimedRotatingFileHandler. - -- Issue #6217: The C implementation of io.TextIOWrapper didn't include the - errors property. Additionally, the errors and encoding properties of StringIO - are always None now. - -- Issue #6137: The pickle module now translates module names when loading - or dumping pickles with a 2.x-compatible protocol, in order to make data - sharing and migration easier. This behaviour can be disabled using the - new `fix_imports` optional argument. - -- Removed the ipaddr module. - -- Issue #3613: base64.{encode,decode}string are now called - base64.{encode,decode}bytes which reflects what type they accept and return. - The old names are still there as deprecated aliases. - -- Issue #5767: Remove sgmlop support from xmlrpc.client. - -- Issue #6150: Fix test_unicode on wide-unicode builds. - -- Issue #6149: Fix initialization of WeakValueDictionary objects from non-empty - parameters. - -Windows -------- - -- Issue #6221: Delete test registry key before running the test. - -- Issue #6158: Package Sine-1000Hz-300ms.aif in MSI file. - -C-API ------ - -- Issue #5735: Python compiled with --with-pydebug should throw an - ImportError when trying to import modules compiled without - --with-pydebug, and vice-versa. - - -Build ------ - -- Issue #6154: Make sure the intl library is added to LIBS if needed. Also - added LIBS to OS X framework builds. - -- Issue #5809: Specifying both --enable-framework and --enable-shared is - an error. Configure now explicitly tells you about this. - - - -What's New in Python 3.1 release candidate 1? -============================================= - -*Release date: 2009-05-30* - -Core and Builtins ------------------ - -- Issue #6097: Escape UTF-8 surrogates resulting from mbstocs conversion - of the command line. - -- Issue #6012: Add cleanup support to O& argument parsing. - -- Issue #6089: Fixed str.format with certain invalid field specifiers - that would raise SystemError. - -- Issue #5982: staticmethod and classmethod now expose the wrapped - function with __func__. - -- Added support for multiple context managers in the same with-statement. - Deprecated contextlib.nested() which is no longer needed. - -- Issue #5829: complex("1e500") no longer raises OverflowError. This - makes it consistent with float("1e500") and interpretation of real - and imaginary literals. - -- Issue #3527: Removed Py_WIN_WIDE_FILENAMES which is not used any more. - -- Issue #5994: the marshal module now has docstrings. - -- Issue #5981: Fix three minor inf/nan issues in float.fromhex: - (1) inf and nan strings with trailing whitespace were incorrectly - rejected; (2) parsing of strings representing infinities and nans - was locale aware; and (3) the interpretation of fromhex('-nan') - didn't match that of float('-nan'). - -Library -------- - -- Issue #4859: Implement PEP 383 for pwd, spwd, and grp. - -- smtplib 'login' and 'cram-md5' login are also fixed (see Issue #5259). - -- Issue #6121: pydoc now ignores leading and trailing spaces in the - argument to the 'help' function. - -- Issue #6118: urllib.parse.quote_plus ignored the encoding and errors - arguments for strings with a space in them. - -- collections.namedtuple() was not working with the following field - names: cls, self, tuple, itemgetter, and property. - -- In unittest, using a skipping decorator on a class is now equivalent to - skipping every test on the class. The ClassTestSuite class has been removed. - -- Issue #6050: Don't fail extracting a directory from a zipfile if - the directory already exists. - -- Issue #1309352: fcntl now converts its third arguments to a C `long` rather - than an int, which makes some operations possible under 64-bit Linux (e.g. - DN_MULTISHOT with F_NOTIFY). - -- Issue #5761: Add the name of the underlying file to the repr() of various - IO objects. - -- Issue #5259: smtplib plain auth login no longer gives a traceback. Fix - by Musashi Tamura, tests by Marcin Bachry. - -- Issue #1983: Fix functions taking or returning a process identifier to use - the dedicated C type ``pid_t`` instead of a C ``int``. Some platforms have - a process identifier type wider than the standard C integer type. - -- Issue #4066: smtplib.SMTP_SSL._get_socket now correctly returns the socket. - Patch by Farhan Ahmad, test by Marcin Bachry. - -- Issue #2116: Weak references and weak dictionaries now support copy()ing and - deepcopy()ing. - -- Issue #1655: Make imaplib IPv6-capable. Patch by Derek Morr. - -- Issue #5918: Fix a crash in the parser module. - -- Issue #1664: Make nntplib IPv6-capable. Patch by Derek Morr. - -- Issue #5006: Better handling of unicode byte-order marks (BOM) in the io - library. This means, for example, that opening a UTF-16 text file in - append mode doesn't add a BOM at the end of the file if the file isn't - empty. - -- Issue #4050: inspect.findsource/getsource now raise an IOError if the 'source' - file is a binary. Patch by Brodie Rao, tests by Daniel Diniz. This fix - corrects a pydoc regression. - -- Issue #5955: aifc's close method did not close the file it wrapped, - now it does. This also means getfp method now returns the real fp. - -Installation ------------- - -- Issue #6047: fullinstall has been removed because Python 3's executable will - now be known as python3. - -- Lib/smtpd.py is no longer installed as a script. - -Extension Modules ------------------ - -- Issue #3061: Use wcsftime for time.strftime where available. - -- Issue #4873: Fix resource leaks in error cases of pwd and grp. - -- Issue #6093: Fix off-by-one error in locale.strxfrm. - -- The _functools and _locale modules are now built into the libpython shared - library instead of as extension modules. - -Build ------ - -- Issue #3585: Add pkg-config support. It creates a python-2.7.pc file - and a python3.pc symlink in the $(LIBDIR)/pkgconfig directory. Patch by - Clinton Roy. - -Tests ------ - -- Issue #5442: Tests for importlib were not properly skipping case-sensitivity - tests on darwin even when the OS was installed on a case-sensitive - filesystem. Also fixed tests that should not be run when - sys.dont_write_bytecode is true. - - -What's New in Python 3.1 beta 1? -================================ - -*Release date: 2009-05-06* - -Core and Builtins ------------------ - -- Issue #5914: Add new C API function PyOS_string_to_double, and - deprecate PyOS_ascii_strtod and PyOS_ascii_atof. - -- Issue #3382: float.__format__, complex.__format__, and %-formatting - no longer map 'F' to 'f'. Because of issue #5859 (below), this only - affects nan -> NAN and inf -> INF. - -- Issue #5799: ntpath (ie, os.path on Windows) fully supports UNC pathnames - in all operations, including splitdrive, split, etc. splitunc() now issues - a PendingDeprecation warning. - -- Issue #5920: For float.__format__, change the behavior with the - empty presentation type (that is, not one of 'e', 'f', 'g', or 'n') - to be like 'g' but with at least one decimal point and with a - default precision of 12. Previously, the behavior the same but with - a default precision of 6. This more closely matches str(), and - reduces surprises when adding alignment flags to the empty - presentation type. This also affects the new complex.__format__ in - the same way. - -- Implement PEP 383, Non-decodable Bytes in System Character Interfaces. - -- Issue #5890: in subclasses of 'property' the __doc__ attribute was - shadowed by classtype's, even if it was None. property now - inserts the __doc__ into the subclass instance __dict__. - -- Issue #4426: The UTF-7 decoder was too strict and didn't accept some legal - sequences. Patch by Nick Barnes and Victor Stinner. - -- Issue #3672: Reject surrogates in utf-8 codec; add surrogatepass error handler. - -- Issue #5883: In the io module, the BufferedIOBase and TextIOBase ABCs have - received a new method, detach(). detach() disconnects the underlying stream - from the buffer or text IO and returns it. - -- Issue #5859: Remove switch from '%f' to '%g'-style formatting for - floats with absolute value over 1e50. Also remove length - restrictions for float formatting: '%.67f' % 12.34 and '%.120e' % - 12.34 no longer raise an exception. - -- Issue #1588: Add complex.__format__. For example, - format(complex(1, 2./3), '.5') now produces a sensible result. - -- Issue #5864: Fix empty format code formatting for floats so that it - never gives more than the requested number of significant digits. - -- Issue #5793: Rationalize isdigit / isalpha / tolower, etc. Includes - new Py_ISDIGIT / Py_ISALPHA / Py_TOLOWER, etc. in pctypes.h. - -- Issue #5835: Deprecate PyOS_ascii_formatd. - -- Issue #4971: Fix titlecase for characters that are their own - titlecase, but not their own uppercase. - -- Issue #5283: Setting __class__ in __del__ caused a segfault. - -- Issue #5816: complex(repr(z)) now recovers z exactly, even when - z involves nans, infs or negative zeros. - -- Issue #3166: Make int -> float conversions correctly rounded. - -- Issue #1869 (and many duplicates): make round(x, n) correctly - rounded for a float x, by using the decimal <-> binary conversions - from Python/dtoa.c. As a consequence, (e.g.) round(x, 2) now - consistently agrees with format(x, '.2f'). - -- Issue #5787: object.__getattribute__(some_type, "__bases__") segfaulted on - some builtin types. - -- Issue #5772: format(1e100, '<') produces '1e+100', not '1.0e+100'. - -- Issue #5515: str.format() type 'n' combined with commas and leading - zeros no longer gives odd results with ints and floats. - -- Implement PEP 378, Format Specifier for Thousands Separator, for - floats. - -- The str function switches to exponential notation at - 1e11, not 1e12. This avoids printing 13 significant digits in - situations where only 12 of them are correct. Example problem - value: str(1e11 + 0.5). (This minor issue has existed in 2.x for a - long time.) - -- Issue #1580: On most platforms, use a 'short' float repr: for a - finite float x, repr(x) now outputs a string based on the shortest - sequence of decimal digits that rounds to x. Previous behaviour was - to output 17 significant digits and then strip trailing zeros. - Another minor difference is that the new repr switches to - exponential notation at 1e16 instead of the previous 1e17; this - avoids misleading output in some cases. - - There's a new sys attribute sys.float_repr_style, which takes - the value 'short' to indicate that we're using short float repr, - and 'legacy' if the short float repr isn't available for one - reason or another. - - The float repr change involves incorporating David Gay's 'perfect - rounding' code into the Python core (it's in Python/dtoa.c). As a - secondary consequence, all string-to-float and float-to-string - conversions (including all float formatting operations) will be - correctly rounded on these platforms. - - See issue #1580 discussions for details of platforms for which - this change does not apply. - -- Issue #5759: float() didn't call __float__ on str subclasses. - -- The string.maketrans() function is deprecated; there is a new static method - maketrans() on the bytes and bytearray classes. This removes confusion about - the types string.maketrans() is supposed to work with, and mirrors the - methods available on the str class. - -- Issue #2170: refactored xml.dom.minidom.normalize, increasing both - its clarity and its speed. - -- Issue #1113244: Py_XINCREF, Py_DECREF, Py_XDECREF: Add ``do { ... } while (0)`` - to avoid compiler warnings. - -- Issue #3739: The unicode-internal encoder now reports the number of characters - consumed like any other encoder (instead of the number of bytes). - -Installation ------------- - -- Issue #5756: Install idle and pydoc with a 3 suffix. - -Library -------- - -- Issue #8203: Fix IDLE Credits dialog: view_file() uses its encoding argument. - -- Issue #5311: bdist_msi can now build packages that do not depend on a - specific Python version. - -- Issue #5150: IDLE's format menu now has an option to strip trailing - whitespace. - -- Issue #5940: distutils.command.build_clib.check_library_list was not doing - the right type checkings anymore. - -- Issue #4875: On win32, ctypes.util.find_library does no longer - return directories. - -- Issue #5142: Add the ability to skip modules while stepping to pdb. - -- Issue #1309567: Fix linecache behavior of stripping subdirectories when - looking for files given by a relative filename. - -- Issue #5923: Update the ``turtle`` module to version 1.1, add two new - turtle demos in Demo/turtle. - -- Issue #5692: In ``zipfile.Zipfile``, fix wrong path calculation when - extracting a file to the root directory. - -- Issue #5913: os.listdir() should fail for empty path on windows. - -- Issue #5084: unpickling now interns the attribute names of pickled objects, - saving memory and avoiding growth in size of subsequent pickles. Proposal - and original patch by Jake McGuire. - -- The json module now works exclusively with str and not bytes. - -- Issue #3959: The ipaddr module has been added to the standard library. - Contributed by Google. - -- Issue #3002: ``shutil.copyfile()`` and ``shutil.copytree()`` now raise an - error when a named pipe is encountered, rather than blocking infinitely. - -- Issue #5857: tokenize.tokenize() now returns named tuples. - -- Issue #4305: ctypes should now build again on mipsel-linux-gnu - -- Issue #1734234: Massively speedup ``unicodedata.normalize()`` when the - string is already in normalized form, by performing a quick check beforehand. - Original patch by Rauli Ruohonen. - -- Issue #5853: calling a function of the mimetypes module from several threads - at once could hit the recursion limit if the mimetypes database hadn't been - initialized before. - -- Issue #5854: Updated __all__ to include some missing names and remove some - names which should not be exported. - -- Issue #3102: All global symbols that the _ctypes extension defines - are now prefixed with 'Py' or '_ctypes'. - -- Issue #5041: ctypes does now allow pickling wide character. - -- Issue #5812: For the two-argument form of the Fraction constructor, - Fraction(m, n), m and n are permitted to be arbitrary Rational - instances. - -- Issue #5812: Fraction('1e6') is valid: more generally, any string - that's valid for float() is now valid for Fraction(), with the - exception of strings representing NaNs and infinities. - -- Issue #5734: BufferedRWPair was poorly tested and had several glaring - bugs. Patch by Brian Quinlan. - -- Issue #1161031: fix readwrite select flag handling: POLLPRI now - results in a handle_expt_event call, not handle_read_event, and POLLERR - and POLLNVAL now call handle_close, not handle_expt_event. Also, - dispatcher now has an 'ignore_log_types' attribute for suppressing - log messages, which is set to 'warning' by default. - -- Issue #2703: SimpleXMLRPCDispatcher.__init__: Provide default values for - new arguments introduced in 2.5. - -- Issue #5828 (Invalid behavior of unicode.lower): Fixed bogus logic in - makeunicodedata.py and regenerated the Unicode database (This fixes - u'\u1d79'.lower() == '\x00'). - -Extension Modules ------------------ - -- Issue #5881: Remove old undocumented compatibility interfaces in hashlib and - pwd. - -- Issue #5463: In struct module, remove deprecated float coercion - for integer type codes: struct.pack('L', 0.3) should now raise - an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. - The version number has been bumped to 0.3. - -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built - using Berkeley DB. - -Tests ------ - -- Issue #5354: New test support function import_fresh_module() makes - it easy to import both normal and optimised versions of modules. - test_heapq and test_warnings have been adjusted to use it, tests for - other modules with both C and Python implementations in the stdlib - can be adjusted to use it over time. - -- Issue #5837: Certain sequences of calls to set() and unset() for - support.EnvironmentVarGuard objects restored the environment variables - incorrectly on __exit__. - -C-API ------ - -- Issue #5630: A replacement PyCObject API, PyCapsule, has been added. - - -What's New in Python 3.1 alpha 2? -================================= - -*Release date: 2009-4-4* - -Core and Builtins ------------------ - -- Implement PEP 378, Format Specifier for Thousands Separator, for - integers. - -- Issue #5666: Py_BuildValue's 'c' code should create byte strings. - -- Issue #5499: The 'c' code for argument parsing functions now only accepts a - byte, and the 'C' code only accepts a unicode character. - -- Fix a problem in PyErr_NormalizeException that leads to "undetected errors" - when hitting the recursion limit under certain circumstances. - -- Issue #1665206: Remove the last eager import in _warnings.c and make it lazy. - -- Fix a segfault when running test_exceptions with coverage, caused by - insufficient checks in accessors of Exception.__context__. - -- Issue #5604: non-ASCII characters in module name passed to - imp.find_module() were converted to UTF-8 while the path is - converted to the default filesystem encoding, causing nonsense. - -- Issue #5126: str.isprintable() returned False for space characters. - -- Issue #4865: On MacOSX /Library/Python/2.7/site-packages is added to - the end sys.path, for compatibility with the system install of Python. - -- Issue #4688: Add a heuristic so that tuples and dicts containing only - untrackable objects are not tracked by the garbage collector. This can - reduce the size of collections and therefore the garbage collection overhead - on long-running programs, depending on their particular use of datatypes. - -- Issue #5512: Rewrite PyLong long division algorithm (x_divrem) to - improve its performance. Long divisions and remainder operations - are now between 50% and 150% faster. - -- Issue #4258: Make it possible to use base 2**30 instead of base - 2**15 for the internal representation of integers, for performance - reasons. Base 2**30 is enabled by default on 64-bit machines. Add - --enable-big-digits option to configure, which overrides the - default. Add sys.int_info structseq to provide information about - the internal format. - -- Issue #4474: PyUnicode_FromWideChar now converts characters outside - the BMP to surrogate pairs, on systems with sizeof(wchar_t) == 4 - and sizeof(Py_UNICODE) == 2. - -- Issue #5237: Allow auto-numbered fields in str.format(). For - example: '{} {}'.format(1, 2) == '1 2'. - -- Issue #5392: when a very low recursion limit was set, the interpreter would - abort with a fatal error after the recursion limit was hit twice. - -- Issue #3845: In PyRun_SimpleFileExFlags avoid invalid memory access with - short file names. - -Library -------- - -- Issue #2625: added missing items() call to the for loop in - mailbox.MH.get_message(). - -- Issue #5640: Fix _multibytecodec so that CJK codecs don't repeat - error substitutions from non-strict codec error callbacks in - incrementalencoder and StreamWriter. - -- Issue #5656: Fix the coverage reporting when running the test suite with - the -T argument. - -- Issue #5647: MutableSet.__iand__() no longer mutates self during iteration. - -- Issue #5624: Fix the _winreg module name still used in several modules. - -- Issue #5628: Fix io.TextIOWrapper.read() with an unreadable buffer. - -- Issue #5619: Multiprocessing children disobey the debug flag and causes - popups on windows buildbots. Patch applied to work around this issue. - -- Issue #5400: Added patch for multiprocessing on netbsd compilation/support - -- Issue #5387: Fixed mmap.move crash by integer overflow. - -- Issue #5261: Patch multiprocessing's semaphore.c to support context - manager use: "with multiprocessing.Lock()" works now. - -- Issue #5236: Change time.strptime() to only take strings. Didn't work with - bytes already but the failure was non-obvious. - -- Issue #5177: Multiprocessing's SocketListener class now uses - socket.SO_REUSEADDR on all connections so that the user no longer needs - to wait 120 seconds for the socket to expire. - -- Issue #5595: Fix UnboundedLocalError in ntpath.ismount(). - -- Issue #1174606: Calling read() without arguments of an unbounded file - (typically /dev/zero under Unix) could crash the interpreter. - -- The max_buffer_size arguments of io.BufferedWriter, io.BufferedRWPair, and - io.BufferedRandom have been deprecated for removal in Python 3.2. - -- Issue #5068: Fixed the tarfile._BZ2Proxy.read() method that would loop - forever on incomplete input. That caused tarfile.open() to hang when used - with mode 'r' or 'r:bz2' and a fileobj argument that contained no data or - partial bzip2 compressed data. - -- Issue #2110: Add support for thousands separator and 'n' type - specifier to Decimal.__format__ - -- Fix Decimal.__format__ bug that swapped the meanings of the '<' and - '>' alignment characters. - -- The error detection code in FileIO.close() could fail to reflect the `errno` - value, and report it as -1 instead. - -- Issue #5016: FileIO.seekable() could return False if the file position - was negative when truncated to a C int. Patch by Victor Stinner. - -Extension Modules ------------------ - -- Issue #5391: mmap now deals exclusively with bytes. - -- Issue #5463: In struct module, remove deprecated overflow wrapping - when packing an integer: struct.pack('=L', -1) now raises - struct.error instead of returning b'\xff\xff\xff\xff'. The - _PY_STRUCT_RANGE_CHECKING and _PY_STRUCT_OVERFLOW_MASKING constants - have been removed from the struct module. - - -What's New in Python 3.1 alpha 1 -================================ - -*Release date: 2009-03-07* - -Core and Builtins ------------------ - -- The io module has been reimplemented in C for speed. - -- Give dict views an informative __repr__. - -- Issue #5247: Improve error message when unknown format codes are - used when using str.format() with str, int, and float arguments. - -- Issue #5249: time.strftime returned malformed string when format string - contained non ascii character on windows. - -- Issue #4626: compile(), exec(), and eval() ignore the coding cookie if the - source has already been decoded into str. - -- Issue #5186: Reduce hash collisions for objects with no __hash__ method by - rotating the object pointer by 4 bits to the right. - -- Issue #4575: Fix Py_IS_INFINITY macro to work correctly on x87 FPUs: - it now forces its argument to double before testing for infinity. - -- Issue #5137: Make len() correctly raise a TypeError when a __len__ method - returns a non-number type. - -- Issue #5182: Removed memoryview.__str__. - -- Issue #1717: Removed builtin cmp() function, dropped tp_compare - slot, the C API functions PyObject_Compare and PyUnicode_Compare and - the type definition cmpfunc. The tp_compare slot has been renamed - to tp_reserved, and is reserved for future usage. - -- Issue #1242657: the __len__() and __length_hint__() calls in several tools - were suppressing all exceptions. These include list() and bytearray(). - -- Issue #4707: round(x, n) now returns an integer if x is an integer. - Previously it returned a float. - -- Issue #4753: By enabling a configure option named '--with-computed-gotos' - on compilers that support it (notably: gcc, SunPro, icc), the bytecode - evaluation loop is compiled with a new dispatch mechanism which gives - speedups of up to 20%, depending on the system, on various benchmarks. - -- Issue #4874: Most builtin decoders now reject unicode input. - -- Issue #4842: Don't allow trailing 'L' when constructing an integer - from a string. - -- Issue #4991: os.fdopen now raises an OSError for invalid file descriptors. - -- Issue #4838: When a module is deallocated, free the memory backing the - optional module state data. - -- Issue #4910: Rename nb_long slot to nb_reserved, and change its - type to ``(void *)``. - -- Issue #4935: The overflow checking code in the expandtabs() method common - to str, bytes and bytearray could be optimized away by the compiler, letting - the interpreter segfault instead of raising an error. - -- Issue #3720: Fix a crash when an iterator modifies its class and removes its - __next__ method. - -- Issue #4910: Builtin int() function and PyNumber_Long/PyNumber_Int API - function no longer attempt to call the __long__ slot to convert an object - to an integer. Only the __int__ and __trunc__ slots are examined. - -- Issue #4893: Use NT threading on CE. - -- Issue #4915: Port sysmodule to Windows CE. - -- Issue #4868: utf-8, utf-16 and latin1 decoding are now 2x to 4x faster. The - common cases are optimized thanks to a dedicated fast path and a moderate - amount of loop unrolling. - -- Issue #4074: Change the criteria for doing a full garbage collection (i.e. - collecting the oldest generation) so that allocating lots of objects without - destroying them does not show quadratic performance. Based on a proposal by - Martin von Löwis at - http://mail.python.org/pipermail/python-dev/2008-June/080579.html. - -- Issue #4604: Some objects of the I/O library could still be used after - having been closed (for instance, a read() call could return some - previously buffered data). Patch by Dmitry Vasiliev. - -- Issue #4705: Fix the -u ("unbuffered binary stdout and stderr") command-line - flag to work properly. Furthermore, when specifying -u, the text stdout - and stderr streams have line-by-line buffering enabled (the default being - to buffer arbitrary chunks of data). - -- The internal table, _PyLong_DigitValue, is now an array of unsigned chars - instead of ints (reducing its size from 4 to 8 times thereby reducing - Python's overall memory). - -- Issue #1180193: When importing a module from a .pyc (or .pyo) file with - an existing .py counterpart, override the co_filename attributes of all - code objects if the original filename is obsolete (which can happen if the - file has been renamed, moved, or if it is accessed through different paths). - Patch by Ziga Seilnacht and Jean-Paul Calderone. - -- Issue #4580: Fix slicing of memoryviews when the item size is greater than - one byte. Also fixes the meaning of len() so that it returns the number of - items, rather than the size in bytes. - -- Issue #4075: Use OutputDebugStringW in Py_FatalError. - -- Issue #4747: When the terminal does not use utf-8, executing a script with - non-ascii characters in its name could fail with a "SyntaxError: None" error. - -- Issue #4797: IOError.filename was not set when ``_fileio.FileIO`` failed - to open file with a bytes filename on Windows. - -- Issue #3680: Reference cycles created through a dict, set or deque iterator - did not get collected. - -- Issue #4701: PyObject_Hash now implicitly calls PyType_Ready on types - where the tp_hash and tp_dict slots are both NULL. - -- Issue #4759: None is now allowed as the first argument of - bytearray.translate(). It was always allowed for bytes.translate(). - -- Added test case to ensure attempts to read from a file opened for writing - fail. - -- Issue #3106: Speedup some comparisons (str/str and int/int). - -- Issue #2183: Simplify and optimize bytecode for list, dict and set - comprehensions. Original patch for list comprehensions by Neal Norwitz. - -- Issue #2467: gc.DEBUG_STATS reported invalid elapsed times. Also, always - print elapsed times, not only when some objects are uncollectable / - unreachable. Original patch by Neil Schemenauer. - -- Issue #3439: Add a bit_length method to int. - -- Issue #2173: When getting device encoding, check that return value of - nl_langinfo is not the empty string. This was causing silent build - failures on OS X. - -- Issue #4597: Fixed several opcodes that weren't always propagating - exceptions. - -- Issue #4589: Fixed exception handling when the __exit__ function of a - context manager returns a value that cannot be converted to a bool. - -- Issue #4445: Replace "sizeof(PyBytesObject)" with - "offsetof(PyBytesObject, ob_sval) + 1" when allocating memory for - bytes instances. On a typical machine this saves 3 bytes of memory - (on average) per allocation of a bytes instance. - -- Issue #4533: File read operation was dreadfully slow due to a slowly - growing read buffer. Fixed by using the same growth rate algorithm as - Python 2.x. - -- Issue #4509: Various issues surrounding resize of bytearray objects to - which there are buffer exports (e.g. memoryview instances). - -- Issue #4233: Changed semantic of ``_fileio.FileIO``'s ``close()`` - method on file objects with closefd=False. The file descriptor is still - kept open but the file object behaves like a closed file. The ``FileIO`` - object also got a new readonly attribute ``closefd``. - -- Issue #4569: Interpreter crash when mutating a memoryview with an item size - larger than 1. - -- Issue #4748: Lambda generators no longer return a value. - -- The re.sub(), re.subn() and re.split() functions now accept a flags parameter. - -- Issue #5108: Handle %s like %S, %R and %A in PyUnicode_FromFormatV(): Call - PyUnicode_DecodeUTF8() once, remember the result and output it in a second - step. This avoids problems with counting UTF-8 bytes that ignores the effect - of using the replace error handler in PyUnicode_DecodeUTF8(). - -Library -------- - -- Issue #7071: byte-compilation in Distutils is now done with respect to - sys.dont_write_bytecode. - -- Issue #7066: archive_util.make_archive now restores the cwd if an error is - raised. Initial patch by Ezio Melotti. - -- Issue #6516: Added owner/group support when creating tar archives in - Distutils. - -- Issue #6954: Fixed crash when using DISTUTILS_DEBUG flag in Distutils. - -- Issue #6163: Fixed HP-UX runtime library dir options in - distutils.unixcompiler. Initial patch by Sridhar Ratnakumar and - Michael Haubenwallner. - -- Issue #6693: New functions in site.py to get user/global site packages paths. - -- Issue #6511: ZipFile now raises BadZipfile (instead of an IOError) when - opening an empty or very small file. - -- Issue #6545: Removed assert statements in distutils.Extension, so the - behavior is similar when used with -O. - -- unittest has been split up into a package. All old names should still work. - -- Issue #6466: now distutils.cygwinccompiler and distutils.emxccompiler - uses the same refactored function to get gcc/ld/dllwrap versions numbers. - It's `distutils.util.get_compiler_versions`. Added deprecation warnings - for the obsolete get_versions() functions. - -- Issue #6433: fixed issues with multiprocessing.pool.map hanging on empty list - -- Issue #6314: logging: Extra checks on the "level" argument in more places. - -- Issue #2622: Fixed an ImportError when importing email.message from a - standalone application built with py2exe or py2app. - -- Issue #6455: Fixed test_build_ext under win32. - -- Issue #6377: Enabled the compiler option, and deprecate its usage as an - attribute. - -- Issue #6413: Fixed the log level in distutils.dist for announce. - -- Issue #6403: Fixed package path usage in build_ext. - -- Issues #5155, 5313, 5331: multiprocessing.Process._bootstrap was - unconditionally calling "os.close(sys.stdin.fileno())" resulting in file - descriptor errors - -- Issue #6365: Distutils build_ext inplace mode was copying the compiled - extension in a subdirectory if the extension name had dots. - -- Issue #6164: Added an AIX specific linker argument in Distutils - unixcompiler. Original patch by Sridhar Ratnakumar. - -- Issue #6286: Now Distutils upload command is based on urllib2 instead of - httplib, allowing the usage of http_proxy. - -- Issue #6287: Added the license field in Distutils documentation. - -- Issue #6263: Fixed syntax error in distutils.cygwincompiler. - -- Issue #5201: distutils.sysconfig.parse_makefile() now understands `$$` - in Makefiles. This prevents compile errors when using syntax like: - `LDFLAGS='-rpath=\$$LIB:/some/other/path'`. Patch by Floris Bruynooghe. - -- Issue #6131: test_modulefinder leaked when run after test_distutils. - Patch by Hirokazu Yamamoto. - -- Issue #6048: Now Distutils uses the tarfile module in archive_util. - -- Issue #6062: In distutils, fixed the package option of build_ext. Feedback - and tests on pywin32 by Tim Golden. - -- Issue #6053: Fixed distutils tests on win32. patch by Hirokazu Yamamoto. - -- Issue #6046: Fixed the library extension when distutils build_ext is used - inplace. Initial patch by Roumen Petrov. - -- Issue #6041: Now distutils `sdist` and `register` commands use `check` as a - subcommand. - -- Issue #6022: a test file was created in the current working directory by - test_get_outputs in Distutils. - -- Issue #5977: distutils build_ext.get_outputs was not taking into account the - inplace option. Initial patch by kxroberto. - -- Issue #5984: distutils.command.build_ext.check_extensions_list checks were broken - for old-style extensions. - -- Issue #5976: Fixed Distutils test_check_environ. - -- Issue #5941: Distutils build_clib command was not working anymore because - of an incomplete customization of the archiver command. Added ARFLAGS in the - Makefile besides AR and make Distutils use it. Original patch by David - Cournapeau. - -- Issue #2245: aifc now skips chunk types it doesn't recognize, per spec. - -- Issue #5874: distutils.tests.test_config_cmd is not locale-sensitive - anymore. - -- Issue #5810: Fixed Distutils test_build_scripts so it uses - sysconfig.get_config_vars. - -- Issue #4951: Fixed failure in test_httpservers. - -- Issue #5795: Fixed test_distutils failure on Debian ppc. - -- Issue #5607: fixed Distutils test_get_platform for Mac OS X fat binaries. - -- Issue #5741: don't disallow "%%" (which is an escape for "%") when setting - a value in SafeConfigParser. - -- Issue #5732: added a new command in Distutils: check. - -- Issue #5731: Distutils bdist_wininst no longer worked on non-Windows - platforms. Initial patch by Paul Moore. - -- Issue #5095: Added bdist_msi to the list of bdist supported formats. - Initial fix by Steven Bethard. - -- Issue #1491431: Fixed distutils.filelist.glob_to_re for edge cases. - Initial fix by Wayne Davison. - -- Issue #5694: removed spurious test output in Distutils (test_clean). - -- Issue #1326077: fix the formatting of SyntaxErrors by the traceback module. - -- Issue #1665206 (partially): Move imports in cgitb to the top of the module - instead of performing them in functions. Helps prevent import deadlocking in - threads. - -- Issue #2522: locale.format now checks its first argument to ensure it has - been passed only one pattern, avoiding mysterious errors where it appeared - that it was failing to do localization. - -- Issue #5583: Added optional Extensions in Distutils. Initial patch by Georg - Brandl. - -- Issue #1222: locale.format() bug when the thousands separator is a space - character. - -- Issue #5472: Fixed distutils.test_util tear down. Original patch by - Tim Golden. - -- collections.deque() objects now have a read-only attribute called maxlen. - -- Issue #2638: Show a window constructed with tkSimpleDialog.Dialog only after - it is has been populated and properly configured in order to prevent - window flashing. - -- Issue #4792: Prevent a segfault in _tkinter by using the - guaranteed to be safe interp argument given to the PythonCmd in place of - the Tcl interpreter taken from a PythonCmd_ClientData. - -- Issue #5193: Guarantee that tkinter.Text.search returns a string. - -- Issue #5394: removed > 2.3 syntax from distutils.msvc9compiler. - Original patch by Akira Kitada. - -- Issue #5334: array.fromfile() failed to insert values when EOFError was raised. - -- Issue #5385: Fixed mmap crash after resize failure on windows. - -- Issue #5179: Fixed subprocess handle leak on failure on windows. - -- PEP 372: Added collections.OrderedDict(). - -- The _asdict() for method for namedtuples now returns an OrderedDict(). - -- configparser now defaults to using an ordered dictionary. - -- Issue #5401: Fixed a performance problem in mimetypes when ``from mimetypes - import guess_extension`` was used. - -- Issue #1733986: Fixed mmap crash in accessing elements of second map object - with same tagname but larger size than first map. (Windows) - -- Issue #5386: mmap.write_byte didn't check map size, so it could cause buffer - overrun. - -- Issue #1533164: Installed but not listed ``*.pyo`` was breaking Distutils - bdist_rpm command. - -- Issue #5378: added --quiet option to Distutils bdist_rpm command. - -- Issue #5052: make Distutils compatible with 2.3 again. - -- Issue #5316: Fixed buildbot failures introduced by multiple inheritance - in Distutils tests. - -- Issue #5287: Add exception handling around findCaller() call to help out - IronPython. - -- Issue #5282: Fixed mmap resize on 32bit windows and unix. When offset > 0, - The file was resized to wrong size. - -- Issue #5292: Fixed mmap crash on its boundary access m[len(m)]. - -- Issue #2279: distutils.sdist.add_defaults now add files - from the package_data and the data_files metadata. - -- Issue #5257: refactored all tests in distutils, so they use - support.TempdirManager, to avoid writing in the tests directory. - -- Issue #4524: distutils build_script command failed with --with-suffix=3. - Initial patch by Amaury Forgeot d'Arc. - -- Issue #2461: added tests for distutils.util - -- Issue #4998: The memory saving effect of __slots__ had been lost on Fractions - which inherited from numbers.py which did not have __slots__ defined. The - numbers hierarchy now has its own __slots__ declarations. - -- Issue #4631: Fix urlopen() result when an HTTP response uses chunked - encoding. - -- Issue #5203: Fixed ctypes segfaults when passing a unicode string to a - function without argtypes (only occurs if HAVE_USABLE_WCHAR_T is false). - -- Issue #3386: distutils.sysconfig.get_python_lib prefix argument was ignored - under NT and OS2. Patch by Philip Jenvey. - -- Issue #5128: Make compileall properly inspect bytecode to determine if needs - to be recreated. This avoids a timing hole thanks to the old reliance on the - ctime of the files involved. - -- Issue #5122: Synchronize tk load failure check to prevent a potential - deadlock. - -- Issue #1818: collections.namedtuple() now supports a keyword argument - 'rename' which lets invalid fieldnames be automatically converted to - positional names in the form, _1, _2, ... - -- Issue #4890: Handle empty text search pattern in Tkinter.Text.search. - -- Issue #4512 (part 2): Promote ``ZipImporter._get_filename()`` to be a - public documented method ``ZipImporter.get_filename()``. - -- Issue #4195: The ``runpy`` module (and the ``-m`` switch) now support - the execution of packages by looking for and executing a ``__main__`` - submodule when a package name is supplied. Initial patch by Andi - Vajda. - -- Issue #1731706: Call Tcl_ConditionFinalize for Tcl_Conditions that will - not be used again (this requires Tcl/Tk 8.3.1), also fix a memory leak in - Tkapp_Call when calling from a thread different than the one that created - the Tcl interpreter. Patch by Robert Hancock. - -- Issue #4285: Change sys.version_info to be a named tuple. Patch by - Ross Light. - -- Issue #1520877: Now distutils.sysconfig reads $AR from the - environment/Makefile. Patch by Douglas Greiman. - -- Issue #1276768: The verbose option was not used in the code of - distutils.file_util and distutils.dir_util. - -- Issue #5132: Fixed trouble building extensions under Solaris with - --enabled-shared activated. Initial patch by Dave Peterson. - -- Issue #1581476: Always use the Tcl global namespace when calling into Tcl. - -- The shelve module now defaults to pickle protocol 3. - -- Fix a bug in the trace module where a bytes object from co_lnotab had its - items being passed through ord(). - -- Issue #2047: shutil.move() could believe that its destination path was - inside its source path if it began with the same letters (e.g. "src" vs. - "src.new"). - -- Added the ttk module. See issue #2983: Ttk support for Tkinter. - -- Removed isSequenceType(), isMappingType, and isNumberType() from the - operator module; use the abstract base classes instead. Also removed - the repeat() function; use mul() instead. - -- Issue #5021: doctest.testfile() did not create __name__ and - collections.namedtuple() relied on __name__ being defined. - -- Backport importlib from Python 3.1. Only the import_module() function has - been backported to help facilitate transitions from 2.7 to 3.1. - -- Issue #1885: distutils. When running sdist with --formats=tar,gztar - the tar file was overridden by the gztar one. - -- Issue #4863: distutils.mwerkscompiler has been removed. - -- Added a new itertools functions: combinations_with_replacement() - and compress(). - -- Issue #5032: added a step argument to itertools.count() and - allowed non-integer arguments. - -- Fix and properly document the multiprocessing module's logging - support, expose the internal levels and provide proper usage - examples. - -- Issue #1672332: fix unpickling of subnormal floats, which was - producing a ValueError on some platforms. - -- Issue #3881: Help Tcl to load even when started through the - unreadable local symlink to "Program Files" on Vista. - -- Issue #4710: Extract directories properly in the zipfile module; - allow adding directories to a zipfile. - -- Issue #3807: _multiprocessing build fails when configure is passed - --without-threads argument. When this occurs, _multiprocessing will - be disabled, and not compiled. - -- Issue #5008: When a file is opened in append mode with the new IO library, - do an explicit seek to the end of file (so that e.g. tell() returns the - file size rather than 0). This is consistent with the behaviour of the - traditional 2.x file object. - -- Issue #5013: Fixed a bug in FileHandler which occurred when the delay - parameter was set. - -- Issue #4842: Always append a trailing 'L' when pickling longs using - pickle protocol 0. When reading, the 'L' is optional. - -- Add the importlib package. - -- Issue #4301: Patch the logging module to add processName support, remove - _check_logger_class from multiprocessing. - -- Issue #3325: Remove python2.x try: except: imports for old cPickle from - multiprocessing. - -- Issue #4959: inspect.formatargspec now works for keyword only arguments - without defaults. - -- Issue #3321: ``_multiprocessing.Connection()`` doesn't check handle; added checks - for Unix machines for negative handles and large int handles. Without this check - it is possible to segfault the interpreter. - -- Issue #4449: AssertionError in mp_benchmarks.py, caused by an underlying issue - in sharedctypes.py. - -- Issue #1225107: inspect.isclass() returned True for instances with a custom - __getattr__. - -- Issue #3826 and #4791: The socket module now closes the underlying socket - appropriately when it is being used via socket.makefile() objects - rather than delaying the close by waiting for garbage collection to do it. - -- Issue #1696199: Add collections.Counter() for rapid and convenient - counting. - -- Issue #3860: GzipFile and BZ2File now support the context management protocol. - -- Issue #4867: Fixed a crash in ctypes when passing a string to a - function without defining argtypes. - -- Issue #4272: Add an optional argument to the GzipFile constructor to override - the timestamp in the gzip stream. The default value remains the current time. - The information can be used by e.g. gunzip when decompressing. Patch by - Jacques Frechet. - -- Restore Python 2.3 compatibility for decimal.py. - -- Issue #3638: Remove functions from _tkinter module level that depend on - TkappObject to work with multiple threads. - -- Issue #4718: Adapt the wsgiref package so that it actually works with - Python 3.x, in accordance with the `official amendments of the spec - `_. - -- Issue #4796: Added Decimal.from_float() and Context.create_decimal_from_float() - to the decimal module. - -- Fractions.from_float() no longer loses precision for integers too big to - cast as floats. - -- Issue #4812: add missing underscore prefix to some internal-use-only - constants in the decimal module. (Dec_0 becomes _Dec_0, etc.) - -- Issue #4790: The nsmallest() and nlargest() functions in the heapq module - did unnecessary work in the common case where no key function was specified. - -- Issue #4795: inspect.isgeneratorfunction() returns False instead of None when - the function is not a generator. - -- Issue #4702: Throwing a DistutilsPlatformError instead of IOError in case - no MSVC compiler is found under Windows. Original patch by Philip Jenvey. - -- Issue #4646: distutils was choking on empty options arg in the setup - function. Original patch by Thomas Heller. - -- Issue #3767: Convert Tk object to string in tkColorChooser. - -- Issue #3248: Allow placing ScrolledText in a PanedWindow. - -- Issue #4444: Allow assertRaises() to be used as a context handler, so that - the code under test can be written inline if more practical. - -- Issue #4739: Add pydoc help topics for symbols, so that e.g. help('@') - works as expected in the interactive environment. - -- Issue #4756: zipfile.is_zipfile() now supports file-like objects. Patch by - Gabriel Genellina. - -- Issue #4574: reading a UTF16-encoded text file crashes if \r on 64-char - boundary. - -- Issue #4223: inspect.getsource() will now correctly display source code - for packages loaded via zipimport (or any other conformant PEP 302 - loader). Original patch by Alexander Belopolsky. - -- Issue #4201: pdb can now access and display source code loaded via - zipimport (or any other conformant PEP 302 loader). Original patch by - Alexander Belopolsky. - -- Issue #4197: doctests in modules loaded via zipimport (or any other PEP - 302 conformant loader) will now work correctly in most cases (they - are still subject to the constraints that exist for all code running - from inside a module loaded via a PEP 302 loader and attempting to - perform IO operations based on __file__). Original patch by - Alexander Belopolsky. - -- Issues #4082 and #4512: Add runpy support to zipimport in a manner that - allows backporting to maintenance branches. Original patch by - Alexander Belopolsky. - -- Issue #4163: textwrap module: allow word splitting on a hyphen preceded by - a non-ASCII letter. - -- Issue #4616: TarFile.utime(): Restore directory times on Windows. - -- Issue #4021: tokenize.detect_encoding() now raises a SyntaxError when the - codec cannot be found. This is for compatibility with the builtin behavior. - -- Issue #4084: Fix max, min, max_mag and min_mag Decimal methods to - give correct results in the case where one argument is a quiet NaN - and the other is a finite number that requires rounding. - -- Issue #4483: _dbm module now builds on systems with gdbm & gdbm_compat - libs. - -- Added the subprocess.check_call_output() convenience function to get output - from a subprocess on success or raise an exception on error. - -- Issue #1055234: cgi.parse_header(): Fixed parsing of header parameters to - support unusual filenames (such as those containing semi-colons) in - Content-Disposition headers. - -- Issue #4384: Added logging integration with warnings module using - captureWarnings(). This change includes a NullHandler which does nothing; - it will be of use to library developers who want to avoid the "No handlers - could be found for logger XXX" message which can appear if the library user - doesn't configure logging. - -- Issue #3741: DISTUTILS_USE_SDK set causes msvc9compiler.py to raise an - exception. - -- Issue #4529: fix the parser module's validation of try-except-finally - statements. - -- Issue #4458: getopt.gnu_getopt() now recognizes a single "-" as an argument, - not a malformed option. - -- Added the subprocess.check_output() convenience function to get output - from a subprocess on success or raise an exception on error. - -- Issue #4542: On Windows, binascii.crc32 still accepted str as binary input; - the corresponding tests now pass. - -- Issue #4537: webbrowser.UnixBrowser would fail to open the browser because - it was calling the wrong open() function. - -- Issue #1055234: cgi.parse_header(): Fixed parsing of header parameters to - support unusual filenames (such as those containing semi-colons) in - Content-Disposition headers. - -- Issue #4861: ctypes.util.find_library(): Robustify. Fix library detection on - biarch systems. Try to rely on ldconfig only, without using objdump and gcc. - -- Issue #5104: The socket module now raises OverflowError when 16-bit port and - protocol numbers are supplied outside the allowed 0-65536 range on bind() - and getservbyport(). - -- Windows locale mapping updated to Vista. - -Tools/Demos ------------ - -- Issue #4704: remove use of cmp() in pybench, bump its version number to 2.1, - and make it 2.6-compatible. - -- Ttk demos added in Demo/tkinter/ttk/ - -- Issue #4677: add two list comprehension tests to pybench. - - -Build ------ - -- Issue #6094: Build correctly with Subversion 1.7. - -- Issue #5847: Remove -n switch on "Edit with IDLE" menu item. - -- Issue #5726: Make Modules/ld_so_aix return the actual exit code of the - linker, rather than always exit successfully. Patch by Floris Bruynooghe. - -- Issue #4587: Add configure option --with-dbmliborder=db1:db2:... to specify - the order that backends for the dbm extension are checked. - -- Link the shared python library with $(MODLIBS). - -- Issue #5134: Silence compiler warnings when compiling sqlite with VC++. - -- Issue #4494: Fix build with Py_NO_ENABLE_SHARED on Windows. - -- Issue #4895: Use _strdup on Windows CE. - -- Issue #4472: "configure --enable-shared" now works on OSX - -- Issues #4728 and #4060: WORDS_BIGEDIAN is now correct in Universal builds. - -- Issue #4389: Add icon to the uninstall entry in "add-and-remove-programs". - -- Issue #4289: Remove Cancel button from AdvancedDlg. - -- Issue #1656675: Register a drop handler for .py* files on Windows. - -- Issue #4120: Exclude manifest from extension modules in VS2008. - -- Issue #4091: Install pythonxy.dll in system32 again. - -- Issue #4018: Disable "for me" installations on Vista. - -- Issue #3758: Add ``patchcheck`` build target to .PHONY. - -- Issue #4204: Fixed module build errors on FreeBSD 4. - - -C-API ------ - -- Issue #6624: yArg_ParseTuple with "s" format when parsing argument with - NUL: Bogus TypeError detail string. - -- Issue #5175: PyLong_AsUnsignedLongLong now raises OverflowError - for negative arguments. Previously, it raised TypeError. - -- Issue #4720: The format for PyArg_ParseTupleAndKeywords can begin with '|'. - -- Issue #3632: from the gdb debugger, the 'pyo' macro can now be called when - the GIL is released, or owned by another thread. - -- Issue #4122: On Windows, fix a compilation error when using the - Py_UNICODE_ISSPACE macro in an extension module. - - -Extension Modules ------------------ - -- Issue #3745: Fix hashlib to always reject unicode and non buffer-api - supporting objects as input no matter how it was compiled (built in - implementations or external openssl library). - -- Issue #4397: Fix occasional test_socket failure on OS X. - -- Issue #4279: Fix build of parsermodule under Cygwin. - -- Issue #4751: hashlib now releases the GIL when hashing large buffers - (with a hardwired threshold of 2048 bytes), allowing better parallelization - on multi-CPU systems. Contributed by Lukas Lueg (ebfe) and Victor Stinner. - -- Issue #4051: Prevent conflict of UNICODE macros in cPickle. - -- Issue #4738: Each zlib object now has a separate lock, allowing several streams - to be compressed or decompressed at once on multi-CPU systems. Also, the GIL - is now released when computing the CRC of a large buffer. Patch by ebfe. - -- Issue #4228: Pack negative values the same way as 2.4 in struct's L format. - -- Issue #1040026: Fix os.times result on systems where HZ is incorrect. - -- Issues #3167, #3682: Fix test_math failures for log, log10 on Solaris, - OpenBSD. - -- Issue #4583: array.array would not always prohibit resizing when a buffer - has been exported, resulting in an interpreter crash when accessing the - buffer. - - -- Issue #5228: Make functools.partial objects can now be pickled. - -Tests ------ - -- Issue #6152: New option '-j'/'--multiprocess' for regrtest allows running - regression tests in parallel, shortening the total runtime. - -- Issue #5450: Moved tests involving loading tk from Lib/test/test_tcl to - Lib/tkinter/test/test_tkinter/test_loadtk. With this, these tests demonstrate - the same behaviour as test_ttkguionly (and now also test_tk) which is to - skip the tests if DISPLAY is defined but can't be used. - -- regrtest no longer treats ImportError as equivalent to SkipTest. Imports - that should cause a test to be skipped are now done using import_module - from test support, which does the conversion. - -- Issue #5083: New 'gui' resource for regrtest. - - -Docs ----- - - -What's New in Python 3.0 final -============================== - -*Release date: 03-Dec-2008* - -Core and Builtins ------------------ - -- Issue #3996: On Windows, the PyOS_CheckStack function would cause the - interpreter to abort ("Fatal Python error: Could not reset the stack!") - instead of throwing a MemoryError. - -- Issue #3689: The list reversed iterator now supports __length_hint__ - instead of __len__. Behavior now matches other reversed iterators. - -- Issue #4367: Python would segfault during compiling when the unicodedata - module couldn't be imported and \N escapes were present. - -- Fix build failure of _cursesmodule.c building with -D_FORTIFY_SOURCE=2. - -Library -------- - -- Issue #4387: binascii now refuses to accept str as binary input. - -- Issue #4073: Add 2to3 support to build_scripts, refactor that support - in build_py. - -- IDLE would print a "Unhandled server exception!" message when internal - debugging is enabled. - -- Issue #4455: IDLE failed to display the windows list when two windows have - the same title. - -- Issue #3741: DISTUTILS_USE_SDK set causes msvc9compiler.py to raise an - exception. - -- Issue #4433: Fixed an access violation when garbage collecting - _ctypes.COMError instances. - -- Issue #4429: Fixed UnicodeDecodeError in ctypes. - -- Issue #4373: Corrected a potential reference leak in the pickle module and - silenced a false positive ref leak in distutils.tests.test_build_ext. - -- Issue #4382: dbm.dumb did not specify the expected file encoding for opened - files. - -- Issue #4383: When IDLE cannot make the connection to its subprocess, it would - fail to properly display the error message. - -Build ------ - -- Issue #4407: Fix source file that caused the compileall step in Windows installer - to fail. - -Docs ----- - -- Issue #4449: Fixed multiprocessing examples - -- Issue #3799: Document that dbm.gnu and dbm.ndbm will accept string arguments - for keys and values which will be converted to bytes before committal. - - -What's New in Python 3.0 release candidate 3? -============================================= - -*Release date: 20-Nov-2008* - - -Core and Builtins ------------------ - -- Issue #4349: sys.path included a non-existent platform directory because of a - faulty Makefile. - -- Issue #3327: Don't overallocate in the modules_by_index list. - -- Issue #1721812: Binary set operations and copy() returned the input type - instead of the appropriate base type. This was incorrect because set - subclasses would be created without their __init__() method being called. - The corrected behavior brings sets into line with lists and dicts. - -- Issue #4296: Fix PyObject_RichCompareBool so that "x in [x]" evaluates to - True, even when x doesn't compare equal to itself. This was a regression - from 2.6. - -- Issue #3705: Command-line arguments were not correctly decoded when the - terminal does not use UTF8. - -Library -------- - -- Issue #4363: The uuid.uuid1() and uuid.uuid4() functions now work even if - the ctypes module is not present. - -- FileIO's mode attribute now always includes ``"b"``. - -- Issue #3799: Fix dbm.dumb to accept strings as well as bytes for keys. String - keys are now written out in UTF-8. - -- Issue #4338: Fix distutils upload command. - -- Issue #4354: Fix distutils register command. - -- Issue #4116: Resolve member name conflict in ScrolledCanvas.__init__. - -- Issue #4307: The named tuple that ``inspect.getfullargspec()`` returns now - uses ``kwonlydefaults`` instead of ``kwdefaults``. - -- Issue #4298: Fix a segfault when pickle.loads is passed ill-formed input. - -- Issue #4283: Fix a left-over "iteritems" call in distutils. - -Build ------ - -- Issue #4389: Add icon to the uninstall entry in "add-and-remove-programs". - -- Issue #4289: Remove Cancel button from AdvancedDlg. - -- Issue #1656675: Register a drop handler for .py* files on Windows. - -Tools/Demos ------------ - -- Demos of the socketserver module now work with Python 3. - - -What's New in Python 3.0 release candidate 2 -============================================ - -*Release date: 05-Nov-2008* - -Core and Builtins ------------------ - -- Issue #4211: The __path__ attribute of frozen packages is now a list instead - of a string as required by PEP 302. - -- Issue #3727: Fixed poplib. - -- Issue #3714: Fixed nntplib by using bytes where appropriate. - -- Issue #1210: Fixed imaplib and its documentation. - -- Issue #4233: Changed semantic of ``_fileio.FileIO``'s ``close()`` - method on file objects with closefd=False. The file descriptor is still - kept open but the file object behaves like a closed file. The ``FileIO`` - object also got a new readonly attribute ``closefd``. - -- Issue #3626: On cygwin, starting python with a non-existent script name - would not display anything if the file name is only 1 character long. - -- Issue #4176: Fixed a crash when pickling an object which ``__reduce__`` - method does not return iterators for the 4th and 5th items. - -- Issue #3723: Fixed initialization of subinterpreters. - -- Issue #4213: The file system encoding is now normalized by the - codec subsystem, for example UTF-8 is turned into utf-8. - -- Issue #4200: Changed the atexit module to store its state in its - PyModuleDef atexitmodule. This fixes a bug with multiple subinterpeters. - -- Issue #4237: io.FileIO() was raising invalid warnings caused by - insufficient initialization of PyFileIOObject struct members. - -- Issue #4170: Pickling a collections.defaultdict object would crash the - interpreter. - -- Issue #4146: Compilation on OpenBSD has been restored. - -- Issue #3574: compile() incorrectly handled source code encoded as Latin-1. - -- Issues #2384 and #3975: Tracebacks were not correctly printed when the - source file contains a ``coding:`` header: the wrong line was displayed, and - the encoding was not respected. - -- Issue #3740: Null-initialize module state. - -- Issue #3946: PyObject_CheckReadBuffer crashed on a memoryview object. - -- Issue #1688: On Windows, the input() prompt was not correctly displayed if it - contains non-ascii characters. - -- Bug #3951: Py_USING_MEMORY_DEBUGGER should not be enabled by default. - -Library -------- - -- Issue #3664: The pickle module could segfault if a subclass of Pickler fails - to call the base __init__ method. - -- Issue #3725: telnetlib now works completely in bytes. - -- Issue #4072: Restore build_py_2to3. - -- Issue #4014: Don't claim that Python has an Alpha release status, in addition - to claiming it is Mature. - -- Issue #3187: Add sys.setfilesystemencoding. - -- Issue #3187: Better support for "undecodable" filenames. Code by Victor - Stinner, with small tweaks by GvR. - -- Issue #3965: Allow repeated calls to turtle.Screen, by making it a - true singleton object. - -- Issue #3911: ftplib.FTP.makeport() could give invalid port numbers. - -- Issue #3929: When the database cannot be opened, dbm.open() would incorrectly - raise a TypeError: "'tuple' object is not callable" instead of the expected - dbm.error. - -- Bug #3884: Make the turtle module toplevel again. - -- Issue #3547: Fixed ctypes structures bitfields of varying integer - sizes. - -Extension Modules ------------------ - -- Issue #3659: Subclasses of str didn't work as SQL parameters. - -Build ------ - -- Issue #4120: Exclude manifest from extension modules in VS2008. - -- Issue #4091: Install pythonxy.dll in system32 again. - -- Issue #4018: Disable "for me" installations on Vista. - -- Issue #4204: Fixed module build errors on FreeBSD 4. - -Tools/Demos ------------ - -- Issue #3717: Fix Demo/embed/demo.c. - -- Issue #4072: Add a distutils demo for build_py_2to3. - - -What's New in Python 3.0 release candidate 1 -============================================ - -*Release date: 17-Sep-2008* - -Core and Builtins ------------------ - -- Issue #3827: memoryview lost its size attribute in favor of using len(view). - -- Issue #3813: could not lanch python.exe via symbolic link on cygwin. - -- Issue #3705: fix crash when given a non-ascii value on the command line for - the "-c" and "-m" parameters. Now the behaviour is as expected under Linux, - although under Windows it fails at a later point. - -- Issue #3279: Importing site at interpreter was failing silently because the - site module uses the open builtin which was not initialized at the time. - -- Issue #3660: Corrected a reference leak in str.encode() when the encoder - does not return a bytes object. - -- Issue #3774: Added a few more checks in PyTokenizer_FindEncoding to handle - error conditions. - -- Issue #3594: Fix Parser/tokenizer.c:fp_setreadl() to open the file being - tokenized by either a file path or file pointer for the benefit of - PyTokenizer_FindEncoding(). - -- Issue #3696: Error parsing arguments on OpenBSD <= 4.4 and Cygwin. On - these systems, the mbstowcs() function is slightly buggy and must be - replaced with strlen() for the purpose of counting of number of wide - characters needed to represent the multi-byte character string. - -- Issue #3697: "Fatal Python error: Cannot recover from stack overflow" - could be easily encountered under Windows in debug mode when exercising - the recursion limit checking code, due to bogus handling of recursion - limit when USE_STACKCHEK was enabled. - -- Issue #3639: The _warnings module could segfault the interpreter when - unexpected types were passed in as arguments. - -- Issue #3712: The memoryview object had a reference leak and didn't support - cyclic garbage collection. - -- Issue #3668: Fix a memory leak with the "s*" argument parser in - PyArg_ParseTuple and friends, which occurred when the argument for "s*" - was correctly parsed but parsing of subsequent arguments failed. - -- Issue #3611: An exception __context__ could be cleared in a complex pattern - involving a __del__ method re-raising an exception. - -- Issue #2534: speed up isinstance() and issubclass() by 50-70%, so as to - match Python 2.5 speed despite the __instancecheck__ / __subclasscheck__ - mechanism. In the process, fix a bug where isinstance() and issubclass(), - when given a tuple of classes as second argument, were looking up - __instancecheck__ / __subclasscheck__ on the tuple rather than on each - type object. - -- Issue #3663: Py_None was decref'd when printing SyntaxErrors. - -- Issue #3651: Fix various memory leaks when using the buffer - interface, or when the "s#" code of PyArg_ParseTuple is given a - bytes object. - -- Issue #3657: Fix uninitialized memory read when pickling longs. - Found by valgrind. - -- Apply security patches from Apple. - -- Fix crashes on memory allocation failure found with failmalloc. - -- Fix memory leaks found with valgrind and update suppressions file. - -- Fix compiler warnings in opt mode which would lead to invalid memory reads. - -- Fix problem using wrong name in decimal module reported by pychecker. - -- Issue #3650: Fixed a reference leak in bytes.split('x'). - -- bytes(o) now tries to use o.__bytes__() before using fallbacks. - -- Issue #1204: The configure script now tests for additional libraries - that may be required when linking against readline. This fixes issues - with x86_64 builds on some platforms (a few Linux flavors and OpenBSD). - -C API ------ - -- PyObject_Bytes and PyBytes_FromObject were added. - -Library -------- - -- Issue #3756: make re.escape() handle bytes as well as str. - -- Issue #3800: fix filter() related bug in formatter.py. - -- Issue #874900: fix behaviour of threading module after a fork. - -- Issue #3535: zipfile couldn't read some zip files larger than 2GB. - -- Issue #3776: Deprecate the bsddb package for removal in 3.0. - -- Issue #3762: platform.architecture() fails if python is lanched via - its symbolic link. - -- Issue #3660: fix a memory leak in the C accelerator of the pickle module. - -- Issue #3160: the "bdist_wininst" distutils command didn't work. - -- Issue #1658: tkinter changes dict size during iteration in both - tkinter.BaseWidget and tkinter.scrolledtext.ScrolledText. - -- The bsddb module (and therefore the dbm.bsd module) has been removed. - It is now maintained outside of the standard library at - http://www.jcea.es/programacion/pybsddb.htm. - -- Issue #600362: Relocated parse_qs() and parse_qsl(), from the cgi module - to the urlparse one. Added a DeprecationWarning in the old module, it - will be deprecated in the future. - -- Issue #3719: platform.architecture() fails if there are spaces in the - path to the Python binary. - -- Issue #3602: As part of the merge of r66135, make the parameters on - warnings.catch_warnings() keyword-only. Also remove a DeprecationWarning. - -- The deprecation warnings for the camelCase threading API names were removed. - -- Issue #3110: multiprocessing fails to compiel on solaris 10 due to missing - SEM_VALUE_MAX. - -Extension Modules ------------------ - -- Issue #3782: os.write() must not accept unicode strings. - -- Issue #2975: When compiling several extension modules with Visual Studio 2008 - from the same python interpreter, some environment variables would grow - without limit. - -- Issue #3643: Added a few more checks to _testcapi to prevent segfaults by - exploitation of poor argument checking. - -- bsddb code updated to version 4.7.3pre2. This code is the same than - Python 2.6 one, since the intention is to keep a unified 2.x/3.x codebase. - The Python code is automatically translated using "2to3". Please, do not - update this code in Python 3.0 by hand. Update the 2.6 one and then - do "2to3". - -- The _bytesio and _stringio modules are now compiled into the python binary. - -- Issue #3492 and #3790: Fixed the zlib module and zipimport module uses of - mutable bytearray objects where they should have been using immutable bytes. - -- Issue #3797: Fixed the dbm, marshal, mmap, ossaudiodev, & winreg modules to - return bytes objects instead of bytearray objects. - - -Tools/Demos ------------ - -- Fix Misc/gdbinit so it works. - - -Build ------ - -- Issue #3812: Failed to build python if configure --without-threads. - -- Issue #3791: Remove the bsddb module from the Windows installer, and the - core bsddb library from the Windows build files. - - -What's new in Python 3.0b3? -=========================== - -*Release date: 20-Aug-2008* - -Core and Builtins ------------------ - -- Issue #3653: Fix a segfault when sys.excepthook was called with invalid - arguments. - -- Issue #2394: implement more of the memoryview API, with the caveat that - only one-dimensional contiguous buffers are supported and exercised right - now. Slicing, slice assignment and comparison (equality and inequality) - have been added. Also, the tolist() method has been implemented, but only - for byte buffers. Finally, the API has been updated to return bytes objects - wherever it used to return bytearrays. - -- Issue #3560: clean up the new C PyMemoryView API so that naming is - internally consistent; add macros PyMemoryView_GET_BASE() and - PyMemoryView_GET_BUFFER() to access useful properties of a memory views - without relying on a particular implementation; remove the ill-named - PyMemoryView() function (PyMemoryView_GET_BUFFER() can be used instead). - -- ctypes function pointers that are COM methods have a boolean True - value again. - -- Issue #1819: function calls with several named parameters are now on - average 35% faster (as measured by pybench). - -- The undocumented C APIs PyUnicode_AsString() and - PyUnicode_AsStringAndSize() were made private to the interpreter, in - order to be able to refine their interfaces for Python 3.1. - - If you need to access the UTF-8 representation of a Unicode object - as bytes string, please use PyUnicode_AsUTF8String() instead. - -- Issue #3460: PyUnicode_Join() implementation is 10% to 80% faster thanks - to Python 3.0's stricter semantics which allow avoiding successive - reallocations of the result string (this also affects str.join()). - - -Library -------- - -- Issue #1276: Added temporary aliases for CJK Mac encodings to resolve - a build problem on MacOS with CJK locales. It adds four temporary - mappings to existing legacy codecs that are virtually compatible - with Mac encodings. They will be replaced by codecs correctly - implemented in 3.1. - -- Issue #3614: Corrected a typo in xmlrpc.client, leading to a NameError - "global name 'header' is not defined". - -- Issue #2834: update the regular expression library to match the unicode - standards of py3k. In other words, mixing bytes and unicode strings - (be it as pattern, search string or replacement string) raises a TypeError. - Moreover, the re.UNICODE flag is enabled automatically for unicode patterns, - and can be disabled by specifying a new re.ASCII flag; as for bytes - patterns, ASCII matching is the only option and trying to specify re.UNICODE - for such patterns raises a ValueError. - -- Issue #3300: make urllib.parse.[un]quote() default to UTF-8. - Code contributed by Matt Giuca. quote() now encodes the input - before quoting, unquote() decodes after unquoting. There are - new arguments to change the encoding and errors settings. - There are also new APIs to skip the encode/decode steps. - [un]quote_plus() are also affected. - -- Issue #2235: numbers.Number now blocks inheritance of the default id() - based hash because that hash mechanism is not correct for numeric types. - All concrete numeric types that inherit from Number (rather than just - registering with it) must explicitly provide a hash implementation in - order for their instances to be hashable. - -- Issue #2676: in the email package, content-type parsing was hanging on - pathological input because of quadratic or exponential behaviour of a - regular expression. - -- Issue #3476: binary buffered reading through the new "io" library is now - thread-safe. - -- Issue #1342811: Fix leak in Tkinter.Menu.delete. Commands associated to - menu entries were not deleted. - -- Remove the TarFileCompat class from tarfile.py. - -- Issue #2491: os.fdopen is now almost an alias for the built-in open(), and - accepts the same parameters. It just checks that its first argument is an - integer. - -- Issue #3394: zipfile.writestr sets external attributes when passed a - file name rather than a ZipInfo instance, so files are extracted with - mode 0600 rather than 000 under Unix. - -- Issue #2523: Fix quadratic behaviour when read()ing a binary file without - asking for a specific length. - -Extension Modules ------------------ - -- Bug #3542: Support Unicode strings in _msi module. - -What's new in Python 3.0b2? -=========================== - -*Release date: 17-Jul-2008* - -Core and Builtins ------------------ - -- Issue #3008: the float type has a new instance method 'float.hex' - and a new class method 'float.fromhex' to convert floating-point - numbers to and from hexadecimal strings, respectively. - -- Issue #3083: Add alternate (#) formatting for bin, oct, hex output - for str.format(). This adds the prefix 0b, 0o, or 0x, respectively. - -- Issue #3280: like chr(), the "%c" format now accepts unicode code points - beyond the Basic Multilingual Plane (above 0xffff) on all configurations. On - "narrow Unicode" builds, the result is a string of 2 code units, forming a - UTF-16 surrogate pair. - -- Issue #3282: str.isprintable() should return False for undefined - Unicode characters. - -- Issue #3236: Return small longs from PyLong_FromString. - -- Exception tracebacks now support exception chaining. - -Library -------- - -- Removed the sunaudio module. Use sunau instead. - -- Issue #3554: ctypes.string_at and ctypes.wstring_at did call Python - api functions without holding the GIL, which could lead to a fatal - error when they failed. - -- Issue #799428: Fix Tkinter.Misc._nametowidget to unwrap Tcl command objects. - -- Removed "ast" function aliases from the parser module. - -- Issue #3313: Fixed a crash when a failed dlopen() call does not set - a valid dlerror() message. - -- Issue #3258: Fixed a crash when a ctypes POINTER type to an - incomplete structure was created. - -- Issue #2683: Fix inconsistency in subprocess.Popen.communicate(): the - argument now must be a bytes object in any case. - -- Issue #3145: help("modules whatever") failed when trying to load the source - code of every single module of the standard library, including invalid files - used in the test suite. - -- The gettext library now consistently uses Unicode strings for message ids - and message strings, and ``ugettext()`` and the like don't exist anymore. - -- The traceback module has been expanded to handle chained exceptions. - -C API ------ - -- Issue #3247: the function Py_FindMethod was removed. Modern types should - use the tp_methods slot instead. - -Tools/Demos ------------ - -- The Mac/Demos directory has been removed. - -- All of the Mac scripts have been removed (including BuildApplet.py). - - -What's new in Python 3.0b1? -=========================== - -*Release date: 18-Jun-2008* - -Core and Builtins ------------------ - -- Issue #3211: warnings.warn_explicit() did not guard against its 'registry' - argument being anything other than a dict or None. Also fixed a bug in error - handling when 'message' and 'category' were both set to None, triggering a - bus error. - -- Issue #3100: Corrected a crash on deallocation of a subclassed weakref which - holds the last (strong) reference to its referent. - -- Issue #2630: implement PEP 3138. repr() now returns printable - Unicode characters unescaped, to get an ASCII-only representation - of an object use ascii(). - -- Issue #1342: On windows, Python could not start when installed in a - directory with non-ascii characters. - -- Implement PEP 3121: new module initialization and finalization API. - -- Removed the already-defunct ``-t`` option. - -- Issue #2957: Corrected a ValueError "recursion limit exceeded", when - unmarshalling many code objects, which happens when importing a - large .pyc file (~1000 functions). - -- Issue #2963: fix merging oversight that disabled method cache for - all types. - -- Issue #2964: fix a missing INCREF in instancemethod_descr_get. - -- Issue #2895: Don't crash when given bytes objects as keyword names. - -- Issue #2798: When parsing arguments with PyArg_ParseTuple, the "s" - code now allows any unicode string and returns a utf-8 encoded - buffer, just like the "s#" code already does. The "z" code was - corrected as well. - -- Issue #2863: generators now have a ``gen.__name__`` attribute that - equals ``gen.gi_code.co_name``, like ``func.__name___`` that equals - ``func.func_code.co_name``. The repr() of a generator now also - contains this name. - -- Issue #2831: enumerate() now has a ``start`` argument. - -- Issue #2801: fix bug in the float.is_integer method where a - ValueError was sometimes incorrectly raised. - -- The ``--with-toolbox-glue`` option (and the associated - pymactoolbox.h) have been removed. - -- Issue #2196: hasattr() now lets exceptions which do not inherit - Exception (KeyboardInterrupt, and SystemExit) propagate instead of - ignoring them. - -- #3021 Exception reraising sematics have been significantly improved. However, - f_exc_type, f_exc_value, and f_exc_traceback cannot be accessed from Python - code anymore. - -- Three of PyNumberMethods' members, nb_coerce, nb_hex, and nb_oct, have been - removed. - -Extension Modules ------------------ - -- Renamed ``_winreg`` module to ``winreg``. - -- Support os.O_ASYNC and fcntl.FASYNC if the constants exist on the - platform. - -- Support for Windows 9x has been removed from the winsound module. - -- Issue #2870: cmathmodule.c compile error. - -Library -------- - -- The methods ``is_in_tuple()``, ``is_vararg()``, and ``is_keywordarg()`` of - symtable.Symbol have been removed. - -- Patch #3133: http.server.CGIHTTPRequestHandler did not work on windows. - -- a new ``urllib`` package was created. It consists of code from - ``urllib``, ``urllib2``, ``urlparse``, and ``robotparser``. The old - modules have all been removed. The new package has five submodules: - ``urllib.parse``, ``urllib.request``, ``urllib.response``, - ``urllib.error``, and ``urllib.robotparser``. The - ``urllib.request.urlopen()`` function uses the url opener from - ``urllib2``. (Note that the unittests have not been renamed for the - beta, but they will be renamed in the future.) - -- rfc822 has been removed in favor of the email package. - -- mimetools has been removed in favor of the email package. - -- Patch #2849: Remove use of rfc822 module from standard library. - -- Added C optimized implementation of io.StringIO. - -- The ``pickle`` module is now automatically use an optimized C - implementation of Pickler and Unpickler when available. The - ``cPickle`` module is no longer needed. - -- Removed the ``htmllib`` and ``sgmllib`` modules. - -- The deprecated ``SmartCookie`` and ``SimpleCookie`` classes have - been removed from ``http.cookies``. - -- The ``commands`` module has been removed. Its getoutput() and - getstatusoutput() functions have been moved to the ``subprocess`` module. - -- The ``http`` package was created; it contains the old ``httplib`` - as ``http.client``, ``Cookie`` as ``http.cookies``, ``cookielib`` - as ``http.cookiejar``, and the content of the three ``HTTPServer`` - modules as ``http.server``. - -- The ``xmlrpc`` package was created; it contains the old - ``xmlrpclib`` module as ``xmlrpc.client`` and the content of - the old ``SimpleXMLRPCServer`` and ``DocXMLRPCServer`` modules - as ``xmlrpc.server``. - -- The ``dbm`` package was created, containing the old modules - ``anydbm`` and ``whichdb`` in its ``__init__.py``, and having - ``dbm.gnu`` (was ``gdbm``), ``dbm.bsd`` (was ``dbhash``), - ``dbm.ndbm`` (was ``dbm``) and ``dbm.dumb`` (was ``dumbdbm``) - as submodules. - -- The ``repr`` module has been renamed to ``reprlib``. - -- The ``statvfs`` module has been removed. - -- Issue #1713041: fix pprint's handling of maximum depth. - -- Issue #2250: Exceptions raised during evaluation of names in - rlcompleter's ``Completer.complete()`` method are now caught and - ignored. - -- Patch #2659: Added ``break_on_hyphens`` option to textwrap's - ``TextWrapper`` class. - -- Issue #2487: change the semantics of math.ldexp(x, n) when n is too - large to fit in a C long. ldexp(x, n) now returns a zero (with - suitable sign) if n is large and negative; previously, it raised - OverflowError. - -- The ``ConfigParser`` module has been renamed to ``configparser``. - -- Issue #2865: webbrowser.open() works again in a KDE environment. - -- The ``multifile`` module has been removed. - -- The ``SocketServer`` module has been renamed to ``socketserver``. - -- Fixed the ``__all__`` setting on ``collections`` to include - ``UserList`` and ``UserString``. - -- The sre module has been removed. - -- The Queue module has been renamed to queue. - -- The copy_reg module has been renamed to copyreg. - -- The mhlib module has been removed. - -- The ihooks module has been removed. - -- The fpformat module has been removed. - -- The dircache module has been removed. - -- The Canvas module has been removed. - -- The Decimal module gained the magic methods __round__, __ceil__, - __floor__ and __trunc__, to give support for round, math.ceil, - math.floor and math.trunc. - -- The user module has been removed. - -- The mutex module has been removed. - -- The imputil module has been removed. - -- os.path.walk has been removed in favor of os.walk. - -- pdb gained the "until" command. - -- The test.test_support module has been renamed to test.support. - -- The threading module API was renamed to be PEP 8 compliant. The - old names are still present, but will be removed in the near future. - -Tools/Demos ------------ - -- The bgen tool has been removed. - -Build ------ - - -What's New in Python 3.0a5? -=========================== - -*Release date: 08-May-2008* - -Core and Builtins ------------------ - -- Fixed misbehaviour of PyLong_FromSsize_t on systems where - sizeof(size_t) > sizeof(long). - -- Issue #2221: Corrected a SystemError "error return without exception - set", when the code executed by exec() raises an exception, and - sys.stdout.flush() also raises an error. - -- Bug #2565: The repr() of type objects now calls them 'class', not - 'type' - whether they are builtin types or not. - -- The command line processing was converted to pass Unicode strings - through as unmodified as possible; as a consequence, the C API - related to command line arguments was changed to use wchar_t. - -- All backslashes in raw strings are interpreted literally. This - means that '\u' and '\U' escapes are not treated specially. - -Extension Modules ------------------ - -Library -------- - -- ctypes objects now support the PEP3118 buffer interface. - -- Issue #2682: ctypes callback functions now longer contain a cyclic - reference to themselves. - -- Issue #2058: Remove the buf attribute and add __slots__ to the - TarInfo class in order to reduce tarfile's memory usage. - -- Bug #2606: Avoid calling .sort() on a dict_keys object. - -- The bundled libffi copy is now in sync with the recently released - libffi3.0.5 version, apart from some small changes to - Modules/_ctypes/libffi/configure.ac. - -Build ------ - -- Issue #1496032: On alpha, use -mieee when gcc is the compiler. - -- "make install" is now an alias for "make altinstall", to prevent - accidentally overwriting a Python 2.x installation. Use "make - fullinstall" to force Python 3.0 to be installed as "python". - -- Issue #2544: On HP-UX systems, use 'gcc -shared' for linking when - gcc is used as compiler. - - -What's New in Python 3.0a4? -=========================== - -*Release date: 02-Apr-2008* - -Core and Builtins ------------------ - -- Bug #2301: Don't try decoding the source code into the original - encoding for syntax errors. - -Extension Modules ------------------ - -- The dl module was removed, use the ctypes module instead. - -- Use wchar_t functions in _locale module. - -Library -------- - -- The class distutils.commands.build_py.build_py_2to3 can be used as a - build_py replacement to automatically run 2to3 on modules that are - going to be installed. - -- A new pickle protocol (protocol 3) is added with explicit support - for bytes. This is the default protocol. It intentionally cannot - be unpickled by Python 2.x. - -- When a pickle written by Python 2.x contains an (8-bit) str - instance, this is now decoded to a (Unicode) str instance. The - encoding used to do this defaults to ASCII, but can be overridden - via two new keyword arguments to the Unpickler class. Previously - this would create bytes instances, which is usually wrong: str - instances are often used to pickle attribute names etc., and text is - more common than binary data anyway. - -- Default to ASCII as the locale.getpreferredencoding, if the POSIX - system doesn't support CODESET and LANG isn't set or doesn't allow - deduction of an encoding. - -- Issue #1202: zlib.crc32 and zlib.adler32 now return an unsigned - value. - -- Issue #719888: Updated tokenize to use a bytes API. generate_tokens - has been renamed tokenize and now works with bytes rather than - strings. A new detect_encoding function has been added for - determining source file encoding according to PEP-0263. Token - sequences returned by tokenize always start with an ENCODING token - which specifies the encoding used to decode the file. This token is - used to encode the output of untokenize back to bytes. - - -What's New in Python 3.0a3? -=========================== - -*Release date: 29-Feb-2008* - -Core and Builtins ------------------ - -- Issue #2282: io.TextIOWrapper was not overriding seekable() from - io.IOBase. - -- Issue #2115: Important speedup in setting __slot__ attributes. Also - prevent a possible crash: an Abstract Base Class would try to access - a slot on a registered virtual subclass. - -- Fixed repr() and str() of complex numbers with infinity or nan as - real or imaginary part. - -- Clear all free list during a gc.collect() of the highest generation - in order to allow pymalloc to free more arenas. Python may give back - memory to the OS earlier. - -- Issue #2045: Fix an infinite recursion triggered when printing a - subclass of collections.defaultdict, if its default_factory is set - to a bound method. - -- Fixed a minor memory leak in dictobject.c. The content of the free - list was not freed on interpreter shutdown. - -- Limit free list of method and builtin function objects to 256 - entries each. - -- Patch #1953: Added ``sys._compact_freelists()`` and the C API - functions ``PyInt_CompactFreeList`` and ``PyFloat_CompactFreeList`` - to compact the internal free lists of pre-allocted ints and floats. - -- Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls. - Python expected the return type int but the fork familie returns - pi_t. - -- Issue #1678380: Fix a bug that identifies 0j and -0j when they - appear in the same code unit. - -- Issue #2025: Added tuple.count() and tuple.index() methods to comply - with the collections.Sequence API. - -- Fixed multiple reinitialization of the Python interpreter. The small - int list in longobject.c has caused a seg fault during the third - finalization. - -- Issue #1973: bytes.fromhex('') raised SystemError. - -- Issue #1771: remove cmp parameter from sorted() and list.sort(). - -- Issue #1969: split and rsplit in bytearray are inconsistent. - -- map() no longer accepts None for the first argument. Use zip() - instead. - -- Issue #1769: Now int("- 1") is not allowed any more. - -- Object/longobject.c: long(float('nan')) raises an OverflowError - instead of returning 0. - -- Issue #1762972: __file__ points to the source file instead of the - pyc/pyo file if the py file exists. - -- Issue #1393: object_richcompare() returns NotImplemented instead of - False if the objects aren't equal, to give the other side a chance. - -- Issue #1692: Interpreter was not displaying location of SyntaxError. - -- Improve some exception messages when Windows fails to load an - extension module. Now we get for example '%1 is not a valid Win32 - application' instead of 'error code 193'. Also use Unicode strings - to deal with non-English locales. - -- Issue #1587: Added instancemethod wrapper for PyCFunctions. The - Python C API has gained a new type *PyInstanceMethod_Type* and the - functions *PyInstanceMethod_Check(o)*, *PyInstanceMethod_New(func)* - and *PyInstanceMethod_Function(im)*. - -- Constants gc.DEBUG_OBJECT and gc.DEBUG_INSTANCE have been removed - from the gc module; gc.DEBUG_COLLECTABLE or gc.DEBUG_UNCOLLECTABLE - are now enough to print the corresponding list of objects considered - by the garbage collector. - -- Issue #1573: Improper use of the keyword-only syntax makes the - parser crash. - -- Issue #1564: The set implementation should special-case PyUnicode - instead of PyString. - -- Patch #1031213: Decode source line in SyntaxErrors back to its - original source encoding. - -- inspect.getsource() includes the decorators again. - -- Bug #1713: posixpath.ismount() claims symlink to a mountpoint is a - mountpoint. - -- Fix utf-8-sig incremental decoder, which didn't recognise a BOM when - the first chunk fed to the decoder started with a BOM, but was - longer than 3 bytes. - -Extension Modules ------------------ - -- Code for itertools ifilter(), imap(), and izip() moved to bultins - and renamed to filter(), map(), and zip(). Also, renamed - izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). - -- Issue #1762972: Readded the reload() function as imp.reload(). - -- Bug #2111: mmap segfaults when trying to write a block opened with - PROT_READ. - -- Issue #2063: correct order of utime and stime in os.times() result - on Windows. - -Library -------- - -- Weakref dictionaries now inherit from MutableMapping. - -- Created new UserDict class in collections module. This one inherits - from and complies with the MutableMapping ABC. Also, moved - UserString and UserList to the collections module. The - MutableUserString class was removed. - -- Removed UserDict.DictMixin. Replaced all its uses with - collections.MutableMapping. - -- Issue #1703: getpass() should flush after writing prompt. - -- Issue #1585: IDLE uses non-existent xrange() function. - -- Issue #1578: Problems in win_getpass. - -Build ------ - -- Renamed --enable-unicode configure flag to --with-wide-unicode, - since Unicode strings can't be disabled anymore. - -C API ------ - -- Issue #1629: Renamed Py_Size, Py_Type and Py_Refcnt to Py_SIZE, - Py_TYPE and Py_REFCNT. - -- New API PyImport_ImportModuleNoBlock(), works like - PyImport_ImportModule() but won't block on the import lock - (returning an error instead). - - -What's New in Python 3.0a2? -=========================== - -*Release date: 07-Dec-2007* - -(Note: this list is incomplete.) - -Core and Builtins ------------------ - -- str8 now has the same construction signature as bytes. - -- Comparisons between str and str8 now return False/True for ==/!=. - sqlite3 returns str8 when recreating on object from it's __conform__ - value. The struct module returns str8 for all string-related - formats. This was true before this change, but becomes more - apparent thanks to string comparisons always being False. - -- Replaced `PyFile_FromFile()` with `PyFile_FromFd(fd, name. mode, - buffer, encoding, newline)`. - -- Fixed `imp.find_module()` to obey the -*- coding: -*- header. - -- Changed `__file__` and `co_filename` to unicode. The path names are decoded - with `Py_FileSystemDefaultEncoding` and a new API method - `PyUnicode_DecodeFSDefault(char*)` was added. - -- io.open() and _fileio.FileIO have grown a new argument closefd. A - false value disables the closing of the file descriptor. - -- Added a new option -b to issues warnings (-bb for errors) about - certain operations between bytes/buffer and str like str(b'') and - comparison. - -- The standard streams sys.stdin, stdout and stderr may be None - when the C runtime library returns an invalid file descriptor - for the streams (fileno(stdin) < 0). For now this happens only for - Windows GUI apps and scripts started with `pythonw.exe`. - -- Added PCbuild9 directory for VS 2008. - -- Renamed structmember.h WRITE_RESTRICTED to PY_WRITE_RESTRICTED to - work around a name clash with VS 2008 on Windows. - -- Unbound methods are gone for good. ClassObject.method returns an - ordinary function object, instance.method still returns a bound - method object. The API of bound methods is cleaned up, too. The - im_class attribute is removed and im_func + im_self are renamed to - __func__ and __self__. The factory PyMethod_New takes only func and - instance as argument. - -- intobject.h is no longer included by Python.h. The remains were - moved to longobject.h. It still exists to define several aliases - from PyInt to PyLong functions. - -- Removed sys.maxint, use sys.maxsize instead. - -Extension Modules ------------------ - -- The `hotshot` profiler has been removed; use `cProfile` instead. - -Library -------- - -- When loading an external file using testfile(), the passed-in - encoding argument was being ignored if __loader__ is defined and - forcing the source to be UTF-8. - -- The methods `os.tmpnam()`, `os.tempnam()` and `os.tmpfile()` have - been removed in favor of the tempfile module. - -- Removed the 'new' module. - -- Removed all types from the 'types' module that are easily accessible - through builtins. - - -What's New in Python 3.0a1? -=========================== - -*Release date: 31-Aug-2007* - -Core and Builtins ------------------ - -- PEP 3131: Support non-ASCII identifiers. - -- PEP 3120: Change default encoding to UTF-8. - -- PEP 3123: Use proper C inheritance for PyObject. - -- Removed the __oct__ and __hex__ special methods and added a bin() - builtin function. - -- PEP 3127: octal literals now start with "0o". Old-style octal - literals are invalid. There are binary literals with a prefix of - "0b". This also affects int(x, 0). - -- None, True, False are now keywords. - -- PEP 3119: isinstance() and issubclass() can be overridden. - -- Remove BaseException.message. - -- Remove tuple parameter unpacking (PEP 3113). - -- Remove the f_restricted attribute from frames. This naturally leads - to the removal of PyEval_GetRestricted() and PyFrame_IsRestricted(). - -- PEP 3132 was accepted. That means that you can do ``a, *b = - range(5)`` to assign 0 to a and [1, 2, 3, 4] to b. - -- range() now returns an iterator rather than a list. Floats are not - allowed. xrange() is no longer defined. - -- Patch #1660500: hide iteration variable in list comps, add set comps - and use common code to handle compilation of iterative expressions. - -- By default, != returns the opposite of ==, unless the latter returns - NotImplemented. - -- Patch #1680961: sys.exitfunc has been removed and replaced with a - private C-level API. - -- PEP 3115: new metaclasses: the metaclass is now specified as a - keyword arg in the class statement, which can now use the full - syntax of a parameter list. Also, the metaclass can implement a - __prepare__ function which will be called to create the dictionary - for the new class namespace. - -- The long-deprecated argument "pend" of PyFloat_FromString() has been - removed. - -- The dir() function has been extended to call the __dir__() method on - its argument, if it exists. If not, it will work like before. This - allows customizing the output of dir() in the presence of a - __getattr__(). - -- Removed support for __members__ and __methods__. - -- Removed indexing/slicing on BaseException. - -- input() became raw_input(): the name input() now implements the - functionality formerly known as raw_input(); the name raw_input() is - no longer defined. - -- Classes listed in an 'except' clause must inherit from - BaseException. - -- PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone; - and .keys(), .items(), .values() return dict views, which behave - like sets. - -- PEP 3105: print is now a function. Also (not in the PEP) the - 'softspace' attribute of files is now gone (since print() doesn't - use it). A side effect of this change is that you can get - incomplete output lines in interactive sessions: - - >>> print(42, end="") - 42>>> - - We may be able to fix this after the I/O library rewrite. - -- PEP 3102: keyword-only arguments. - -- Int/Long unification is complete. The 'long' built-in type and - literals with trailing 'L' or 'l' have been removed. Performance - may be sub-optimal (haven't really benchmarked). - -- 'except E, V' must now be spelled as 'except E as V' and deletes V - at the end of the except clause; V must be a simple name. - -- Added function annotations per PEP 3107. - -- Added nonlocal declaration from PEP 3104: - - >>> def f(x): - ... def inc(): - ... nonlocal x - ... x += 1 - ... return x - ... return inc - ... - >>> inc = f(0) - >>> inc() - 1 - >>> inc() - 2 - -- Moved intern() to sys.intern(). - -- exec is now a function. - -- Renamed nb_nonzero to nb_bool and __nonzero__ to __bool__. - -- Classic classes are a thing of the past. All classes are new style. - -- Exceptions *must* derive from BaseException. - -- Integer division always returns a float. The -Q option is no more. - All the following are gone: - - * PyNumber_Divide and PyNumber_InPlaceDivide - * __div__, __rdiv__, and __idiv__ - * nb_divide, nb_inplace_divide - * operator.div, operator.idiv, operator.__div__, operator.__idiv__ - (Only __truediv__ and __floordiv__ remain, not sure how to handle - them if we want to re-use __div__ and friends. If we do, it will - make it harder to write code for both 2.x and 3.x.) - -- 'as' and 'with' are keywords. - -- Absolute import is the default behavior for 'import foo' etc. - -- Removed support for syntax: backticks (ie, `x`), <>. - -- Removed these Python builtins: apply(), callable(), coerce(), - execfile(), file(), reduce(), reload(). - -- Removed these Python methods: {}.has_key. - -- Removed these opcodes: BINARY_DIVIDE, INPLACE_DIVIDE, UNARY_CONVERT. - -- Remove C API support for restricted execution. - -- zip(), map() and filter() now return iterators, behaving like their - itertools counterparts. This also affect map()'s behavior on - sequences of unequal length -- it now stops after the shortest one - is exhausted. - -- Additions: set literals, set comprehensions, ellipsis literal. - -- Added class decorators per PEP 3129. - - -Extension Modules ------------------ - -- Removed the imageop module. Obsolete long with its unit tests - becoming useless from the removal of rgbimg and imgfile. - -- Removed these attributes from the operator module: div, idiv, - __div__, __idiv__, isCallable, sequenceIncludes. - -- Removed these attributes from the sys module: exc_clear(), exc_type, - exc_value, exc_traceback. - - -Library -------- - -- Removed the compiler package. Use of the _ast module and (an - eventual) AST -> bytecode mechanism. - -- Removed these modules: audiodev, Bastion, bsddb185, exceptions, - linuxaudiodev, md5, MimeWriter, mimify, popen2, rexec, sets, sha, - stringold, strop, sunaudiodev, timing, xmllib. - -- Moved the toaiff module to Tools/Demos. - -- Removed obsolete IRIX modules: al/AL, cd/CD, cddb, cdplayer, cl/CL, - DEVICE, ERRNO, FILE, fl/FL, flp, fm, GET, gl/GL, GLWS, IN, imgfile, - IOCTL, jpeg, panel, panelparser, readcd, sgi, sv/SV, torgb, WAIT. - -- Removed obsolete functions: commands.getstatus(), os.popen*(). - -- Removed functions in the string module that are also string methods; - Remove string.{letters, lowercase, uppercase}. - -- Removed support for long obsolete platforms: plat-aix3, plat-irix5. - -- Removed xmlrpclib.SlowParser. It was based on xmllib. - -- Patch #1680961: atexit has been reimplemented in C. - -- Add new codecs for UTF-32, UTF-32-LE and UTF-32-BE. - -Build ------ - -C API ------ - -- Removed these Python slots: __coerce__, __div__, __idiv__, __rdiv__. - -- Removed these C APIs: PyNumber_Coerce(), PyNumber_CoerceEx(), - PyMember_Get, PyMember_Set. - -- Removed these C slots/fields: nb_divide, nb_inplace_divide. - -- Removed these macros: staticforward, statichere, PyArg_GetInt, - PyArg_NoArgs, _PyObject_Del. - -- Removed these typedefs: intargfunc, intintargfunc, intobjargproc, - intintobjargproc, getreadbufferproc, getwritebufferproc, - getsegcountproc, getcharbufferproc, memberlist. - -Tests ------ - -- Removed test.testall as test.regrtest replaces it. - -Documentation -------------- - -Mac ---- - -- The cfmfile module was removed. - -Platforms ---------- - -- Support for BeOS and AtheOS was removed (according to PEP 11). - -- Support for RiscOS, Irix, Tru64 was removed (allegedly). - -Tools/Demos ------------ - - -What's New in Python 2.5 release candidate 1? -============================================= - -*Release date: 17-AUG-2006* - -Core and builtins ------------------ - -- Unicode objects will no longer raise an exception when being - compared equal or unequal to a string and a UnicodeDecodeError - exception occurs, e.g. as result of a decoding failure. - - Instead, the equal (==) and unequal (!=) comparison operators will - now issue a UnicodeWarning and interpret the two objects as - unequal. The UnicodeWarning can be filtered as desired using - the warning framework, e.g. silenced completely, turned into an - exception, logged, etc. - - Note that compare operators other than equal and unequal will still - raise UnicodeDecodeError exceptions as they've always done. - -- Fix segfault when doing string formatting on subclasses of long. - -- Fix bug related to __len__ functions using values > 2**32 on 64-bit machines - with new-style classes. - -- Fix bug related to __len__ functions returning negative values with - classic classes. - -- Patch #1538606, Fix __index__() clipping. There were some problems - discovered with the API and how integers that didn't fit into Py_ssize_t - were handled. This patch attempts to provide enough alternatives - to effectively use __index__. - -- Bug #1536021: __hash__ may now return long int; the final hash - value is obtained by invoking hash on the long int. - -- Bug #1536786: buffer comparison could emit a RuntimeWarning. - -- Bug #1535165: fixed a segfault in input() and raw_input() when - sys.stdin is closed. - -- On Windows, the PyErr_Warn function is now exported from - the Python dll again. - -- Bug #1191458: tracing over for loops now produces a line event - on each iteration. Fixing this problem required changing the .pyc - magic number. This means that .pyc files generated before 2.5c1 - will be regenerated. - -- Bug #1333982: string/number constants were inappropriately stored - in the byte code and co_consts even if they were not used, ie - immediately popped off the stack. - -- Fixed a reference-counting problem in property(). - - -Library -------- - -- Fix a bug in the ``compiler`` package that caused invalid code to be - generated for generator expressions. - -- The distutils version has been changed to 2.5.0. The change to - keep it programmatically in sync with the Python version running - the code (introduced in 2.5b3) has been reverted. It will continue - to be maintained manually as static string literal. - -- If the Python part of a ctypes callback function returns None, - and this cannot be converted to the required C type, an exception is - printed with PyErr_WriteUnraisable. Before this change, the C - callback returned arbitrary values to the calling code. - -- The __repr__ method of a NULL ctypes.py_object() no longer raises - an exception. - -- uuid.UUID now has a bytes_le attribute. This returns the UUID in - little-endian byte order for Windows. In addition, uuid.py gained some - workarounds for clocks with low resolution, to stop the code yielding - duplicate UUIDs. - -- Patch #1540892: site.py Quitter() class attempts to close sys.stdin - before raising SystemExit, allowing IDLE to honor quit() and exit(). - -- Bug #1224621: make tabnanny recognize IndentationErrors raised by tokenize. - -- Patch #1536071: trace.py should now find the full module name of a - file correctly even on Windows. - -- logging's atexit hook now runs even if the rest of the module has - already been cleaned up. - -- Bug #1112549, fix DoS attack on cgi.FieldStorage. - -- Bug #1531405, format_exception no longer raises an exception if - str(exception) raised an exception. - -- Fix a bug in the ``compiler`` package that caused invalid code to be - generated for nested functions. - - -Extension Modules ------------------ - -- Patch #1511317: don't crash on invalid hostname (alias) info. - -- Patch #1535500: fix segfault in BZ2File.writelines and make sure it - raises the correct exceptions. - -- Patch # 1536908: enable building ctypes on OpenBSD/AMD64. The - '-no-stack-protector' compiler flag for OpenBSD has been removed. - -- Patch #1532975 was applied, which fixes Bug #1533481: ctypes now - uses the _as_parameter_ attribute when objects are passed to foreign - function calls. The ctypes version number was changed to 1.0.1. - -- Bug #1530559, struct.pack raises TypeError where it used to convert. - Passing float arguments to struct.pack when integers are expected - now triggers a DeprecationWarning. - - -Tests ------ - -- test_socketserver should now work on cygwin and not fail sporadically - on other platforms. - -- test_mailbox should now work on cygwin versions 2006-08-10 and later. - -- Bug #1535182: really test the xreadlines() method of bz2 objects. - -- test_threading now skips testing alternate thread stack sizes on - platforms that don't support changing thread stack size. - - -Documentation -------------- - -- Patch #1534922: unittest docs were corrected and enhanced. - - -Build ------ - -- Bug #1535502, build _hashlib on Windows, and use masm assembler - code in OpenSSL. - -- Bug #1534738, win32 debug version of _msi should be _msi_d.pyd. - -- Bug #1530448, ctypes build failure on Solaris 10 was fixed. - - -C API ------ - -- New API for Unicode rich comparisons: PyUnicode_RichCompare() - -- Bug #1069160. Internal correctness changes were made to - ``PyThreadState_SetAsyncExc()``. A test case was added, and - the documentation was changed to state that the return value - is always 1 (normal) or 0 (if the specified thread wasn't found). - - -What's New in Python 2.5 beta 3? -================================ - -*Release date: 03-AUG-2006* - -Core and builtins ------------------ - -- _PyWeakref_GetWeakrefCount() now returns a Py_ssize_t; it previously - returned a long (see PEP 353). - -- Bug #1515471: string.replace() accepts character buffers again. - -- Add PyErr_WarnEx() so C code can pass the stacklevel to warnings.warn(). - This provides the proper warning for struct.pack(). - PyErr_Warn() is now deprecated in favor of PyErr_WarnEx(). - -- Patch #1531113: Fix augmented assignment with yield expressions. - Also fix a SystemError when trying to assign to yield expressions. - -- Bug #1529871: The speed enhancement patch #921466 broke Python's compliance - with PEP 302. This was fixed by adding an ``imp.NullImporter`` type that is - used in ``sys.path_importer_cache`` to cache non-directory paths and avoid - excessive filesystem operations during imports. - -- Bug #1521947: When checking for overflow, ``PyOS_strtol()`` used some - operations on signed longs that are formally undefined by C. - Unfortunately, at least one compiler now cares about that, so complicated - the code to make that compiler happy again. - -- Bug #1524310: Properly report errors from FindNextFile in os.listdir. - -- Patch #1232023: Stop including current directory in search - path on Windows. - -- Fix some potential crashes found with failmalloc. - -- Fix warnings reported by Klocwork's static analysis tool. - -- Bug #1512814, Fix incorrect lineno's when code within a function - had more than 255 blank lines. - -- Patch #1521179: Python now accepts the standard options ``--help`` and - ``--version`` as well as ``/?`` on Windows. - -- Bug #1520864: unpacking singleton tuples in a 'for' loop (for x, in) works - again. Fixing this problem required changing the .pyc magic number. - This means that .pyc files generated before 2.5b3 will be regenerated. - -- Bug #1524317: Compiling Python ``--without-threads`` failed. - The Python core compiles again, and, in a build without threads, the - new ``sys._current_frames()`` returns a dictionary with one entry, - mapping the faux "thread id" 0 to the current frame. - -- Bug #1525447: build on MacOS X on a case-sensitive filesystem. - - -Library -------- - -- Fix #1693149. Now you can pass several modules separated by - comma to trace.py in the same --ignore-module option. - -- Correction of patch #1455898: In the mbcs decoder, set final=False - for stream decoder, but final=True for the decode function. - -- os.urandom no longer masks unrelated exceptions like SystemExit or - KeyboardInterrupt. - -- Bug #1525866: Don't copy directory stat times in - shutil.copytree on Windows - -- Bug #1002398: The documentation for os.path.sameopenfile now correctly - refers to file descriptors, not file objects. - -- The renaming of the xml package to xmlcore, and the import hackery done - to make it appear at both names, has been removed. Bug #1511497, - #1513611, and probably others. - -- Bug #1441397: The compiler module now recognizes module and function - docstrings correctly as it did in Python 2.4. - -- Bug #1529297: The rewrite of doctest for Python 2.4 unintentionally - lost that tests are sorted by name before being run. This rarely - matters for well-written tests, but can create baffling symptoms if - side effects from one test to the next affect outcomes. ``DocTestFinder`` - has been changed to sort the list of tests it returns. - -- The distutils version has been changed to 2.5.0, and is now kept - in sync with sys.version_info[:3]. - -- Bug #978833: Really close underlying socket in _socketobject.close. - -- Bug #1459963: urllib and urllib2 now normalize HTTP header names with - title(). - -- Patch #1525766: In pkgutil.walk_packages, correctly pass the onerror callback - to recursive calls and call it with the failing package name. - -- Bug #1525817: Don't truncate short lines in IDLE's tool tips. - -- Patch #1515343: Fix printing of deprecated string exceptions with a - value in the traceback module. - -- Resync optparse with Optik 1.5.3: minor tweaks for/to tests. - -- Patch #1524429: Use repr() instead of backticks in Tkinter again. - -- Bug #1520914: Change time.strftime() to accept a zero for any position in its - argument tuple. For arguments where zero is illegal, the value is forced to - the minimum value that is correct. This is to support an undocumented but - common way people used to fill in inconsequential information in the time - tuple pre-2.4. - -- Patch #1220874: Update the binhex module for Mach-O. - -- The email package has improved RFC 2231 support, specifically for - recognizing the difference between encoded (name*0*=) and non-encoded - (name*0=) parameter continuations. This may change the types of - values returned from email.message.Message.get_param() and friends. - Specifically in some cases where non-encoded continuations were used, - get_param() used to return a 3-tuple of (None, None, string) whereas now it - will just return the string (since non-encoded continuations don't have - charset and language parts). - - Also, whereas % values were decoded in all parameter continuations, they are - now only decoded in encoded parameter parts. - -- Bug #1517990: IDLE keybindings on MacOS X now work correctly - -- Bug #1517996: IDLE now longer shows the default Tk menu when a - path browser, class browser or debugger is the frontmost window on MacOS X - -- Patch #1520294: Support for getset and member descriptors in types.py, - inspect.py, and pydoc.py. Specifically, this allows for querying the type - of an object against these built-in types and more importantly, for getting - their docstrings printed in the interactive interpreter's help() function. - - -Extension Modules ------------------ - -- Patch #1519025 and bug #926423: If a KeyboardInterrupt occurs during - a socket operation on a socket with a timeout, the exception will be - caught correctly. Previously, the exception was not caught. - -- Patch #1529514: The _ctypes extension is now compiled on more - openbsd target platforms. - -- The ``__reduce__()`` method of the new ``collections.defaultdict`` had - a memory leak, affecting pickles and deep copies. - -- Bug #1471938: Fix curses module build problem on Solaris 8; patch by - Paul Eggert. - -- Patch #1448199: Release interpreter lock in _winreg.ConnectRegistry. - -- Patch #1521817: Index range checking on ctypes arrays containing - exactly one element enabled again. This allows iterating over these - arrays, without the need to check the array size before. - -- Bug #1521375: When the code in ctypes.util.find_library was - run with root privileges, it could overwrite or delete - /dev/null in certain cases; this is now fixed. - -- Bug #1467450: On Mac OS X 10.3, RTLD_GLOBAL is now used as the - default mode for loading shared libraries in ctypes. - -- Because of a misspelled preprocessor symbol, ctypes was always - compiled without thread support; this is now fixed. - -- pybsddb Bug #1527939: bsddb module DBEnv dbremove and dbrename - methods now allow their database parameter to be None as the - sleepycat API allows. - -- Bug #1526460: Fix socketmodule compile on NetBSD as it has a different - bluetooth API compared with Linux and FreeBSD. - -Tests ------ - -- Bug #1501330: Change test_ossaudiodev to be much more tolerant in terms of - how long the test file should take to play. Now accepts taking 2.93 secs - (exact time) +/- 10% instead of the hard-coded 3.1 sec. - -- Patch #1529686: The standard tests ``test_defaultdict``, ``test_iterlen``, - ``test_uuid`` and ``test_email_codecs`` didn't actually run any tests when - run via ``regrtest.py``. Now they do. - -Build ------ - -- Bug #1439538: Drop usage of test -e in configure as it is not portable. - -Mac ---- - -- PythonLauncher now works correctly when the path to the script contains - characters that are treated specially by the shell (such as quotes). - -- Bug #1527397: PythonLauncher now launches scripts with the working directory - set to the directory that contains the script instead of the user home - directory. That latter was an implementation accident and not what users - expect. - - -What's New in Python 2.5 beta 2? -================================ - -*Release date: 11-JUL-2006* - -Core and builtins ------------------ - -- Bug #1441486: The literal representation of -(sys.maxint - 1) - again evaluates to an int object, not a long. - -- Bug #1501934: The scope of global variables that are locally assigned - using augmented assignment is now correctly determined. - -- Bug #927248: Recursive method-wrapper objects can now safely - be released. - -- Bug #1417699: Reject locale-specific decimal point in float() - and atof(). - -- Bug #1511381: codec_getstreamcodec() in codec.c is corrected to - omit a default "error" argument for NULL pointer. This allows - the parser to take a codec from cjkcodecs again. - -- Bug #1519018: 'as' is now validated properly in import statements. - -- On 64 bit systems, int literals that use less than 64 bits are - now ints rather than longs. - -- Bug #1512814, Fix incorrect lineno's when code at module scope - started after line 256. - -- New function ``sys._current_frames()`` returns a dict mapping thread - id to topmost thread stack frame. This is for expert use, and is - especially useful for debugging application deadlocks. The functionality - was previously available in Fazal Majid's ``threadframe`` extension - module, but it wasn't possible to do this in a wholly threadsafe way from - an extension. - -Library -------- - -- Bug #1257728: Mention Cygwin in distutils error message about a missing - VS 2003. - -- Patch #1519566: Update turtle demo, make begin_fill idempotent. - -- Bug #1508010: msvccompiler now requires the DISTUTILS_USE_SDK - environment variable to be set in order to the SDK environment - for finding the compiler, include files, etc. - -- Bug #1515998: Properly generate logical ids for files in bdist_msi. - -- warnings.py now ignores ImportWarning by default - -- string.Template() now correctly handles tuple-values. Previously, - multi-value tuples would raise an exception and single-value tuples would - be treated as the value they contain, instead. - -- Bug #822974: Honor timeout in telnetlib.{expect,read_until} - even if some data are received. - -- Bug #1267547: Put proper recursive setup.py call into the - spec file generated by bdist_rpm. - -- Bug #1514693: Update turtle's heading when switching between - degrees and radians. - -- Reimplement turtle.circle using a polyline, to allow correct - filling of arcs. - -- Bug #1514703: Only setup canvas window in turtle when the canvas - is created. - -- Bug #1513223: .close() of a _socketobj now releases the underlying - socket again, which then gets closed as it becomes unreferenced. - -- Bug #1504333: Make sgmllib support angle brackets in quoted - attribute values. - -- Bug #853506: Fix IPv6 address parsing in unquoted attributes in - sgmllib ('[' and ']' were not accepted). - -- Fix a bug in the turtle module's end_fill function. - -- Bug #1510580: The 'warnings' module improperly required that a Warning - category be either a types.ClassType and a subclass of Warning. The proper - check is just that it is a subclass with Warning as the documentation states. - -- The compiler module now correctly compiles the new try-except-finally - statement (bug #1509132). - -- The wsgiref package is now installed properly on Unix. - -- A bug was fixed in logging.config.fileConfig() which caused a crash on - shutdown when fileConfig() was called multiple times. - -- The sqlite3 module did cut off data from the SQLite database at the first - null character before sending it to a custom converter. This has been fixed - now. - -Extension Modules ------------------ - -- #1494314: Fix a regression with high-numbered sockets in 2.4.3. This - means that select() on sockets > FD_SETSIZE (typically 1024) work again. - The patch makes sockets use poll() internally where available. - -- Assigning None to pointer type fields in ctypes structures possible - overwrote the wrong fields, this is fixed now. - -- Fixed a segfault in _ctypes when ctypes.wintypes were imported - on non-Windows platforms. - -- Bug #1518190: The ctypes.c_void_p constructor now accepts any - integer or long, without range checking. - -- Patch #1517790: It is now possible to use custom objects in the ctypes - foreign function argtypes sequence as long as they provide a from_param - method, no longer is it required that the object is a ctypes type. - -- The '_ctypes' extension module now works when Python is configured - with the --without-threads option. - -- Bug #1513646: os.access on Windows now correctly determines write - access, again. - -- Bug #1512695: cPickle.loads could crash if it was interrupted with - a KeyboardInterrupt. - -- Bug #1296433: parsing XML with a non-default encoding and - a CharacterDataHandler could crash the interpreter in pyexpat. - -- Patch #1516912: improve Modules support for OpenVMS. - -Build ------ - -- Automate Windows build process for the Win64 SSL module. - -- 'configure' now detects the zlib library the same way as distutils. - Previously, the slight difference could cause compilation errors of the - 'zlib' module on systems with more than one version of zlib. - -- The MSI compileall step was fixed to also support a TARGETDIR - with spaces in it. - -- Bug #1517388: sqlite3.dll is now installed on Windows independent - of Tcl/Tk. - -- Bug #1513032: 'make install' failed on FreeBSD 5.3 due to lib-old - trying to be installed even though it's empty. - -Tests ------ - -- Call os.waitpid() at the end of tests that spawn child processes in order - to minimize resources (zombies). - -Documentation -------------- - -- Cover ImportWarning, PendingDeprecationWarning and simplefilter() in the - documentation for the warnings module. - -- Patch #1509163: MS Toolkit Compiler no longer available. - -- Patch #1504046: Add documentation for xml.etree. - - -What's New in Python 2.5 beta 1? -================================ - -*Release date: 20-JUN-2006* - -Core and builtins ------------------ - -- Patch #1507676: Error messages returned by invalid abstract object operations - (such as iterating over an integer) have been improved and now include the - type of the offending object to help with debugging. - -- Bug #992017: A classic class that defined a __coerce__() method that returned - its arguments swapped would infinitely recurse and segfault the interpreter. - -- Fix the socket tests so they can be run concurrently. - -- Removed 5 integers from C frame objects (PyFrameObject). - f_nlocals, f_ncells, f_nfreevars, f_stack_size, f_restricted. - -- Bug #532646: object.__call__() will continue looking for the __call__ - attribute on objects until one without one is found. This leads to recursion - when you take a class and set its __call__ attribute to an instance of the - class. Originally fixed for classic classes, but this fix is for new-style. - Removes the infinite_rec_3 crasher. - -- The string and unicode methods startswith() and endswith() now accept - a tuple of prefixes/suffixes to look for. Implements RFE #1491485. - -- Buffer objects, at the C level, never used the char buffer - implementation even when the char buffer for the wrapped object was - explicitly requested (originally returned the read or write buffer). - Now a TypeError is raised if the char buffer is not present but is - requested. - -- Patch #1346214: Statements like "if 0: suite" are now again optimized - away like they were in Python 2.4. - -- Builtin exceptions are now full-blown new-style classes instead of - instances pretending to be classes, which speeds up exception handling - by about 80% in comparison to 2.5a2. - -- Patch #1494554: Update unicodedata.numeric and unicode.isnumeric to - Unicode 4.1. - -- Patch #921466: sys.path_importer_cache is now used to cache valid and - invalid file paths for the built-in import machinery which leads to - fewer open calls on startup. - -- Patch #1442927: ``long(str, base)`` is now up to 6x faster for non-power- - of-2 bases. The largest speedup is for inputs with about 1000 decimal - digits. Conversion from non-power-of-2 bases remains quadratic-time in - the number of input digits (it was and remains linear-time for bases - 2, 4, 8, 16 and 32). - -- Bug #1334662: ``int(string, base)`` could deliver a wrong answer - when ``base`` was not 2, 4, 8, 10, 16 or 32, and ``string`` represented - an integer close to ``sys.maxint``. This was repaired by patch - #1335972, which also gives a nice speedup. - -- Patch #1337051: reduced size of frame objects. - -- PyErr_NewException now accepts a tuple of base classes as its - "base" parameter. - -- Patch #876206: function call speedup by retaining allocated frame - objects. - -- Bug #1462152: file() now checks more thoroughly for invalid mode - strings and removes a possible "U" before passing the mode to the - C library function. - -- Patch #1488312, Fix memory alignment problem on SPARC in unicode - -- Bug #1487966: Fix SystemError with conditional expression in assignment - -- WindowsError now has two error code attributes: errno, which carries - the error values from errno.h, and winerror, which carries the error - values from winerror.h. Previous versions put the winerror.h values - (from GetLastError()) into the errno attribute. - -- Patch #1475845: Raise IndentationError for unexpected indent. - -- Patch #1479181: split open() and file() from being aliases for each other. - -- Patch #1497053 & bug #1275608: Exceptions occurring in ``__eq__()`` - methods were always silently ignored by dictionaries when comparing keys. - They are now passed through (except when using the C API function - ``PyDict_GetItem()``, whose semantics did not change). - -- Bug #1456209: In some obscure cases it was possible for a class with a - custom ``__eq__()`` method to confuse dict internals when class instances - were used as a dict's keys and the ``__eq__()`` method mutated the dict. - No, you don't have any code that did this ;-) - -Extension Modules ------------------ - -- Bug #1295808: expat symbols should be namespaced in pyexpat - -- Patch #1462338: Upgrade pyexpat to expat 2.0.0 - -- Change binascii.hexlify to accept a read-only buffer instead of only a char - buffer and actually follow its documentation. - -- Fixed a potentially invalid memory access of CJKCodecs' shift-jis decoder. - -- Patch #1478788 (modified version): The functional extension module has - been renamed to _functools and a functools Python wrapper module added. - This provides a home for additional function related utilities that are - not specifically about functional programming. See PEP 309. - -- Patch #1493701: performance enhancements for struct module. - -- Patch #1490224: time.altzone is now set correctly on Cygwin. - -- Patch #1435422: zlib's compress and decompress objects now have a - copy() method. - -- Patch #1454481: thread stack size is now tunable at runtime for thread - enabled builds on Windows and systems with Posix threads support. - -- On Win32, os.listdir now supports arbitrarily-long Unicode path names - (up to the system limit of 32K characters). - -- Use Win32 API to implement os.{access,chdir,chmod,mkdir,remove,rename,rmdir,utime}. - As a result, these functions now raise WindowsError instead of OSError. - -- ``time.clock()`` on Win64 should use the high-performance Windows - ``QueryPerformanceCounter()`` now (as was already the case on 32-bit - Windows platforms). - -- Calling Tk_Init twice is refused if the first call failed as that - may deadlock. - -- bsddb: added the DB_ARCH_REMOVE flag and fixed db.DBEnv.log_archive() to - accept it without potentially using an uninitialized pointer. - -- bsddb: added support for the DBEnv.log_stat() and DBEnv.lsn_reset() methods - assuming BerkeleyDB >= 4.0 and 4.4 respectively. [pybsddb project SF - patch numbers 1494885 and 1494902] - -- bsddb: added an interface for the BerkeleyDB >= 4.3 DBSequence class. - [pybsddb project SF patch number 1466734] - -- bsddb: fix DBCursor.pget() bug with keyword argument names when no data - parameter is supplied. [SF pybsddb bug #1477863] - -- bsddb: the __len__ method of a DB object has been fixed to return correct - results. It could previously incorrectly return 0 in some cases. - Fixes SF bug 1493322 (pybsddb bug 1184012). - -- bsddb: the bsddb.dbtables Modify method now raises the proper error and - aborts the db transaction safely when a modifier callback fails. - Fixes SF python patch/bug #1408584. - -- bsddb: multithreaded DB access using the simple bsddb module interface - now works reliably. It has been updated to use automatic BerkeleyDB - deadlock detection and the bsddb.dbutils.DeadlockWrap wrapper to retry - database calls that would previously deadlock. [SF python bug #775414] - -- Patch #1446489: add support for the ZIP64 extensions to zipfile. - -- Patch #1506645: add Python wrappers for the curses functions - is_term_resized, resize_term and resizeterm. - -Library -------- - -- Patch #815924: Restore ability to pass type= and icon= in tkMessageBox - functions. - -- Patch #812986: Update turtle output even if not tracing. - -- Patch #1494750: Destroy master after deleting children in - Tkinter.BaseWidget. - -- Patch #1096231: Add ``default`` argument to Tkinter.Wm.wm_iconbitmap. - -- Patch #763580: Add name and value arguments to Tkinter variable - classes. - -- Bug #1117556: SimpleHTTPServer now tries to find and use the system's - mime.types file for determining MIME types. - -- Bug #1339007: Shelf objects now don't raise an exception in their - __del__ method when initialization failed. - -- Patch #1455898: The MBCS codec now supports the incremental mode for - double-byte encodings. - -- ``difflib``'s ``SequenceMatcher.get_matching_blocks()`` was changed to - guarantee that adjacent triples in the return list always describe - non-adjacent blocks. Previously, a pair of matching blocks could end - up being described by multiple adjacent triples that formed a partition - of the matching pair. - -- Bug #1498146: fix optparse to handle Unicode strings in option help, - description, and epilog. - -- Bug #1366250: minor optparse documentation error. - -- Bug #1361643: fix textwrap.dedent() so it handles tabs appropriately; - clarify docs. - -- The wsgiref package has been added to the standard library. - -- The functions update_wrapper() and wraps() have been added to the functools - module. These make it easier to copy relevant metadata from the original - function when writing wrapper functions. - -- The optional ``isprivate`` argument to ``doctest.testmod()``, and the - ``doctest.is_private()`` function, both deprecated in 2.4, were removed. - -- Patch #1359618: Speed up charmap encoder by using a trie structure - for lookup. - -- The functions in the ``pprint`` module now sort dictionaries by key - before computing the display. Before 2.5, ``pprint`` sorted a dictionary - if and only if its display required more than one line, although that - wasn't documented. The new behavior increases predictability; e.g., - using ``pprint.pprint(a_dict)`` in a doctest is now reliable. - -- Patch #1497027: try HTTP digest auth before basic auth in urllib2 - (thanks for J. J. Lee). - -- Patch #1496206: improve urllib2 handling of passwords with respect to - default HTTP and HTTPS ports. - -- Patch #1080727: add "encoding" parameter to doctest.DocFileSuite. - -- Patch #1281707: speed up gzip.readline. - -- Patch #1180296: Two new functions were added to the locale module: - format_string() to get the effect of "format % items" but locale-aware, - and currency() to format a monetary number with currency sign. - -- Patch #1486962: Several bugs in the turtle Tk demo module were fixed - and several features added, such as speed and geometry control. - -- Patch #1488881: add support for external file objects in bz2 compressed - tarfiles. - -- Patch #721464: pdb.Pdb instances can now be given explicit stdin and - stdout arguments, making it possible to redirect input and output - for remote debugging. - -- Patch #1484695: Update the tarfile module to version 0.8. This fixes - a couple of issues, notably handling of long file names using the - GNU LONGNAME extension. - -- Patch #1478292. ``doctest.register_optionflag(name)`` shouldn't create a - new flag when ``name`` is already the name of an option flag. - -- Bug #1385040: don't allow "def foo(a=1, b): pass" in the compiler - package. - -- Patch #1472854: make the rlcompleter.Completer class usable on non- - UNIX platforms. - -- Patch #1470846: fix urllib2 ProxyBasicAuthHandler. - -- Bug #1472827: correctly escape newlines and tabs in attribute values in - the saxutils.XMLGenerator class. - - -Build ------ - -- Bug #1502728: Correctly link against librt library on HP-UX. - -- OpenBSD 3.9 is supported now. - -- Patch #1492356: Port to Windows CE. - -- Bug/Patch #1481770: Use .so extension for shared libraries on HP-UX for ia64. - -- Patch #1471883: Add --enable-universalsdk. - -C API ------ - -Tests ------ - -Tools ------ - -Documentation -------------- - - - -What's New in Python 2.5 alpha 2? -================================= - -*Release date: 27-APR-2006* - -Core and builtins ------------------ - -- Bug #1465834: 'bdist_wininst preinstall script support' was fixed - by converting these apis from macros into exported functions again: - - PyParser_SimpleParseFile PyParser_SimpleParseString PyRun_AnyFile - PyRun_AnyFileEx PyRun_AnyFileFlags PyRun_File PyRun_FileEx - PyRun_FileFlags PyRun_InteractiveLoop PyRun_InteractiveOne - PyRun_SimpleFile PyRun_SimpleFileEx PyRun_SimpleString - PyRun_String Py_CompileString - -- Under COUNT_ALLOCS, types are not necessarily immortal anymore. - -- All uses of PyStructSequence_InitType have been changed to initialize - the type objects only once, even if the interpreter is initialized - multiple times. - -- Bug #1454485, array.array('u') could crash the interpreter. This was - due to PyArgs_ParseTuple(args, 'u#', ...) trying to convert buffers (strings) - to unicode when it didn't make sense. 'u#' now requires a unicode string. - -- Py_UNICODE is unsigned. It was always documented as unsigned, but - due to a bug had a signed value in previous versions. - -- Patch #837242: ``id()`` of any Python object always gives a positive - number now, which might be a long integer. ``PyLong_FromVoidPtr`` and - ``PyLong_AsVoidPtr`` have been changed accordingly. Note that it has - never been correct to implement a ``__hash()__`` method that returns the - ``id()`` of an object: - - def __hash__(self): - return id(self) # WRONG - - because a hash result must be a (short) Python int but it was always - possible for ``id()`` to return a Python long. However, because ``id()`` - could return negative values before, on a 32-bit box an ``id()`` result - was always usable as a hash value before this patch. That's no longer - necessarily so. - -- Python on OS X 10.3 and above now uses dlopen() (via dynload_shlib.c) - to load extension modules and now provides the dl module. As a result, - sys.setdlopenflags() now works correctly on these systems. (SF patch - #1454844) - -- Patch #1463867: enhanced garbage collection to allow cleanup of cycles - involving generators that have paused outside of any ``try`` or ``with`` - blocks. (In 2.5a1, a paused generator that was part of a reference - cycle could not be garbage collected, regardless of whether it was - paused in a ``try`` or ``with`` block.) - -Extension Modules ------------------ - -- Patch #1191065: Fix preprocessor problems on systems where recvfrom - is a macro. - -- Bug #1467952: os.listdir() now correctly raises an error if readdir() - fails with an error condition. - -- Fixed bsddb.db.DBError derived exceptions so they can be unpickled. - -- Bug #1117761: bsddb.*open() no longer raises an exception when using - the cachesize parameter. - -- Bug #1149413: bsddb.*open() no longer raises an exception when using - a temporary db (file=None) with the 'n' flag to truncate on open. - -- Bug #1332852: bsddb module minimum BerkeleyDB version raised to 3.3 - as older versions cause excessive test failures. - -- Patch #1062014: AF_UNIX sockets under Linux have a special - abstract namespace that is now fully supported. - -Library -------- - -- Bug #1223937: subprocess.CalledProcessError reports the exit status - of the process using the returncode attribute, instead of - abusing errno. - -- Patch #1475231: ``doctest`` has a new ``SKIP`` option, which causes - a doctest to be skipped (the code is not run, and the expected output - or exception is ignored). - -- Fixed contextlib.nested to cope with exceptions being raised and - caught inside exit handlers. - -- Updated optparse module to Optik 1.5.1 (allow numeric constants in - hex, octal, or binary; add ``append_const`` action; keep going if - gettext cannot be imported; added ``OptionParser.destroy()`` method; - added ``epilog`` for better help generation). - -- Bug #1473760: ``tempfile.TemporaryFile()`` could hang on Windows, when - called from a thread spawned as a side effect of importing a module. - -- The pydoc module now supports documenting packages contained in - .zip or .egg files. - -- The pkgutil module now has several new utility functions, such - as ``walk_packages()`` to support working with packages that are either - in the filesystem or zip files. - -- The mailbox module can now modify and delete messages from - mailboxes, in addition to simply reading them. Thanks to Gregory - K. Johnson for writing the code, and to the 2005 Google Summer of - Code for funding his work. - -- The ``__del__`` method of class ``local`` in module ``_threading_local`` - returned before accomplishing any of its intended cleanup. - -- Patch #790710: Add breakpoint command lists in pdb. - -- Patch #1063914: Add Tkinter.Misc.clipboard_get(). - -- Patch #1191700: Adjust column alignment in bdb breakpoint lists. - -- SimpleXMLRPCServer relied on the fcntl module, which is unavailable on - Windows. Bug #1469163. - -- The warnings, linecache, inspect, traceback, site, and doctest modules - were updated to work correctly with modules imported from zipfiles or - via other PEP 302 __loader__ objects. - -- Patch #1467770: Reduce usage of subprocess._active to processes which - the application hasn't waited on. - -- Patch #1462222: Fix Tix.Grid. - -- Fix exception when doing glob.glob('anything*/') - -- The pstats.Stats class accepts an optional stream keyword argument to - direct output to an alternate file-like object. - -Build ------ - -- The Makefile now has a reindent target, which runs reindent.py on - the library. - -- Patch #1470875: Building Python with MS Free Compiler - -- Patch #1161914: Add a python-config script. - -- Patch #1324762:Remove ccpython.cc; replace --with-cxx with - --with-cxx-main. Link with C++ compiler only if --with-cxx-main was - specified. (Can be overridden by explicitly setting LINKCC.) Decouple - CXX from --with-cxx-main, see description in README. - -- Patch #1429775: Link extension modules with the shared libpython. - -- Fixed a libffi build problem on MIPS systems. - -- ``PyString_FromFormat``, ``PyErr_Format``, and ``PyString_FromFormatV`` - now accept formats "%u" for unsigned ints, "%lu" for unsigned longs, - and "%zu" for unsigned integers of type ``size_t``. - -Tests ------ - -- test_contextlib now checks contextlib.nested can cope with exceptions - being raised and caught inside exit handlers. - -- test_cmd_line now checks operation of the -m and -c command switches - -- The test_contextlib test in 2.5a1 wasn't actually run unless you ran - it separately and by hand. It also wasn't cleaning up its changes to - the current Decimal context. - -- regrtest.py now has a -M option to run tests that test the new limits of - containers, on 64-bit architectures. Running these tests is only sensible - on 64-bit machines with more than two gigabytes of memory. The argument - passed is the maximum amount of memory for the tests to use. - -Tools ------ - -- Added the Python benchmark suite pybench to the Tools/ directory; - contributed by Marc-Andre Lemburg. - -Documentation -------------- - -- Patch #1473132: Improve docs for ``tp_clear`` and ``tp_traverse``. - -- PEP 343: Added Context Types section to the library reference - and attempted to bring other PEP 343 related documentation into - line with the implementation and/or python-dev discussions. - -- Bug #1337990: clarified that ``doctest`` does not support examples - requiring both expected output and an exception. - - -What's New in Python 2.5 alpha 1? -================================= - -*Release date: 05-APR-2006* - -Core and builtins ------------------ - -- PEP 338: -m command line switch now delegates to runpy.run_module - allowing it to support modules in packages and zipfiles - -- On Windows, .DLL is not an accepted file name extension for - extension modules anymore; extensions are only found if they - end in .PYD. - -- Bug #1421664: sys.stderr.encoding is now set to the same value as - sys.stdout.encoding. - -- __import__ accepts keyword arguments. - -- Patch #1460496: round() now accepts keyword arguments. - -- Fixed bug #1459029 - unicode reprs were double-escaped. - -- Patch #1396919: The system scope threads are reenabled on FreeBSD - 5.4 and later versions. - -- Bug #1115379: Compiling a Unicode string with an encoding declaration - now gives a SyntaxError. - -- Previously, Python code had no easy way to access the contents of a - cell object. Now, a ``cell_contents`` attribute has been added - (closes patch #1170323). - -- Patch #1123430: Python's small-object allocator now returns an arena to - the system ``free()`` when all memory within an arena becomes unused - again. Prior to Python 2.5, arenas (256KB chunks of memory) were never - freed. Some applications will see a drop in virtual memory size now, - especially long-running applications that, from time to time, temporarily - use a large number of small objects. Note that when Python returns an - arena to the platform C's ``free()``, there's no guarantee that the - platform C library will in turn return that memory to the operating system. - The effect of the patch is to stop making that impossible, and in tests it - appears to be effective at least on Microsoft C and gcc-based systems. - Thanks to Evan Jones for hard work and patience. - -- Patch #1434038: property() now uses the getter's docstring if there is - no "doc" argument given. This makes it possible to legitimately use - property() as a decorator to produce a read-only property. - -- PEP 357, patch 1436368: add an __index__ method to int/long and a matching - nb_index slot to the PyNumberMethods struct. The slot is consulted instead - of requiring an int or long in slicing and a few other contexts, enabling - other objects (e.g. Numeric Python's integers) to be used as slice indices. - -- Fixed various bugs reported by Coverity's Prevent tool. - -- PEP 352, patch #1104669: Make exceptions new-style objects. Introduced the - new exception base class, BaseException, which has a new message attribute. - KeyboardInterrupt and SystemExit to directly inherit from BaseException now. - Raising a string exception now raises a DeprecationWarning. - -- Patch #1438387, PEP 328: relative and absolute imports. Imports can now be - explicitly relative, using 'from .module import name' to mean 'from the same - package as this module is in. Imports without dots still default to the - old relative-then-absolute, unless 'from __future__ import - absolute_import' is used. - -- Properly check if 'warnings' raises an exception (usually when a filter set - to "error" is triggered) when raising a warning for raising string - exceptions. - -- CO_GENERATOR_ALLOWED is no longer defined. This behavior is the default. - The name was removed from Include/code.h. - -- PEP 308: conditional expressions were added: (x if cond else y). - -- Patch 1433928: - - The copy module now "copies" function objects (as atomic objects). - - dict.__getitem__ now looks for a __missing__ hook before raising - KeyError. - -- PEP 343: with statement implemented. Needs ``from __future__ import - with_statement``. Use of 'with' as a variable will generate a warning. - Use of 'as' as a variable will also generate a warning (unless it's - part of an import statement). - The following objects have __context__ methods: - - The built-in file type. - - The thread.LockType type. - - The following types defined by the threading module: - Lock, RLock, Condition, Semaphore, BoundedSemaphore. - - The decimal.Context class. - -- Fix the encodings package codec search function to only search - inside its own package. Fixes problem reported in patch #1433198. - - Note: Codec packages should implement and register their own - codec search function. PEP 100 has the details. - -- PEP 353: Using ``Py_ssize_t`` as the index type. - -- ``PYMALLOC_DEBUG`` builds now add ``4*sizeof(size_t)`` bytes of debugging - info to each allocated block, since the ``Py_ssize_t`` changes (PEP 353) - now allow Python to make use of memory blocks exceeding 2**32 bytes for - some purposes on 64-bit boxes. A ``PYMALLOC_DEBUG`` build was limited - to 4-byte allocations before. - -- Patch #1400181, fix unicode string formatting to not use the locale. - This is how string objects work. u'%f' could use , instead of . - for the decimal point. Now both strings and unicode always use periods. - -- Bug #1244610, #1392915, fix build problem on OpenBSD 3.7 and 3.8. - configure would break checking curses.h. - -- Bug #959576: The pwd module is now built in. This allows Python to be - built on UNIX platforms without $HOME set. - -- Bug #1072182, fix some potential problems if characters are signed. - -- Bug #889500, fix line number on SyntaxWarning for global declarations. - -- Bug #1378022, UTF-8 files with a leading BOM crashed the interpreter. - -- Support for converting hex strings to floats no longer works. - This was not portable. float('0x3') now raises a ValueError. - -- Patch #1382163: Expose Subversion revision number to Python. New C API - function Py_GetBuildNumber(). New attribute sys.subversion. Build number - is now displayed in interactive prompt banner. - -- Implementation of PEP 341 - Unification of try/except and try/finally. - "except" clauses can now be written together with a "finally" clause in - one try statement instead of two nested ones. Patch #1355913. - -- Bug #1379994: Builtin unicode_escape and raw_unicode_escape codec - now encodes backslash correctly. - -- Patch #1350409: Work around signal handling bug in Visual Studio 2005. - -- Bug #1281408: Py_BuildValue now works correctly even with unsigned longs - and long longs. - -- SF Bug #1350188, "setdlopenflags" leads to crash upon "import" - It was possible for dlerror() to return a NULL pointer, so - it will now use a default error message in this case. - -- Replaced most Unicode charmap codecs with new ones using the - new Unicode translate string feature in the built-in charmap - codec; the codecs were created from the mapping tables available - at ftp.unicode.org and contain a few updates (e.g. the Mac OS - encodings now include a mapping for the Apple logo) - -- Added a few more codecs for Mac OS encodings - -- Sped up some Unicode operations. - -- A new AST parser implementation was completed. The abstract - syntax tree is available for read-only (non-compile) access - to Python code; an _ast module was added. - -- SF bug #1167751: fix incorrect code being produced for generator expressions. - The following code now raises a SyntaxError: foo(a = i for i in range(10)) - -- SF Bug #976608: fix SystemError when mtime of an imported file is -1. - -- SF Bug #887946: fix segfault when redirecting stdin from a directory. - Provide a warning when a directory is passed on the command line. - -- Fix segfault with invalid coding. - -- SF bug #772896: unknown encoding results in MemoryError. - -- All iterators now have a Boolean value of True. Formerly, some iterators - supported a __len__() method which evaluated to False when the iterator - was empty. - -- On 64-bit platforms, when __len__() returns a value that cannot be - represented as a C int, raise OverflowError. - -- test__locale is skipped on OS X < 10.4 (only partial locale support is - present). - -- SF bug #893549: parsing keyword arguments was broken with a few format - codes. - -- Changes donated by Elemental Security to make it work on AIX 5.3 - with IBM's 64-bit compiler (SF patch #1284289). This also closes SF - bug #105470: test_pwd fails on 64bit system (Opteron). - -- Changes donated by Elemental Security to make it work on HP-UX 11 on - Itanium2 with HP's 64-bit compiler (SF patch #1225212). - -- Disallow keyword arguments for type constructors that don't use them - (fixes bug #1119418). - -- Forward UnicodeDecodeError into SyntaxError for source encoding errors. - -- SF bug #900092: When tracing (e.g. for hotshot), restore 'return' events for - exceptions that cause a function to exit. - -- The implementation of set() and frozenset() was revised to use its - own internal data structure. Memory consumption is reduced by 1/3 - and there are modest speed-ups as well. The API is unchanged. - -- SF bug #1238681: freed pointer is used in longobject.c:long_pow(). - -- SF bug #1229429: PyObject_CallMethod failed to decrement some - reference counts in some error exit cases. - -- SF bug #1185883: Python's small-object memory allocator took over - a block managed by the platform C library whenever a realloc specified - a small new size. However, there's no portable way to know then how - much of the address space following the pointer is valid, so there's no - portable way to copy data from the C-managed block into Python's - small-object space without risking a memory fault. Python's small-object - realloc now leaves such blocks under the control of the platform C - realloc. - -- SF bug #1232517: An overflow error was not detected properly when - attempting to convert a large float to an int in os.utime(). - -- SF bug #1224347: hex longs now print with lowercase letters just - like their int counterparts. - -- SF bug #1163563: the original fix for bug #1010677 ("thread Module - Breaks PyGILState_Ensure()") broke badly in the case of multiple - interpreter states; back out that fix and do a better job (see - http://mail.python.org/pipermail/python-dev/2005-June/054258.html - for a longer write-up of the problem). - -- SF patch #1180995: marshal now uses a binary format by default when - serializing floats. - -- SF patch #1181301: on platforms that appear to use IEEE 754 floats, - the routines that promise to produce IEEE 754 binary representations - of floats now simply copy bytes around. - -- bug #967182: disallow opening files with 'wU' or 'aU' as specified by PEP - 278. - -- patch #1109424: int, long, float, complex, and unicode now check for the - proper magic slot for type conversions when subclassed. Previously the - magic slot was ignored during conversion. Semantics now match the way - subclasses of str always behaved. int/long/float, conversion of an instance - to the base class has been moved to the proper nb_* magic slot and out of - PyNumber_*(). - Thanks Walter Dörwald. - -- Descriptors defined in C with a PyGetSetDef structure, where the setter is - NULL, now raise an AttributeError when attempting to set or delete the - attribute. Previously a TypeError was raised, but this was inconsistent - with the equivalent pure-Python implementation. - -- It is now safe to call PyGILState_Release() before - PyEval_InitThreads() (note that if there is reason to believe there - are multiple threads around you still must call PyEval_InitThreads() - before using the Python API; this fix is for extension modules that - have no way of knowing if Python is multi-threaded yet). - -- Typing Ctrl-C whilst raw_input() was waiting in a build with threads - disabled caused a crash. - -- Bug #1165306: instancemethod_new allowed the creation of a method - with im_class == im_self == NULL, which caused a crash when called. - -- Move exception finalisation later in the shutdown process - this - fixes the crash seen in bug #1165761 - -- Added two new builtins, any() and all(). - -- Defining a class with empty parentheses is now allowed - (e.g., ``class C(): pass`` is no longer a syntax error). - Patch #1176012 added support to the 'parser' module and 'compiler' package - (thanks to logistix for that added support). - -- Patch #1115086: Support PY_LONGLONG in structmember. - -- Bug #1155938: new style classes did not check that __init__() was - returning None. - -- Patch #802188: Report characters after line continuation character - ('\') with a specific error message. - -- Bug #723201: Raise a TypeError for passing bad objects to 'L' format. - -- Bug #1124295: the __name__ attribute of file objects was - inadvertently made inaccessible in restricted mode. - -- Bug #1074011: closing sys.std{out,err} now causes a flush() and - an ferror() call. - -- min() and max() now support key= arguments with the same meaning as in - list.sort(). - -- The peephole optimizer now performs simple constant folding in expressions: - (2+3) --> (5). - -- set and frozenset objects can now be marshalled. SF #1098985. - -- Bug #1077106: Poor argument checking could cause memory corruption - in calls to os.read(). - -- The parser did not complain about future statements in illegal - positions. It once again reports a syntax error if a future - statement occurs after anything other than a doc string. - -- Change the %s format specifier for str objects so that it returns a - unicode instance if the argument is not an instance of basestring and - calling __str__ on the argument returns a unicode instance. - -- Patch #1413181: changed ``PyThreadState_Delete()`` to forget about the - current thread state when the auto-GIL-state machinery knows about - it (since the thread state is being deleted, continuing to remember it - can't help, but can hurt if another thread happens to get created with - the same thread id). - -Extension Modules ------------------ - -- Patch #1380952: fix SSL objects timing out on consecutive read()s - -- Patch #1309579: wait3 and wait4 were added to the posix module. - -- Patch #1231053: The audioop module now supports encoding/decoding of alaw. - In addition, the existing ulaw code was updated. - -- RFE #567972: Socket objects' family, type and proto properties are - now exposed via new attributes. - -- Everything under lib-old was removed. This includes the following modules: - Para, addpack, cmp, cmpcache, codehack, dircmp, dump, find, fmt, grep, - lockfile, newdir, ni, packmail, poly, rand, statcache, tb, tzparse, - util, whatsound, whrandom, zmod - -- The following modules were removed: regsub, reconvert, regex, regex_syntax. - -- re and sre were swapped, so help(re) provides full help. importing sre - is deprecated. The undocumented re.engine variable no longer exists. - -- Bug #1448490: Fixed a bug that ISO-2022 codecs could not handle - SS2 (single-shift 2) escape sequences correctly. - -- The unicodedata module was updated to the 4.1 version of the Unicode - database. The 3.2 version is still available as unicodedata.db_3_2_0 - for applications that require this specific version (such as IDNA). - -- The timing module is no longer built by default. It was deprecated - in PEP 4 in Python 2.0 or earlier. - -- Patch 1433928: Added a new type, defaultdict, to the collections module. - This uses the new __missing__ hook behavior added to dict (see above). - -- Bug #854823: socketmodule now builds on Sun platforms even when - INET_ADDRSTRLEN is not defined. - -- Patch #1393157: os.startfile() now has an optional argument to specify - a "command verb" to invoke on the file. - -- Bug #876637, prevent stack corruption when socket descriptor - is larger than FD_SETSIZE. - -- Patch #1407135, bug #1424041: harmonize mmap behavior of anonymous memory. - mmap.mmap(-1, size) now returns anonymous memory in both Unix and Windows. - mmap.mmap(0, size) should not be used on Windows for anonymous memory. - -- Patch #1422385: The nis module now supports access to domains other - than the system default domain. - -- Use Win32 API to implement os.stat/fstat. As a result, subsecond timestamps - are reported, the limit on path name lengths is removed, and stat reports - WindowsError now (instead of OSError). - -- Add bsddb.db.DBEnv.set_tx_timestamp allowing time based database recovery. - -- Bug #1413192, fix seg fault in bsddb if a transaction was deleted - before the env. - -- Patch #1103116: Basic AF_NETLINK support. - -- Bug #1402308, (possible) segfault when using mmap.mmap(-1, ...) - -- Bug #1400822, _curses over{lay,write} doesn't work when passing 6 ints. - Also fix ungetmouse() which did not accept arguments properly. - The code now conforms to the documented signature. - -- Bug #1400115, Fix segfault when calling curses.panel.userptr() - without prior setting of the userptr. - -- Fix 64-bit problems in bsddb. - -- Patch #1365916: fix some unsafe 64-bit mmap methods. - -- Bug #1290333: Added a workaround for cjkcodecs' _codecs_cn build - problem on AIX. - -- Bug #869197: os.setgroups rejects long integer arguments - -- Bug #1346533, select.poll() doesn't raise an error if timeout > sys.maxint - -- Bug #1344508, Fix UNIX mmap leaking file descriptors - -- Patch #1338314, Bug #1336623: fix tarfile so it can extract - REGTYPE directories from tarfiles written by old programs. - -- Patch #1407992, fixes broken bsddb module db associate when using - BerkeleyDB 3.3, 4.0 or 4.1. - -- Get bsddb module to build with BerkeleyDB version 4.4 - -- Get bsddb module to build with BerkeleyDB version 3.2 - -- Patch #1309009, Fix segfault in pyexpat when the XML document is in latin_1, - but Python incorrectly assumes it is in UTF-8 format - -- Fix parse errors in the readline module when compiling without threads. - -- Patch #1288833: Removed thread lock from socket.getaddrinfo on - FreeBSD 5.3 and later versions which got thread-safe getaddrinfo(3). - -- Patches #1298449 and #1298499: Add some missing checks for error - returns in cStringIO.c. - -- Patch #1297028: fix segfault if call type on MultibyteCodec, - MultibyteStreamReader, or MultibyteStreamWriter - -- Fix memory leak in posix.access(). - -- Patch #1213831: Fix typo in unicodedata._getcode. - -- Bug #1007046: os.startfile() did not accept unicode strings encoded in - the file system encoding. - -- Patch #756021: Special-case socket.inet_aton('255.255.255.255') for - platforms that don't have inet_aton(). - -- Bug #1215928: Fix bz2.BZ2File.seek() for 64-bit file offsets. - -- Bug #1191043: Fix bz2.BZ2File.(x)readlines for files containing one - line without newlines. - -- Bug #728515: mmap.resize() now resizes the file on Unix as it did - on Windows. - -- Patch #1180695: Add nanosecond stat resolution, and st_gen, - st_birthtime for FreeBSD. - -- Patch #1231069: The fcntl.ioctl function now uses the 'I' code for - the request code argument, which results in more C-like behaviour - for large or negative values. - -- Bug #1234979: For the argument of thread.Lock.acquire, the Windows - implementation treated all integer values except 1 as false. - -- Bug #1194181: bz2.BZ2File didn't handle mode 'U' correctly. - -- Patch #1212117: os.stat().st_flags is now accessible as an attribute - if available on the platform. - -- Patch #1103951: Expose O_SHLOCK and O_EXLOCK in the posix module if - available on the platform. - -- Bug #1166660: The readline module could segfault if hook functions - were set in a different thread than that which called readline. - -- collections.deque objects now support a remove() method. - -- operator.itemgetter() and operator.attrgetter() now support retrieving - multiple fields. This provides direct support for sorting on multiple - keys (primary, secondary, etc). - -- os.access now supports Unicode path names on non-Win32 systems. - -- Patches #925152, #1118602: Avoid reading after the end of the buffer - in pyexpat.GetInputContext. - -- Patches #749830, #1144555: allow UNIX mmap size to default to current - file size. - -- Added functional.partial(). See PEP309. - -- Patch #1093585: raise a ValueError for negative history items in readline. - {remove_history,replace_history} - -- The spwd module has been added, allowing access to the shadow password - database. - -- stat_float_times is now True. - -- array.array objects are now picklable. - -- the cPickle module no longer accepts the deprecated None option in the - args tuple returned by __reduce__(). - -- itertools.islice() now accepts None for the start and step arguments. - This allows islice() to work more readily with slices: - islice(s.start, s.stop, s.step) - -- datetime.datetime() now has a strptime class method which can be used to - create datetime object using a string and format. - -- Patch #1117961: Replace the MD5 implementation from RSA Data Security Inc - with the implementation from http://sourceforge.net/projects/libmd5-rfc/. - -Library -------- - -- Patch #1388073: Numerous __-prefixed attributes of unittest.TestCase have - been renamed to have only a single underscore prefix. This was done to - make subclassing easier. - -- PEP 338: new module runpy defines a run_module function to support - executing modules which provide access to source code or a code object - via the PEP 302 import mechanisms. - -- The email module's parsedate_tz function now sets the daylight savings - flag to -1 (unknown) since it can't tell from the date whether it should - be set. - -- Patch #624325: urlparse.urlparse() and urlparse.urlsplit() results - now sport attributes that provide access to the parts of the result. - -- Patch #1462498: sgmllib now handles entity and character references - in attribute values. - -- Added the sqlite3 package. This is based on pysqlite2.1.3, and provides - a DB-API interface in the standard library. You'll need sqlite 3.0.8 or - later to build this - if you have an earlier version, the C extension - module will not be built. - -- Bug #1460340: ``random.sample(dict)`` failed in various ways. Dicts - aren't officially supported here, and trying to use them will probably - raise an exception some day. But dicts have been allowed, and "mostly - worked", so support for them won't go away without warning. - -- Bug #1445068: getpass.getpass() can now be given an explicit stream - argument to specify where to write the prompt. - -- Patch #1462313, bug #1443328: the pickle modules now can handle classes - that have __private names in their __slots__. - -- Bug #1250170: mimetools now handles socket.gethostname() failures gracefully. - -- patch #1457316: "setup.py upload" now supports --identity to select the - key to be used for signing the uploaded code. - -- Queue.Queue objects now support .task_done() and .join() methods - to make it easier to monitor when daemon threads have completed - processing all enqueued tasks. Patch #1455676. - -- popen2.Popen objects now preserve the command in a .cmd attribute. - -- Added the ctypes ffi package. - -- email 4.0 package now integrated. This is largely the same as the email 3.0 - package that was included in Python 2.3, except that PEP 8 module names are - now used (e.g. mail.message instead of email.Message). The MIME classes - have been moved to a subpackage (e.g. email.mime.text instead of - email.MIMEText). The old names are still supported for now. Several - deprecated Message methods have been removed and lots of bugs have been - fixed. More details can be found in the email package documentation. - -- Patches #1436130/#1443155: codecs.lookup() now returns a CodecInfo object - (a subclass of tuple) that provides incremental decoders and encoders - (a way to use stateful codecs without the stream API). Python functions - codecs.getincrementaldecoder() and codecs.getincrementalencoder() as well - as C functions PyCodec_IncrementalEncoder() and PyCodec_IncrementalDecoder() - have been added. - -- Patch #1359365: Calling next() on a closed StringIO.String object raises - a ValueError instead of a StopIteration now (like file and cString.String do). - cStringIO.StringIO.isatty() will raise a ValueError now if close() has been - called before (like file and StringIO.StringIO do). - -- A regrtest option -w was added to re-run failed tests in verbose mode. - -- Patch #1446372: quit and exit can now be called from the interactive - interpreter to exit. - -- The function get_count() has been added to the gc module, and gc.collect() - grew an optional 'generation' argument. - -- A library msilib to generate Windows Installer files, and a distutils - command bdist_msi have been added. - -- PEP 343: new module contextlib.py defines decorator @contextmanager - and helpful context managers nested() and closing(). - -- The compiler package now supports future imports after the module docstring. - -- Bug #1413790: zipfile now sanitizes absolute archive names that are - not allowed by the specs. - -- Patch #1215184: FileInput now can be given an opening hook which can - be used to control how files are opened. - -- Patch #1212287: fileinput.input() now has a mode parameter for - specifying the file mode input files should be opened with. - -- Patch #1215184: fileinput now has a fileno() function for getting the - current file number. - -- Patch #1349274: gettext.install() now optionally installs additional - translation functions other than _() in the builtins namespace. - -- Patch #1337756: fileinput now accepts Unicode filenames. - -- Patch #1373643: The chunk module can now read chunks larger than - two gigabytes. - -- Patch #1417555: SimpleHTTPServer now returns Last-Modified headers. - -- Bug #1430298: It is now possible to send a mail with an empty - return address using smtplib. - -- Bug #1432260: The names of lambda functions are now properly displayed - in pydoc. - -- Patch #1412872: zipfile now sets the creator system to 3 (Unix) - unless the system is Win32. - -- Patch #1349118: urllib now supports user:pass@ style proxy - specifications, raises IOErrors when proxies for unsupported protocols - are defined, and uses the https proxy on https redirections. - -- Bug #902075: urllib2 now supports 'host:port' style proxy specifications. - -- Bug #1407902: Add support for sftp:// URIs to urlparse. - -- Bug #1371247: Update Windows locale identifiers in locale.py. - -- Bug #1394565: SimpleHTTPServer now doesn't choke on query parameters - any more. - -- Bug #1403410: The warnings module now doesn't get confused - when it can't find out the module name it generates a warning for. - -- Patch #1177307: Added a new codec utf_8_sig for UTF-8 with a BOM signature. - -- Patch #1157027: cookielib mishandles RFC 2109 cookies in Netscape mode - -- Patch #1117398: cookielib.LWPCookieJar and .MozillaCookieJar now raise - LoadError as documented, instead of IOError. For compatibility, - LoadError subclasses IOError. - -- Added the hashlib module. It provides secure hash functions for MD5 and - SHA1, 224, 256, 384, and 512. Note that recent developments make the - historic MD5 and SHA1 unsuitable for cryptographic-strength applications. - In - Ronald L. Rivest offered this advice for Python: - - "The consensus of researchers in this area (at least as - expressed at the NIST Hash Function Workshop 10/31/05), - is that SHA-256 is a good choice for the time being, but - that research should continue, and other alternatives may - arise from this research. The larger SHA's also seem OK." - -- Added a subset of Fredrik Lundh's ElementTree package. Available - modules are xml.etree.ElementTree, xml.etree.ElementPath, and - xml.etree.ElementInclude, from ElementTree 1.2.6. - -- Patch #1162825: Support non-ASCII characters in IDLE window titles. - -- Bug #1365984: urllib now opens "data:" URLs again. - -- Patch #1314396: prevent deadlock for threading.Thread.join() when an exception - is raised within the method itself on a previous call (e.g., passing in an - illegal argument) - -- Bug #1340337: change time.strptime() to always return ValueError when there - is an error in the format string. - -- Patch #754022: Greatly enhanced webbrowser.py (by Oleg Broytmann). - -- Bug #729103: pydoc.py: Fix docother() method to accept additional - "parent" argument. - -- Patch #1300515: xdrlib.py: Fix pack_fstring() to really use null bytes - for padding. - -- Bug #1296004: httplib.py: Limit maximal amount of data read from the - socket to avoid a MemoryError on Windows. - -- Patch #1166948: locale.py: Prefer LC_ALL, LC_CTYPE and LANG over LANGUAGE - to get the correct encoding. - -- Patch #1166938: locale.py: Parse LANGUAGE as a colon separated list of - languages. - -- Patch #1268314: Cache lines in StreamReader.readlines for performance. - -- Bug #1290505: Fix clearing the regex cache for time.strptime(). - -- Bug #1167128: Fix size of a symlink in a tarfile to be 0. - -- Patch #810023: Fix off-by-one bug in urllib.urlretrieve reporthook - functionality. - -- Bug #1163178: Make IDNA return an empty string when the input is empty. - -- Patch #848017: Make Cookie more RFC-compliant. Use CRLF as default output - separator and do not output trailing semicolon. - -- Patch #1062060: urllib.urlretrieve() now raises a new exception, named - ContentTooShortException, when the actually downloaded size does not - match the Content-Length header. - -- Bug #1121494: distutils.dir_utils.mkpath now accepts Unicode strings. - -- Bug #1178484: Return complete lines from codec stream readers - even if there is an exception in later lines, resulting in - correct line numbers for decoding errors in source code. - -- Bug #1192315: Disallow negative arguments to clear() in pdb. - -- Patch #827386: Support absolute source paths in msvccompiler.py. - -- Patch #1105730: Apply the new implementation of commonprefix in posixpath - to ntpath, macpath, os2emxpath and riscospath. - -- Fix a problem in Tkinter introduced by SF patch #869468: delete bogus - __hasattr__ and __delattr__ methods on class Tk that were breaking - Tkdnd. - -- Bug #1015140: disambiguated the term "article id" in nntplib docs and - docstrings to either "article number" or "message id". - -- Bug #1238170: threading.Thread.__init__ no longer has "kwargs={}" as a - parameter, but uses the usual "kwargs=None". - -- textwrap now processes text chunks at O(n) speed instead of O(n**2). - Patch #1209527 (Contributed by Connelly). - -- urllib2 has now an attribute 'httpresponses' mapping from HTTP status code - to W3C name (404 -> 'Not Found'). RFE #1216944. - -- Bug #1177468: Don't cache the /dev/urandom file descriptor for os.urandom, - as this can cause problems with apps closing all file descriptors. - -- Bug #839151: Fix an attempt to access sys.argv in the warnings module; - it can be missing in embedded interpreters - -- Bug #1155638: Fix a bug which affected HTTP 0.9 responses in httplib. - -- Bug #1100201: Cross-site scripting was possible on BaseHTTPServer via - error messages. - -- Bug #1108948: Cookie.py produced invalid JavaScript code. - -- The tokenize module now detects and reports indentation errors. - Bug #1224621. - -- The tokenize module has a new untokenize() function to support a full - roundtrip from lexed tokens back to Python source code. In addition, - the generate_tokens() function now accepts a callable argument that - terminates by raising StopIteration. - -- Bug #1196315: fix weakref.WeakValueDictionary constructor. - -- Bug #1213894: os.path.realpath didn't resolve symlinks that were the first - component of the path. - -- Patch #1120353: The xmlrpclib module provides better, more transparent, - support for datetime.{datetime,date,time} objects. With use_datetime set - to True, applications shouldn't have to fiddle with the DateTime wrapper - class at all. - -- distutils.commands.upload was added to support uploading distribution - files to PyPI. - -- distutils.commands.register now encodes the data as UTF-8 before posting - them to PyPI. - -- decimal operator and comparison methods now return NotImplemented - instead of raising a TypeError when interacting with other types. This - allows other classes to implement __radd__ style methods and have them - work as expected. - -- Bug #1163325: Decimal infinities failed to hash. Attempting to - hash a NaN raised an InvalidOperation instead of a TypeError. - -- Patch #918101: Add tarfile open mode r|* for auto-detection of the - stream compression; add, for symmetry reasons, r:* as a synonym of r. - -- Patch #1043890: Add extractall method to tarfile. - -- Patch #1075887: Don't require MSVC in distutils if there is nothing - to build. - -- Patch #1103407: Properly deal with tarfile iterators when untarring - symbolic links on Windows. - -- Patch #645894: Use getrusage for computing the time consumption in - profile.py if available. - -- Patch #1046831: Use get_python_version where appropriate in sysconfig.py. - -- Patch #1117454: Remove code to special-case cookies without values - in LWPCookieJar. - -- Patch #1117339: Add cookielib special name tests. - -- Patch #1112812: Make bsddb/__init__.py more friendly for modulefinder. - -- Patch #1110248: SYNC_FLUSH the zlib buffer for GZipFile.flush. - -- Patch #1107973: Allow iterating over the lines of a tarfile.ExFileObject. - -- Patch #1104111: Alter setup.py --help and --help-commands. - -- Patch #1121234: Properly cleanup _exit and tkerror commands. - -- Patch #1049151: xdrlib now unpacks booleans as True or False. - -- Fixed bug in a NameError bug in cookielib. Patch #1116583. - -- Applied a security fix to SimpleXMLRPCserver (PSF-2005-001). This - disables recursive traversal through instance attributes, which can - be exploited in various ways. - -- Bug #1222790: in SimpleXMLRPCServer, set the reuse-address and close-on-exec - flags on the HTTP listening socket. - -- Bug #792570: SimpleXMLRPCServer had problems if the request grew too large. - Fixed by reading the HTTP body in chunks instead of one big socket.read(). - -- Patches #893642, #1039083: add allow_none, encoding arguments to - constructors of SimpleXMLRPCServer and CGIXMLRPCRequestHandler. - -- Bug #1110478: Revert os.environ.update to do putenv again. - -- Bug #1103844: fix distutils.install.dump_dirs() with negated options. - -- os.{SEEK_SET, SEEK_CUR, SEEK_END} have been added for convenience. - -- Enhancements to the csv module: - - + Dialects are now validated by the underlying C code, better - reflecting its capabilities, and improving its compliance with - PEP 305. - + Dialect parameter parsing has been re-implemented to improve error - reporting. - + quotechar=None and quoting=QUOTE_NONE now work the way PEP 305 - dictates. - + the parser now removes the escapechar prefix from escaped characters. - + when quoting=QUOTE_NONNUMERIC, the writer now tests for numeric - types, rather than any object that can be represented as a numeric. - + when quoting=QUOTE_NONNUMERIC, the reader now casts unquoted fields - to floats. - + reader now allows \r characters to be quoted (previously it only allowed - \n to be quoted). - + writer doublequote handling improved. - + Dialect classes passed to the module are no longer instantiated by - the module before being parsed (the former validation scheme required - this, but the mechanism was unreliable). - + The dialect registry now contains instances of the internal - C-coded dialect type, rather than references to python objects. - + the internal c-coded dialect type is now immutable. - + register_dialect now accepts the same keyword dialect specifications - as the reader and writer, allowing the user to register dialects - without first creating a dialect class. - + a configurable limit to the size of parsed fields has been added - - previously, an unmatched quote character could result in the entire - file being read into the field buffer before an error was reported. - + A new module method csv.field_size_limit() has been added that sets - the parser field size limit (returning the former limit). The initial - limit is 128kB. - + A line_num attribute has been added to the reader object, which tracks - the number of lines read from the source iterator. This is not - the same as the number of records returned, as records can span - multiple lines. - + reader and writer objects were not being registered with the cyclic-GC. - This has been fixed. - -- _DummyThread objects in the threading module now delete self.__block that is - inherited from _Thread since it uses up a lock allocated by 'thread'. The - lock primitives tend to be limited in number and thus should not be wasted on - a _DummyThread object. Fixes bug #1089632. - -- The imghdr module now detects Exif files. - -- StringIO.truncate() now correctly adjusts the size attribute. - (Bug #951915). - -- locale.py now uses an updated locale alias table (built using - Tools/i18n/makelocalealias.py, a tool to parse the X11 locale - alias file); the encoding lookup was enhanced to use Python's - encoding alias table. - -- moved deprecated modules to Lib/lib-old: whrandom, tzparse, statcache. - -- the pickle module no longer accepts the deprecated None option in the - args tuple returned by __reduce__(). - -- optparse now optionally imports gettext. This allows its use in setup.py. - -- the pickle module no longer uses the deprecated bin parameter. - -- the shelve module no longer uses the deprecated binary parameter. - -- the pstats module no longer uses the deprecated ignore() method. - -- the filecmp module no longer uses the deprecated use_statcache argument. - -- unittest.TestCase.run() and unittest.TestSuite.run() can now be successfully - extended or overridden by subclasses. Formerly, the subclassed method would - be ignored by the rest of the module. (Bug #1078905). - -- heapq.nsmallest() and heapq.nlargest() now support key= arguments with - the same meaning as in list.sort(). - -- Bug #1076985: ``codecs.StreamReader.readline()`` now calls ``read()`` only - once when a size argument is given. This prevents a buffer overflow in the - tokenizer with very long source lines. - -- Bug #1083110: ``zlib.decompress.flush()`` would segfault if called - immediately after creating the object, without any intervening - ``.decompress()`` calls. - -- The reconvert.quote function can now emit triple-quoted strings. The - reconvert module now has some simple documentation. - -- ``UserString.MutableString`` now supports negative indices in - ``__setitem__`` and ``__delitem__`` - -- Bug #1149508: ``textwrap`` now handles hyphenated numbers (eg. "2004-03-05") - correctly. - -- Partial fixes for SF bugs #1163244 and #1175396: If a chunk read by - ``codecs.StreamReader.readline()`` has a trailing "\r", read one more - character even if the user has passed a size parameter to get a proper - line ending. Remove the special handling of a "\r\n" that has been split - between two lines. - -- Bug #1251300: On UCS-4 builds the "unicode-internal" codec will now complain - about illegal code points. The codec now supports PEP 293 style error - handlers. - -- Bug #1235646: ``codecs.StreamRecoder.next()`` now reencodes the data it reads - from the input stream, so that the output is a byte string in the correct - encoding instead of a unicode string. - -- Bug #1202493: Fixing SRE parser to handle '{}' as perl does, rather than - considering it exactly like a '*'. - -- Bug #1245379: Add "unicode-1-1-utf-7" as an alias for "utf-7" to - ``encodings.aliases``. - -- ` uu.encode()`` and ``uu.decode()`` now support unicode filenames. - -- Patch #1413711: Certain patterns of differences were making difflib - touch the recursion limit. - -- Bug #947906: An object oriented interface has been added to the calendar - module. It's possible to generate HTML calendar now and the module can be - called as a script (e.g. via ``python -mcalendar``). Localized month and - weekday names can be output (even if an exotic encoding is used) using - special classes that use unicode. - -Build ------ - -- Fix test_float, test_long, and test_struct failures on Tru64 with gcc - by using -mieee gcc option. - -- Patch #1432345: Make python compile on DragonFly. - -- Build support for Win64-AMD64 was added. - -- Patch #1428494: Prefer linking against ncursesw over ncurses library. - -- Patch #881820: look for openpty and forkpty also in libbsd. - -- The sources of zlib are now part of the Python distribution (zlib 1.2.3). - The zlib module is now built in on Windows. - -- Use -xcode=pic32 for CCSHARED on Solaris with SunPro. - -- Bug #1189330: configure did not correctly determine the necessary - value of LINKCC if python was built with GCC 4.0. - -- Upgrade Windows build to zlib 1.2.3 which eliminates a potential security - vulnerability in zlib 1.2.1 and 1.2.2. - -- EXTRA_CFLAGS has been introduced as an environment variable to hold compiler - flags that change binary compatibility. Changes were also made to - distutils.sysconfig to also use the environment variable when used during - compilation of the interpreter and of C extensions through distutils. - -- SF patch 1171735: Darwin 8's headers are anal about POSIX compliance, - and linking has changed (prebinding is now deprecated, and libcc_dynamic - no longer exists). This configure patch makes things right. - -- Bug #1158607: Build with --disable-unicode again. - -- spwdmodule.c is built only if either HAVE_GETSPNAM or HAVE_HAVE_GETSPENT is - defined. Discovered as a result of not being able to build on OS X. - -- setup.py now uses the directories specified in LDFLAGS using the -L option - and in CPPFLAGS using the -I option for adding library and include - directories, respectively, for compiling extension modules against. This has - led to the core being compiled using the values in CPPFLAGS. It also removes - the need for the special-casing of both DarwinPorts and Fink for darwin since - the proper directories can be specified in LDFLAGS (``-L/sw/lib`` for Fink, - ``-L/opt/local/lib`` for DarwinPorts) and CPPFLAGS (``-I/sw/include`` for - Fink, ``-I/opt/local/include`` for DarwinPorts). - -- Test in configure.in that checks for tzset no longer dependent on tm->tm_zone - to exist in the struct (not required by either ISO C nor the UNIX 2 spec). - Tests for sanity in tzname when HAVE_TZNAME defined were also defined. - Closes bug #1096244. Thanks Gregory Bond. - -C API ------ - -- ``PyMem_{Del, DEL}`` and ``PyMem_{Free, FREE}`` no longer map to - ``PyObject_{Free, FREE}``. They map to the system ``free()`` now. If memory - is obtained via the ``PyObject_`` family, it must be released via the - ``PyObject_`` family, and likewise for the ``PyMem_`` family. This has - always been officially true, but when Python's small-object allocator was - introduced, an attempt was made to cater to a few extension modules - discovered at the time that obtained memory via ``PyObject_New`` but - released it via ``PyMem_DEL``. It's years later, and if such code still - exists it will fail now (probably with segfaults, but calling wrong - low-level memory management functions can yield many symptoms). - -- Added a C API for set and frozenset objects. - -- Removed PyRange_New(). - -- Patch #1313939: PyUnicode_DecodeCharmap() accepts a unicode string as the - mapping argument now. This string is used as a mapping table. Byte values - greater than the length of the string and 0xFFFE are treated as undefined - mappings. - - -Tests ------ - -- In test_os, st_?time is now truncated before comparing it with ST_?TIME. - -- Patch #1276356: New resource "urlfetch" is implemented. This enables - even impatient people to run tests that require remote files. - - -Documentation -------------- - -- Bug #1402224: Add warning to dl docs about crashes. - -- Bug #1396471: Document that Windows' ftell() can return invalid - values for text files with UNIX-style line endings. - -- Bug #1274828: Document os.path.splitunc(). - -- Bug #1190204: Clarify which directories are searched by site.py. - -- Bug #1193849: Clarify os.path.expanduser() documentation. - -- Bug #1243192: re.UNICODE and re.LOCALE affect \d, \D, \s and \S. - -- Bug #755617: Document the effects of os.chown() on Windows. - -- Patch #1180012: The documentation for modulefinder is now in the library reference. - -- Patch #1213031: Document that os.chown() accepts argument values of -1. - -- Bug #1190563: Document os.waitpid() return value with WNOHANG flag. - -- Bug #1175022: Correct the example code for property(). - -- Document the IterableUserDict class in the UserDict module. - Closes bug #1166582. - -- Remove all latent references for "Macintosh" that referred to semantics for - Mac OS 9 and change to reflect the state for OS X. - Closes patch #1095802. Thanks Jack Jansen. - -Mac ---- - - -New platforms -------------- - -- FreeBSD 7 support is added. - - -Tools/Demos ------------ - -- Created Misc/Vim/vim_syntax.py to auto-generate a python.vim file in that - directory for syntax highlighting in Vim. Vim directory was added and placed - vimrc to it (was previous up a level). - -- Added two new files to Tools/scripts: pysource.py, which recursively - finds Python source files, and findnocoding.py, which finds Python - source files that need an encoding declaration. - Patch #784089, credits to Oleg Broytmann. - -- Bug #1072853: pindent.py used an uninitialized variable. - -- Patch #1177597: Correct Complex.__init__. - -- Fixed a display glitch in Pynche, which could cause the right arrow to - wiggle over by a pixel. - - -What's New in Python 2.4 final? -=============================== - -*Release date: 30-NOV-2004* - -Core and builtins ------------------ - -- Bug 875692: Improve signal handling, especially when using threads, by - forcing an early re-execution of PyEval_EvalFrame() "periodic" code when - things_to_do is not cleared by Py_MakePendingCalls(). - - -What's New in Python 2.4 (release candidate 1) -============================================== - -*Release date: 18-NOV-2004* - -Core and builtins ------------------ - -- Bug 1061968: Fixes in 2.4a3 to address thread bug 1010677 reintroduced - the years-old thread shutdown race bug 225673. Numeric history lesson - aside, all bugs in all three reports are fixed now. - - -Library -------- - -- Bug 1052242: If exceptions are raised by an atexit handler function an - attempt is made to execute the remaining handlers. The last exception - raised is re-raised. - -- ``doctest``'s new support for adding ``pdb.set_trace()`` calls to - doctests was broken in a dramatic but shallow way. Fixed. - -- Bug 1065388: ``calendar``'s ``day_name``, ``day_abbr``, ``month_name``, - and ``month_abbr`` attributes emulate sequences of locale-correct - spellings of month and day names. Because the locale can change at - any time, the correct spelling is recomputed whenever one of these is - indexed. In the worst case, the index may be a slice object, so these - recomputed every day or month name each time they were indexed. This is - much slower than necessary in the usual case, when the index is just an - integer. In that case, only the single spelling needed is recomputed - now; and, when the index is a slice object, only the spellings needed - by the slice are recomputed now. - -- Patch 1061679: Added ``__all__`` to pickletools.py. - -Build ------ - -- Bug 1034277 / Patch 1035255: Remove compilation of core against CoreServices - and CoreFoundation on OS X. Involved removing PyMac_GetAppletScriptFile() - which has no known users. Thanks Bob Ippolito. - -C API ------ - -- The PyRange_New() function is deprecated. - - -What's New in Python 2.4 beta 2? -================================ - -*Release date: 03-NOV-2004* - -License -------- - -The Python Software Foundation changed the license under which Python -is released, to remove Python version numbers. There were no other -changes to the license. So, for example, wherever the license for -Python 2.3 said "Python 2.3", the new license says "Python". The -intent is to make it possible to refer to the PSF license in a more -durable way. For example, some people say they're confused by that -the Open Source Initiative's entry for the Python Software Foundation -License:: - - http://www.opensource.org/licenses/PythonSoftFoundation.php - -says "Python 2.1.1" all over it, wondering whether it applies only -to Python 2.1.1. - -The official name of the new license is the Python Software Foundation -License Version 2. - -Core and builtins ------------------ - -- Bug #1055820 Cyclic garbage collection was not protecting against that - calling a live weakref to a piece of cyclic trash could resurrect an - insane mutation of the trash if any Python code ran during gc (via - running a dead object's __del__ method, running another callback on a - weakref to a dead object, or via any Python code run in any other thread - that managed to obtain the GIL while a __del__ or callback was running - in the thread doing gc). The most likely symptom was "impossible" - ``AttributeError`` exceptions, appearing seemingly at random, on weakly - referenced objects. The cure was to clear all weakrefs to unreachable - objects before allowing any callbacks to run. - -- Bug #1054139 _PyString_Resize() now invalidates its cached hash value. - -Extension Modules ------------------ - -- Bug #1048870: the compiler now generates distinct code objects for - functions with identical bodies. This was producing confusing - traceback messages which pointed to the function where the code - object was first defined rather than the function being executed. - -Library -------- - -- Patch #1056967 changes the semantics of Template.safe_substitute() so that - no ValueError is raised on an 'invalid' match group. Now the delimiter is - returned. - -- Bug #1052503 pdb.runcall() was not passing along keyword arguments. - -- Bug #902037: XML.sax.saxutils.prepare_input_source() now combines relative - paths with a base path before checking os.path.isfile(). - -- The whichdb module can now be run from the command line. - -- Bug #1045381: time.strptime() can now infer the date using %U or %W (week of - the year) when the day of the week and year are also specified. - -- Bug #1048816: fix bug in Ctrl-K at start of line in curses.textpad.Textbox - -- Bug #1017553: fix bug in tarfile.filemode() - -- Patch #737473: fix bug that old source code is shown in tracebacks even if - the source code is updated and reloaded. - -Build ------ - -- Patch #1044395: --enable-shared is allowed in FreeBSD also. - -What's New in Python 2.4 beta 1? -================================ - -*Release date: 15-OCT-2004* - -Core and builtins ------------------ - -- Patch #975056: Restartable signals were not correctly disabled on - BSD systems. Consistently use PyOS_setsig() instead of signal(). - -- The internal portable implementation of thread-local storage (TLS), used - by the ``PyGILState_Ensure()``/``PyGILState_Release()`` API, was not - thread-correct. This could lead to a variety of problems, up to and - including segfaults. See bug 1041645 for an example. - -- Added a command line option, -m module, which searches sys.path for the - module and then runs it. (Contributed by Nick Coghlan.) - -- The bytecode optimizer now folds tuples of constants into a single - constant. - -- SF bug #513866: Float/long comparison anomaly. Prior to 2.4b1, when - an integer was compared to a float, the integer was coerced to a float. - That could yield spurious overflow errors (if the integer was very - large), and to anomalies such as - ``long(1e200)+1 == 1e200 == long(1e200)-1``. Coercion to float is no - longer performed, and cases like ``long(1e200)-1 < 1e200``, - ``long(1e200)+1 > 1e200`` and ``(1 << 20000) > 1e200`` are computed - correctly now. - -Extension modules ------------------ - -- ``collections.deque`` objects didn't play quite right with garbage - collection, which could lead to a segfault in a release build, or - an assert failure in a debug build. Also, added overflow checks, - better detection of mutation during iteration, and shielded deque - comparisons from unusual subclass overrides of the __iter__() method. - -Library -------- - -- Patch 1046644: distutils build_ext grew two new options - --swig for - specifying the swig executable to use, and --swig-opts to specify - options to pass to swig. --swig-opts="-c++" is the new way to spell - --swig-cpp. - -- Patch 983206: distutils now obeys environment variable LDSHARED, if - it is set. - -- Added Peter Astrand's subprocess.py module. See PEP 324 for details. - -- time.strptime() now properly escapes timezones and all other locale-specific - strings for regex-specific symbols. Was breaking under Japanese Windows when - the timezone was specified as "Tokyo (standard time)". - Closes bug #1039270. - -- Updates for the email package: - - + email.Utils.formatdate() grew a 'usegmt' argument for HTTP support. - + All deprecated APIs that in email 2.x issued warnings have been removed: - _encoder argument to the MIMEText constructor, Message.add_payload(), - Utils.dump_address_pair(), Utils.decode(), Utils.encode() - + New deprecations: Generator.__call__(), Message.get_type(), - Message.get_main_type(), Message.get_subtype(), the 'strict' argument to - the Parser constructor. These will be removed in email 3.1. - + Support for Python earlier than 2.3 has been removed (see PEP 291). - + All defect classes have been renamed to end in 'Defect'. - + Some FeedParser fixes; also a MultipartInvariantViolationDefect will be - added to messages that claim to be multipart but really aren't. - + Updates to documentation. - -- re's findall() and finditer() functions now take an optional flags argument - just like the compile(), search(), and match() functions. Also, documented - the previously existing start and stop parameters for the findall() and - finditer() methods of regular expression objects. - -- rfc822 Messages now support iterating over the headers. - -- The (undocumented) tarfile.Tarfile.membernames has been removed; - applications should use the getmember function. - -- httplib now offers symbolic constants for the HTTP status codes. - -- SF bug #1028306: Trying to compare a ``datetime.date`` to a - ``datetime.datetime`` mistakenly compared only the year, month and day. - Now it acts like a mixed-type comparison: ``False`` for ``==``, - ``True`` for ``!=``, and raises ``TypeError`` for other comparison - operators. Because datetime is a subclass of date, comparing only the - base class (date) members can still be done, if that's desired, by - forcing using of the appropriate date method; e.g., - ``a_date.__eq__(a_datetime)`` is true if and only if the year, month - and day members of ``a_date`` and ``a_datetime`` are equal. - -- bdist_rpm now supports command line options --force-arch, - {pre,post}-install, {pre,post}-uninstall, and - {prep,build,install,clean,verify}-script. - -- SF patch #998993: The UTF-8 and the UTF-16 stateful decoders now support - decoding incomplete input (when the input stream is temporarily exhausted). - ``codecs.StreamReader`` now implements buffering, which enables proper - readline support for the UTF-16 decoders. ``codecs.StreamReader.read()`` - has a new argument ``chars`` which specifies the number of characters to - return. ``codecs.StreamReader.readline()`` and - ``codecs.StreamReader.readlines()`` have a new argument ``keepends``. - Trailing "\n"s will be stripped from the lines if ``keepends`` is false. - -- The documentation for doctest is greatly expanded, and now covers all - the new public features (of which there are many). - -- ``doctest.master`` was put back in, and ``doctest.testmod()`` once again - updates it. This isn't good, because every ``testmod()`` call - contributes to bloating the "hidden" state of ``doctest.master``, but - some old code apparently relies on it. For now, all we can do is - encourage people to stitch doctests together via doctest's unittest - integration features instead. - -- httplib now handles ipv6 address/port pairs. - -- SF bug #1017864: ConfigParser now correctly handles default keys, - processing them with ``ConfigParser.optionxform`` when supplied, - consistent with the handling of config file entries and runtime-set - options. - -- SF bug #997050: Document, test, & check for non-string values in - ConfigParser. Moved the new string-only restriction added in - rev. 1.65 to the SafeConfigParser class, leaving existing - ConfigParser & RawConfigParser behavior alone, and documented the - conditions under which non-string values work. - -Build ------ - -- Building on darwin now includes /opt/local/include and /opt/local/lib for - building extension modules. This is so as to include software installed as - a DarwinPorts port - -- pyport.h now defines a Py_IS_NAN macro. It works as-is when the - platform C computes true for ``x != x`` if and only if X is a NaN. - Other platforms can override the default definition with a platform- - specific spelling in that platform's pyconfig.h. You can also override - pyport.h's default Py_IS_INFINITY definition now. - -C API ------ - -- SF patch 1044089: New function ``PyEval_ThreadsInitialized()`` returns - non-zero if PyEval_InitThreads() has been called. - -- The undocumented and unused extern int ``_PyThread_Started`` was removed. - -- The C API calls ``PyInterpreterState_New()`` and ``PyThreadState_New()`` - are two of the very few advertised as being safe to call without holding - the GIL. However, this wasn't true in a debug build, as bug 1041645 - demonstrated. In a debug build, Python redirects the ``PyMem`` family - of calls to Python's small-object allocator, to get the benefit of - its extra debugging capabilities. But Python's small-object allocator - isn't threadsafe, relying on the GIL to avoid the expense of doing its - own locking. ``PyInterpreterState_New()`` and ``PyThreadState_New()`` - call the platform ``malloc()`` directly now, regardless of build type. - -- PyLong_AsUnsignedLong[Mask] now support int objects as well. - -- SF patch #998993: ``PyUnicode_DecodeUTF8Stateful`` and - ``PyUnicode_DecodeUTF16Stateful`` have been added, which implement stateful - decoding. - -Tests ------ - -- test__locale ported to unittest - -Mac ---- - -- ``plistlib`` now supports non-dict root objects. There is also a new - interface for reading and writing plist files: ``readPlist(pathOrFile)`` - and ``writePlist(rootObject, pathOrFile)`` - -Tools/Demos ------------ - -- The text file comparison scripts ``ndiff.py`` and ``diff.py`` now - read the input files in universal-newline mode. This spares them - from consuming a great deal of time to deduce the useless result that, - e.g., a file with Windows line ends and a file with Linux line ends - have no lines in common. - - -What's New in Python 2.4 alpha 3? -================================= - -*Release date: 02-SEP-2004* - -Core and builtins ------------------ - -- SF patch #1007189: ``from ... import ...`` statements now allow the name - list to be surrounded by parentheses. - -- Some speedups for long arithmetic, thanks to Trevor Perrin. Gradeschool - multiplication was sped a little by optimizing the C code. Gradeschool - squaring was sped by about a factor of 2, by exploiting that about half - the digit products are duplicates in a square. Because exponentiation - uses squaring often, this also speeds long power. For example, the time - to compute 17**1000000 dropped from about 14 seconds to 9 on my box due - to this much. The cutoff for Karatsuba multiplication was raised, - since gradeschool multiplication got quicker, and the cutoff was - aggressively small regardless. The exponentiation algorithm was switched - from right-to-left to left-to-right, which is more efficient for small - bases. In addition, if the exponent is large, the algorithm now does - 5 bits (instead of 1 bit) at a time. That cut the time to compute - 17**1000000 on my box in half again, down to about 4.5 seconds. - -- OverflowWarning is no longer generated. PEP 237 scheduled this to - occur in Python 2.3, but since OverflowWarning was disabled by default, - nobody realized it was still being generated. On the chance that user - code is still using them, the Python builtin OverflowWarning, and - corresponding C API PyExc_OverflowWarning, will exist until Python 2.5. - -- Py_InitializeEx has been added. - -- Fix the order of application of decorators. The proper order is bottom-up; - the first decorator listed is the last one called. - -- SF patch #1005778. Fix a seg fault if the list size changed while - calling list.index(). This could happen if a rich comparison function - modified the list. - -- The ``func_name`` (a.k.a. ``__name__``) attribute of user-defined - functions is now writable. - -- code_new (a.k.a new.code()) now checks its arguments sufficiently - carefully that passing them on to PyCode_New() won't trigger calls - to Py_FatalError() or PyErr_BadInternalCall(). It is still the case - that the returned code object might be entirely insane. - -- Subclasses of string can no longer be interned. The semantics of - interning were not clear here -- a subclass could be mutable, for - example -- and had bugs. Explicitly interning a subclass of string - via intern() will raise a TypeError. Internal operations that attempt - to intern a string subclass will have no effect. - -- Bug 1003935: xrange() could report bogus OverflowErrors. Documented - what xrange() intends, and repaired tests accordingly. - -Extension modules ------------------ - -- difflib now supports HTML side-by-side diff. - -- os.urandom has been added for systems that support sources of random - data. - -- Patch 1012740: truncate() on a writable cStringIO now resets the - position to the end of the stream. This is consistent with the original - StringIO module and avoids inadvertently resurrecting data that was - supposed to have been truncated away. - -- Added socket.socketpair(). - -- Added CurrentByteIndex, CurrentColumnNumber, CurrentLineNumber - members to xml.parsers.expat.XMLParser object. - -- The mpz, rotor, and xreadlines modules, all deprecated in earlier - versions of Python, have now been removed. - -Library -------- - -- Patch #934356: if a module defines __all__, believe that rather than using - heuristics for filtering out imported names. - -- Patch #941486: added os.path.lexists(), which returns True for broken - symlinks, unlike os.path.exists(). - -- the random module now uses os.urandom() for seeding if it is available. - Added a new generator based on os.urandom(). - -- difflib and diff.py can now generate HTML. - -- bdist_rpm now includes version and release in the BuildRoot, and - replaces - by ``_`` in version and release. - -- distutils build/build_scripts now has an -e option to specify the - path to the Python interpreter for installed scripts. - -- PEP 292 classes Template and SafeTemplate are added to the string module. - -- tarfile now generates GNU tar files by default. - -- HTTPResponse has now a getheaders method. - -- Patch #1006219: let inspect.getsource handle '@' decorators. Thanks Simon - Percivall. - -- logging.handlers.SMTPHandler.date_time has been removed; - the class now uses email.Utils.formatdate to generate the time stamp. - -- A new function tkFont.nametofont was added to return an existing - font. The Font class constructor now has an additional exists argument - which, if True, requests to return/configure an existing font, rather - than creating a new one. - -- Updated the decimal package's min() and max() methods to match the - latest revision of the General Decimal Arithmetic Specification. - Quiet NaNs are ignored and equal values are sorted based on sign - and exponent. - -- The decimal package's Context.copy() method now returns deep copies. - -- Deprecated sys.exitfunc in favor of the atexit module. The sys.exitfunc - attribute will be kept around for backwards compatibility and atexit - will just become the one preferred way to do it. - -- patch #675551: Add get_history_item and replace_history_item functions - to the readline module. - -- bug #989672: pdb.doc and the help messages for the help_d and help_u methods - of the pdb.Pdb class gives have been corrected. d(own) goes to a newer - frame, u(p) to an older frame, not the other way around. - -- bug #990669: os.path.realpath() will resolve symlinks before normalizing the - path, as normalizing the path may alter the meaning of the path if it - contains symlinks. - -- bug #851123: shutil.copyfile will raise an exception when trying to copy a - file onto a link to itself. Thanks Gregory Ball. - -- bug #570300: Fix inspect to resolve file locations using os.path.realpath() - so as to properly list all functions in a module when the module itself is - reached through a symlink. Thanks Johannes Gijsbers. - -- doctest refactoring continued. See the docs for details. As part of - this effort, some old and little- (never?) used features are now - deprecated: the Tester class, the module is_private() function, and the - isprivate argument to testmod(). The Tester class supplied a feeble - "by hand" way to combine multiple doctests, if you knew exactly what - you were doing. The newer doctest features for unittest integration - already did a better job of that, are stronger now than ever, and the - new DocTestRunner class is a saner foundation if you want to do it by - hand. The "private name" filtering gimmick was a mistake from the - start, and testmod() changed long ago to ignore it by default. If - you want to filter out tests, the new DocTestFinder class can be used - to return a list of all doctests, and you can filter that list by - any computable criteria before passing it to a DocTestRunner instance. - -- Bug #891637, patch #1005466: fix inspect.getargs() crash on def foo((bar)). - -Tools/Demos ------------ - -- IDLE's shortcut keys for windows are now case insensitive so that - Control-V works the same as Control-v. - -- pygettext.py: Generate POT-Creation-Date header in ISO format. - -Build ------ - -- Backward incompatibility: longintrepr.h now triggers a compile-time - error if SHIFT (the number of bits in a Python long "digit") isn't - divisible by 5. This new requirement allows simple code for the new - 5-bits-at-a-time long_pow() implementation. If necessary, the - restriction could be removed (by complicating long_pow(), or by - falling back to the 1-bit-at-a-time algorithm), but there are no - plans to do so. - -- bug #991962: When building with --disable-toolbox-glue on Darwin no - attempt to build Mac-specific modules occurs. - -- The --with-tsc flag to configure to enable VM profiling with the - processor's timestamp counter now works on PPC platforms. - -- patch #1006629: Define _XOPEN_SOURCE to 500 on Solaris 8/9 to match - GCC's definition and avoid redefinition warnings. - -- Detect pthreads support (provided by gnu pth pthread emulation) on - GNU/k*BSD systems. - -- bug #1005737, #1007249: Fixed several build problems and warnings - found on old/legacy C compilers of HP-UX, IRIX and Tru64. - -C API ------ - -.. - -Documentation -------------- - -- patch #1005936, bug #1009373: fix index entries which contain - an underscore when viewed with Acrobat. - -- bug #990669: os.path.normpath may alter the meaning of a path if - it contains symbolic links. This has been documented in a comment - since 1992, but is now in the library reference as well. - -New platforms -------------- - -- FreeBSD 6 is now supported. - -Tests ------ - -.. - -Windows -------- - -- Boosted the stack reservation for python.exe and pythonw.exe from - the default 1MB to 2MB. Stack frames under VC 7.1 for 2.4 are enough - bigger than under VC 6.0 for 2.3.4 that deeply recursive progams - within the default sys.getrecursionlimit() default value of 1000 were - able to suffer undetected C stack overflows. The standard test program - test_compiler was one such program. If a Python process on Windows - "just vanishes" without a trace, and without an error message of any - kind, but with an exit code of 128, undetected stack overflow may be - the problem. - -Mac ---- - -.. - - -What's New in Python 2.4 alpha 2? -================================= - -*Release date: 05-AUG-2004* - -Core and builtins ------------------ - -- Patch #980695: Implements efficient string concatenation for statements - of the form s=s+t and s+=t. This will vary across implementations. - Accordingly, the str.join() method is strongly preferred for performance - sensitive code. - -- PEP-0318, Function Decorators have been added to the language. These are - implemented using the Java-style @decorator syntax, like so:: - - @staticmethod - def foo(bar): - - (The PEP needs to be updated to reflect the current state) - -- When importing a module M raises an exception, Python no longer leaves M - in sys.modules. Before 2.4a2 it did, and a subsequent import of M would - succeed, picking up a module object from sys.modules reflecting as much - of the initialization of M as completed before the exception was raised. - Subsequent imports got no indication that M was in a partially- - initialized state, and the importers could get into arbitrarily bad - trouble as a result (the M they got was in an unintended state, - arbitrarily far removed from M's author's intent). Now subsequent - imports of M will continue raising exceptions (but if, for example, the - source code for M is edited between import attempts, then perhaps later - attempts will succeed, or raise a different exception). - - This can break existing code, but in such cases the code was probably - working before by accident. In the Python source, the only case of - breakage discovered was in a test accidentally relying on a damaged - module remaining in sys.modules. Cases are also known where tests - deliberately provoking import errors remove damaged modules from - sys.modules themselves, and such tests will break now if they do an - unconditional del sys.modules[M]. - -- u'%s' % obj will now try obj.__unicode__() first and fallback to - obj.__str__() if no __unicode__ method can be found. - -- Patch #550732: Add PyArg_VaParseTupleAndKeywords(). Analogous to - PyArg_VaParse(). Both are now documented. Thanks Greg Chapman. - -- Allow string and unicode return types from .encode()/.decode() - methods on string and unicode objects. Added unicode.decode() - which was missing for no apparent reason. - -- An attempt to fix the mess that is Python's behaviour with - signal handlers and threads, complicated by readline's behaviour. - It's quite possible that there are still bugs here. - -- Added C macros Py_CLEAR and Py_VISIT to ease the implementation of - types that support garbage collection. - -- Compiler now treats None as a constant. - -- The type of values returned by __int__, __float__, __long__, - __oct__, and __hex__ are now checked. Returning an invalid type - will cause a TypeError to be raised. This matches the behavior of - Jython. - -- Implemented bind_textdomain_codeset() in locale module. - -- Added a workaround for proper string operations in BSDs. str.split - and str.is* methods can now work correctly with UTF-8 locales. - -- Bug #989185: unicode.iswide() and unicode.width() is dropped and - the East Asian Width support is moved to unicodedata extension - module. - -- Patch #941229: The source code encoding in interactive mode - now refers sys.stdin.encoding not just ISO-8859-1 anymore. This - allows for non-latin-1 users to write unicode strings directly. - -Extension modules ------------------ - -- cpickle now supports the same keyword arguments as pickle. - -Library -------- - -- Added new codecs and aliases for ISO_8859-11, ISO_8859-16 and - TIS-620 - -- Thanks to Edward Loper, doctest has been massively refactored, and - many new features were added. Full docs will appear later. For now - the doctest module comments and new test cases give good coverage. - The refactoring provides many hook points for customizing behavior - (such as how to report errors, and how to compare expected to actual - output). New features include a marker for expected - output containing blank lines, options to produce unified or context - diffs when actual output doesn't match expectations, an option to - normalize whitespace before comparing, and an option to use an - ellipsis to signify "don't care" regions of output. - -- Tkinter now supports the wish -sync and -use options. - -- The following methods in time support passing of None: ctime(), gmtime(), - and localtime(). If None is provided, the current time is used (the - same as when the argument is omitted). - [SF bug 658254, patch 663482] - -- nntplib does now allow ignoring a .netrc file. - -- urllib2 now recognizes Basic authentication even if other authentication - schemes are offered. - -- Bug #1001053. wave.open() now accepts unicode filenames. - -- gzip.GzipFile has a new fileno() method, to retrieve the handle of the - underlying file object (provided it has a fileno() method). This is - needed if you want to use os.fsync() on a GzipFile. - -- imaplib has two new methods: deleteacl and myrights. - -- nntplib has two new methods: description and descriptions. They - use a more RFC-compliant way of getting a newsgroup description. - -- Bug #993394. Fix a possible red herring of KeyError in 'threading' being - raised during interpreter shutdown from a registered function with atexit - when dummy_threading is being used. - -- Bug #857297/Patch #916874. Fix an error when extracting a hard link - from a tarfile. - -- Patch #846659. Fix an error in tarfile.py when using - GNU longname/longlink creation. - -- The obsolete FCNTL.py has been deleted. The built-in fcntl module - has been available (on platforms that support fcntl) since Python - 1.5a3, and all FCNTL.py did is export fcntl's names, after generating - a deprecation warning telling you to use fcntl directly. - -- Several new unicode codecs are added: big5hkscs, euc_jis_2004, - iso2022_jp_2004, shift_jis_2004. - -- Bug #788520. Queue.{get, get_nowait, put, put_nowait} have new - implementations, exploiting Conditions (which didn't exist at the time - Queue was introduced). A minor semantic change is that the Full and - Empty exceptions raised by non-blocking calls now occur only if the - queue truly was full or empty at the instant the queue was checked (of - course the Queue may no longer be full or empty by the time a calling - thread sees those exceptions, though). Before, the exceptions could - also be raised if it was "merely inconvenient" for the implementation - to determine the true state of the Queue (because the Queue was locked - by some other method in progress). - -- Bugs #979794 and #980117: difflib.get_grouped_opcodes() now handles the - case of comparing two empty lists. This affected both context_diff() and - unified_diff(), - -- Bug #980938: smtplib now prints debug output to sys.stderr. - -- Bug #930024: posixpath.realpath() now handles infinite loops in symlinks by - returning the last point in the path that was not part of any loop. Thanks - AM Kuchling. - -- Bug #980327: ntpath not handles compressing erroneous slashes between the - drive letter and the rest of the path. Also clearly handles UNC addresses now - as well. Thanks Paul Moore. - -- bug #679953: zipfile.py should now work for files over 2 GB. The packed data - for file sizes (compressed and uncompressed) was being stored as signed - instead of unsigned. - -- decimal.py now only uses signals in the IBM spec. The other conditions are - no longer part of the public API. - -- codecs module now has two new generic APIs: encode() and decode() - which don't restrict the return types (unlike the unicode and - string methods of the same name). - -- Non-blocking SSL sockets work again; they were broken in Python 2.3. - SF patch 945642. - -- doctest unittest integration improvements: - - o Improved the unitest test output for doctest-based unit tests - - o Can now pass setUp and tearDown functions when creating - DocTestSuites. - -- The threading module has a new class, local, for creating objects - that provide thread-local data. - -- Bug #990307: when keep_empty_values is True, cgi.parse_qsl() - no longer returns spurious empty fields. - -- Implemented bind_textdomain_codeset() in gettext module. - -- Introduced in gettext module the l*gettext() family of functions, - which return translation strings encoded in the preferred encoding, - as informed by locale module's getpreferredencoding(). - -- optparse module (and tests) upgraded to Optik 1.5a1. Changes: - - - Add expansion of default values in help text: the string - "%default" in an option's help string is expanded to str() of - that option's default value, or "none" if no default value. - - - Bug #955889: option default values that happen to be strings are - now processed in the same way as values from the command line; this - allows generation of nicer help when using custom types. Can - be disabled with parser.set_process_default_values(False). - - - Bug #960515: don't crash when generating help for callback - options that specify 'type', but not 'dest' or 'metavar'. - - - Feature #815264: change the default help format for short options - that take an argument from e.g. "-oARG" to "-o ARG"; add - set_short_opt_delimiter() and set_long_opt_delimiter() methods to - HelpFormatter to allow (slight) customization of the formatting. - - - Patch #736940: internationalize Optik: all built-in user- - targeted literal strings are passed through gettext.gettext(). (If - you want translations (.po files), they're not included with Python - -- you'll find them in the Optik source distribution from - http://optik.sourceforge.net/ .) - - - Bug #878453: respect $COLUMNS environment variable for - wrapping help output. - - - Feature #988122: expand "%prog" in the 'description' passed - to OptionParser, just like in the 'usage' and 'version' strings. - (This is *not* done in the 'description' passed to OptionGroup.) - -C API ------ - -- PyImport_ExecCodeModule() and PyImport_ExecCodeModuleEx(): if an - error occurs while loading the module, these now delete the module's - entry from sys.modules. All ways of loading modules eventually call - one of these, so this is an error-case change in semantics for all - ways of loading modules. In rare cases, a module loader may wish - to keep a module object in sys.modules despite that the module's - code cannot be executed. In such cases, the module loader must - arrange to reinsert the name and module object in sys.modules. - PyImport_ReloadModule() has been changed to reinsert the original - module object into sys.modules if the module reload fails, so that - its visible semantics have not changed. - -- A large pile of datetime field-extraction macros is now documented, - thanks to Anthony Tuininga (patch #986010). - -Documentation -------------- - -- Improved the tutorial on creating types in C. - - - point out the importance of reassigning data members before - assigning their values - - - correct my misconception about return values from visitprocs. Sigh. - - - mention the labor saving Py_VISIT and Py_CLEAR macros. - -- Major rewrite of the math module docs, to address common confusions. - -Tests ------ - -- The test data files for the decimal test suite are now installed on - platforms that use the Makefile. - -- SF patch 995225: The test file testtar.tar accidentally contained - CVS keywords (like $Id$), which could cause spurious failures in - test_tarfile.py depending on how the test file was checked out. - - -What's New in Python 2.4 alpha 1? -================================= - -*Release date: 08-JUL-2004* - -Core and builtins ------------------ - -- weakref.ref is now the type object also known as - weakref.ReferenceType; it can be subclassed like any other new-style - class. There's less per-entry overhead in WeakValueDictionary - objects now (one object instead of three). - -- Bug #951851: Python crashed when reading import table of certain - Windows DLLs. - -- Bug #215126. The locals argument to eval(), execfile(), and exec now - accept any mapping type. - -- marshal now shares interned strings. This change introduces - a new .pyc magic. - -- Bug #966623. classes created with type() in an exec(, {}) don't - have a __module__, but code in typeobject assumed it would always - be there. - -- Python no longer relies on the LC_NUMERIC locale setting to be - the "C" locale; as a result, it no longer tries to prevent changing - the LC_NUMERIC category. - -- Bug #952807: Unpickling pickled instances of subclasses of - datetime.date, datetime.datetime and datetime.time could yield insane - objects. Thanks to Jiwon Seo for a fix. - -- Bug #845802: Python crashes when __init__.py is a directory. - -- Unicode objects received two new methods: iswide() and width(). - These query East Asian width information, as specified in Unicode - TR11. - -- Improved the tuple hashing algorithm to give fewer collisions in - common cases. Fixes bug #942952. - -- Implemented generator expressions (PEP 289). Coded by Jiwon Seo. - -- Enabled the profiling of C extension functions (and builtins) - check - new documentation and modified profile and bdb modules for more details - -- Set file.name to the object passed to open (instead of a new string) - -- Moved tracebackobject into traceback.h and renamed to PyTracebackObject - -- Optimized the byte coding for multiple assignments like "a,b=b,a" and - "a,b,c=1,2,3". Improves their speed by 25% to 30%. - -- Limit the nested depth of a tuple for the second argument to isinstance() - and issubclass() to the recursion limit of the interpreter. - Fixes bug #858016 . - -- Optimized dict iterators, creating separate types for each - and having them reveal their length. Also optimized the - methods: keys(), values(), and items(). - -- Implemented a newcode opcode, LIST_APPEND, that simplifies - the generated bytecode for list comprehensions and further - improves their performance (about 35%). - -- Implemented rich comparisons for floats, which seems to make - comparisons involving NaNs somewhat less surprising when the - underlying C compiler actually implements C99 semantics. - -- Optimized list.extend() to save memory and no longer create - intermediate sequences. Also, extend() now pre-allocates the - needed memory whenever the length of the iterable is known in - advance -- this halves the time to extend the list. - -- Optimized list resize operations to make fewer calls to the system - realloc(). Significantly speeds up list appends, list pops, - list comprehensions, and the list constructor (when the input iterable - length is not known). - -- Changed the internal list over-allocation scheme. For larger lists, - overallocation ranged between 3% and 25%. Now, it is a constant 12%. - For smaller lists (n<8), overallocation was upto eight elements. Now, - the overallocation is no more than three elements -- this improves space - utilization for applications that have large numbers of small lists. - -- Most list bodies now get re-used rather than freed. Speeds up list - instantiation and deletion by saving calls to malloc() and free(). - -- The dict.update() method now accepts all the same argument forms - as the dict() constructor. This now includes item lists and/or - keyword arguments. - -- Support for arbitrary objects supporting the read-only buffer - interface as the co_code field of code objects (something that was - only possible to create from C code) has been removed. - -- Made omitted callback and None equivalent for weakref.ref() and - weakref.proxy(); the None case wasn't handled correctly in all - cases. - -- Fixed problem where PyWeakref_NewRef() and PyWeakref_NewProxy() - assumed that initial existing entries in an object's weakref list - would not be removed while allocating a new weakref object. Since - GC could be invoked at that time, however, that assumption was - invalid. In a truly obscure case of GC being triggered during - creation for a new weakref object for a referent which already - has a weakref without a callback which is only referenced from - cyclic trash, a memory error can occur. This consistently created a - segfault in a debug build, but provided less predictable behavior in - a release build. - -- input() built-in function now respects compiler flags such as - __future__ statements. SF patch 876178. - -- Removed PendingDeprecationWarning from apply(). apply() remains - deprecated, but the nuisance warning will not be issued. - -- At Python shutdown time (Py_Finalize()), 2.3 called cyclic garbage - collection twice, both before and after tearing down modules. The - call after tearing down modules has been disabled, because too much - of Python has been torn down then for __del__ methods and weakref - callbacks to execute sanely. The most common symptom was a sequence - of uninformative messages on stderr when Python shut down, produced - by threads trying to raise exceptions, but unable to report the nature - of their problems because too much of the sys module had already been - destroyed. - -- Removed FutureWarnings related to hex/oct literals and conversions - and left shifts. (Thanks to Kalle Svensson for SF patch 849227.) - This addresses most of the remaining semantic changes promised by - PEP 237, except for repr() of a long, which still shows the trailing - 'L'. The PEP appears to promise warnings for operations that - changed semantics compared to Python 2.3, but this is not - implemented; we've suffered through enough warnings related to - hex/oct literals and I think it's best to be silent now. - -- For str and unicode objects, the ljust(), center(), and rjust() - methods now accept an optional argument specifying a fill - character other than a space. - -- When method objects have an attribute that can be satisfied either - by the function object or by the method object, the function - object's attribute usually wins. Christian Tismer pointed out that - this is really a mistake, because this only happens for special - methods (like __reduce__) where the method object's version is - really more appropriate than the function's attribute. So from now - on, all method attributes will have precedence over function - attributes with the same name. - -- Critical bugfix, for SF bug 839548: if a weakref with a callback, - its callback, and its weakly referenced object, all became part of - cyclic garbage during a single run of garbage collection, the order - in which they were torn down was unpredictable. It was possible for - the callback to see partially-torn-down objects, leading to immediate - segfaults, or, if the callback resurrected garbage objects, to - resurrect insane objects that caused segfaults (or other surprises) - later. In one sense this wasn't surprising, because Python's cyclic gc - had no knowledge of Python's weakref objects. It does now. When - weakrefs with callbacks become part of cyclic garbage now, those - weakrefs are cleared first. The callbacks don't trigger then, - preventing the problems. If you need callbacks to trigger, then just - as when cyclic gc is not involved, you need to write your code so - that weakref objects outlive the objects they weakly reference. - -- Critical bugfix, for SF bug 840829: if cyclic garbage collection - happened to occur during a weakref callback for a new-style class - instance, subtle memory corruption was the result (in a release build; - in a debug build, a segfault occurred reliably very soon after). - This has been repaired. - -- Compiler flags set in PYTHONSTARTUP are now active in __main__. - -- Added two built-in types, set() and frozenset(). - -- Added a reversed() built-in function that returns a reverse iterator - over a sequence. - -- Added a sorted() built-in function that returns a new sorted list - from any iterable. - -- CObjects are now mutable (on the C level) through PyCObject_SetVoidPtr. - -- list.sort() now supports three keyword arguments: cmp, key, and reverse. - The key argument can be a function of one argument that extracts a - comparison key from the original record: mylist.sort(key=str.lower). - The reverse argument is a boolean value and if True will change the - sort order as if the comparison arguments were reversed. In addition, - the documentation has been amended to provide a guarantee that all sorts - starting with Py2.3 are guaranteed to be stable (the relative order of - records with equal keys is unchanged). - -- Added test whether wchar_t is signed or not. A signed wchar_t is not - usable as internal unicode type base for Py_UNICODE since the - unicode implementation assumes an unsigned type. - -- Fixed a bug in the cache of length-one Unicode strings that could - lead to a seg fault. The specific problem occurred when an earlier, - non-fatal error left an uninitialized Unicode object in the - freelist. - -- The % formatting operator now supports '%F' which is equivalent to - '%f'. This has always been documented but never implemented. - -- complex(obj) could leak a little memory if obj wasn't a string or - number. - -- zip() with no arguments now returns an empty list instead of raising - a TypeError exception. - -- obj.__contains__() now returns True/False instead of 1/0. SF patch - 820195. - -- Python no longer tries to be smart about recursive comparisons. - When comparing containers with cyclic references to themselves it - will now just hit the recursion limit. See SF patch 825639. - -- str and unicode built-in types now have an rsplit() method that is - same as split() except that it scans the string from the end - working towards the beginning. See SF feature request 801847. - -- Fixed a bug in object.__reduce_ex__ when using protocol 2. Failure - to clear the error when attempts to get the __getstate__ attribute - fail caused intermittent errors and odd behavior. - -- buffer objects based on other objects no longer cache a pointer to - the data and the data length. Instead, the appropriate tp_as_buffer - method is called as necessary. - -- fixed: if a file is opened with an explicit buffer size >= 1, repeated - close() calls would attempt to free() the buffer already free()ed on - the first call. - - -Extension modules ------------------ - -- Added socket.getservbyport(), and make the second argument in - getservbyname() and getservbyport() optional. - -- time module code that deals with input POSIX timestamps will now raise - ValueError if more than a second is lost in precision when the - timestamp is cast to the platform C time_t type. There's no chance - that the platform will do anything sensible with the result in such - cases. This includes ctime(), localtime() and gmtime(). Assorted - fromtimestamp() and utcfromtimestamp() methods in the datetime module - were also protected. Closes bugs #919012 and 975996. - -- fcntl.ioctl now warns if the mutate flag is not specified. - -- nt now properly allows referring to UNC roots, e.g. in nt.stat(). - -- the weakref module now supports additional objects: array.array, - sre.pattern_objects, file objects, and sockets. - -- operator.isMappingType() and operator.isSequenceType() now give - fewer false positives. - -- socket.sslerror is now a subclass of socket.error . Also added - socket.error to the socket module's C API. - -- Bug #920575: A problem where the _locale module segfaults on - nl_langinfo(ERA) caused by GNU libc's illegal NULL return is fixed. - -- array objects now support the copy module. Also, their resizing - scheme has been updated to match that used for list objects. This improves - the performance (speed and memory usage) of append() operations. - Also, array.array() and array.extend() now accept any iterable argument - for repeated appends without needing to create another temporary array. - -- cStringIO.writelines() now accepts any iterable argument and writes - the lines one at a time rather than joining them and writing once. - Made a parallel change to StringIO.writelines(). Saves memory and - makes suitable for use with generator expressions. - -- time.strftime() now checks that the values in its time tuple argument - are within the proper boundaries to prevent possible crashes from the - platform's C library implementation of strftime(). Can possibly - break code that uses values outside the range that didn't cause - problems previously (such as sitting day of year to 0). Fixes bug - #897625. - -- The socket module now supports Bluetooth sockets, if the - system has - -- Added a collections module containing a new datatype, deque(), - offering high-performance, thread-safe, memory friendly appends - and pops on either side of the deque. - -- Several modules now take advantage of collections.deque() for - improved performance: Queue, mutex, shlex, threading, and pydoc. - -- The operator module has two new functions, attrgetter() and - itemgetter() which are useful for creating fast data extractor - functions for map(), list.sort(), itertools.groupby(), and - other functions that expect a function argument. - -- socket.SHUT_{RD,WR,RDWR} was added. - -- os.getsid was added. - -- The pwd module incorrectly advertised its struct type as - struct_pwent; this has been renamed to struct_passwd. (The old name - is still supported for backwards compatibility.) - -- The xml.parsers.expat module now provides Expat 1.95.7. - -- socket.IPPROTO_IPV6 was added. - -- readline.clear_history was added. - -- select.select() now accepts sequences for its first three arguments. - -- cStringIO now supports the f.closed attribute. - -- The signal module now exposes SIGRTMIN and SIGRTMAX (if available). - -- curses module now supports use_default_colors(). [patch #739124] - -- Bug #811028: ncurses.h breakage on FreeBSD/MacOS X - -- Bug #814613: INET_ADDRSTRLEN fix needed for all compilers on SGI - -- Implemented non-recursive SRE matching scheme (#757624). - -- Implemented (?(id/name)yes|no) support in SRE (#572936). - -- random.seed() with no arguments or None uses time.time() as a default - seed. Modified to match Py2.2 behavior and use fractional seconds so - that successive runs are more likely to produce different sequences. - -- random.Random has a new method, getrandbits(k), which returns an int - with k random bits. This method is now an optional part of the API - for user defined generators. Any generator that defines genrandbits() - can now use randrange() for ranges with a length >= 2**53. Formerly, - randrange would return only even numbers for ranges that large (see - SF bug #812202). Generators that do not define genrandbits() now - issue a warning when randrange() is called with a range that large. - -- itertools has a new function, groupby() for aggregating iterables - into groups sharing the same key (as determined by a key function). - It offers some of functionality of SQL's groupby keyword and of - the Unix uniq filter. - -- itertools now has a new tee() function which produces two independent - iterators from a single iterable. - -- itertools.izip() with no arguments now returns an empty iterator instead - of raising a TypeError exception. - -- Fixed #853061: allow BZ2Compressor.compress() to receive an empty string - as parameter. - -Library -------- - -- Added a new module: cProfile, a C profiler with the same interface as the - profile module. cProfile avoids some of the drawbacks of the hotshot - profiler and provides a bit more information than the other two profilers. - Based on "lsprof" (patch #1212837). - -- Bug #1266283: The new function "lexists" is now in os.path.__all__. - -- Bug #981530: Fix UnboundLocalError in shutil.rmtree(). This affects - the documented behavior: the function passed to the onerror() - handler can now also be os.listdir. - -- Bug #754449: threading.Thread objects no longer mask exceptions raised during - interpreter shutdown with another exception from attempting to handle the - original exception. - -- Added decimal.py per PEP 327. - -- Bug #981299: rsync is now a recognized protocol in urlparse that uses a - "netloc" portion of a URL. - -- Bug #919012: shutil.move() will not try to move a directory into itself. - Thanks Johannes Gijsbers. - -- Bug #934282: pydoc.stripid() is now case-insensitive. Thanks Robin Becker. - -- Bug #823209: cmath.log() now takes an optional base argument so that its - API matches math.log(). - -- Bug #957381: distutils bdist_rpm no longer fails on recent RPM versions - that generate a -debuginfo.rpm - -- os.path.devnull has been added for all supported platforms. - -- Fixed #877165: distutils now picks the right C++ compiler command - on cygwin and mingw32. - -- urllib.urlopen().readline() now handles HTTP/0.9 correctly. - -- refactored site.py into functions. Also wrote regression tests for the - module. - -- The distutils install command now supports the --home option and - installation scheme for all platforms. - -- asyncore.loop now has a repeat count parameter that defaults to - looping forever. - -- The distutils sdist command now ignores all .svn directories, in - addition to CVS and RCS directories. .svn directories hold - administrative files for the Subversion source control system. - -- Added a new module: cookielib. Automatic cookie handling for HTTP - clients. Also, support for cookielib has been added to urllib2, so - urllib2.urlopen() can transparently handle cookies. - -- stringprep.py now uses built-in set() instead of sets.Set(). - -- Bug #876278: Unbounded recursion in modulefinder - -- Bug #780300: Swap public and system ID in LexicalHandler.startDTD. - Applications relying on the wrong order need to be corrected. - -- Bug #926075: Fixed a bug that returns a wrong pattern object - for a string or unicode object in sre.compile() when a different - type pattern with the same value exists. - -- Added countcallers arg to trace.Trace class (--trackcalls command line arg - when run from the command prompt). - -- Fixed a caching bug in platform.platform() where the argument of 'terse' was - not taken into consideration when caching value. - -- Added two new command-line arguments for profile (output file and - default sort). - -- Added global runctx function to profile module - -- Add hlist missing entryconfigure and entrycget methods. - -- The ptcp154 codec was added for Kazakh character set support. - -- Support non-anonymous ftp URLs in urllib2. - -- The encodings package will now apply codec name aliases - first before starting to try the import of the codec module. - This simplifies overriding built-in codecs with external - packages, e.g. the included CJK codecs with the JapaneseCodecs - package, by adjusting the aliases dictionary in encodings.aliases - accordingly. - -- base64 now supports RFC 3548 Base16, Base32, and Base64 encoding and - decoding standards. - -- urllib2 now supports processors. A processor is a handler that - implements an xxx_request or xxx_response method. These methods are - called for all requests. - -- distutils compilers now compile source files in the same order as - they are passed to the compiler. - -- pprint.pprint() and pprint.pformat() now have additional parameters - indent, width and depth. - -- Patch #750542: pprint now will pretty print subclasses of list, tuple - and dict too, as long as they don't overwrite __repr__(). - -- Bug #848614: distutils' msvccompiler fails to find the MSVC6 - compiler because of incomplete registry entries. - -- httplib.HTTP.putrequest now offers to omit the implicit Accept-Encoding. - -- Patch #841977: modulefinder didn't find extension modules in packages - -- imaplib.IMAP4.thread was added. - -- Plugged a minor hole in tempfile.mktemp() due to the use of - os.path.exists(), switched to using os.lstat() directly if possible. - -- bisect.py and heapq.py now have underlying C implementations - for better performance. - -- heapq.py has two new functions, nsmallest() and nlargest(). - -- traceback.format_exc has been added (similar to print_exc but it returns - a string). - -- xmlrpclib.MultiCall has been added. - -- poplib.POP3_SSL has been added. - -- tmpfile.mkstemp now returns an absolute path even if dir is relative. - -- urlparse is RFC 2396 compliant. - -- The fieldnames argument to the csv module's DictReader constructor is now - optional. If omitted, the first row of the file will be used as the - list of fieldnames. - -- encodings.bz2_codec was added for access to bz2 compression - using "a long string".encode('bz2') - -- Various improvements to unittest.py, realigned with PyUnit CVS. - -- dircache now passes exceptions to the caller, instead of returning - empty lists. - -- The bsddb module and dbhash module now support the iterator and - mapping protocols which make them more substitutable for dictionaries - and shelves. - -- The csv module's DictReader and DictWriter classes now accept keyword - arguments. This was an omission in the initial implementation. - -- The email package handles some RFC 2231 parameters with missing - CHARSET fields better. It also includes a patch to parameter - parsing when semicolons appear inside quotes. - -- sets.py now runs under Py2.2. In addition, the argument restrictions - for most set methods (but not the operators) have been relaxed to - allow any iterable. - -- _strptime.py now has a behind-the-scenes caching mechanism for the most - recent TimeRE instance used along with the last five unique directive - patterns. The overall module was also made more thread-safe. - -- random.cunifvariate() and random.stdgamma() were deprecated in Py2.3 - and removed in Py2.4. - -- Bug #823328: urllib2.py's HTTP Digest Auth support works again. - -- Patch #873597: CJK codecs are imported into rank of default codecs. - -Tools/Demos ------------ - -- A hotshotmain script was added to the Tools/scripts directory that - makes it easy to run a script under control of the hotshot profiler. - -- The db2pickle and pickle2db scripts can now dump/load gdbm files. - -- The file order on the command line of the pickle2db script was reversed. - It is now [ picklefile ] dbfile. This provides better symmetry with - db2pickle. The file arguments to both scripts are now source followed by - destination in situations where both files are given. - -- The pydoc script will display a link to the module documentation for - modules determined to be part of the core distribution. The documentation - base directory defaults to http://www.python.org/doc/current/lib/ but can - be changed by setting the PYTHONDOCS environment variable. - -- texcheck.py now detects double word errors. - -- md5sum.py mistakenly opened input files in text mode by default, a - silent and dangerous change from previous releases. It once again - opens input files in binary mode by default. The -t and -b flags - remain for compatibility with the 2.3 release, but -b is the default - now. - -- py-electric-colon now works when pending-delete/delete-selection mode is - in effect - -- py-help-at-point is no longer bound to the F1 key - it's still bound to - C-c C-h - -- Pynche was fixed to not crash when there is no ~/.pynche file and no - -d option was given. - -Build ------ - -- Bug #978645: Modules/getpath.c now builds properly in --disable-framework - build under OS X. - -- Profiling using gprof is now available if Python is configured with - --enable-profiling. - -- Profiling the VM using the Pentium TSC is now possible if Python - is configured --with-tsc. - -- In order to find libraries, setup.py now also looks in /lib64, for use - on AMD64. - -- Bug #934635: Fixed a bug where the configure script couldn't detect - getaddrinfo() properly if the KAME stack had SCTP support. - -- Support for missing ANSI C header files (limits.h, stddef.h, etc) was - removed. - -- Systems requiring the D4, D6 or D7 variants of pthreads are no longer - supported (see PEP 11). - -- Universal newline support can no longer be disabled (see PEP 11). - -- Support for DGUX, SunOS 4, IRIX 4 and Minix was removed (see PEP 11). - -- Support for systems requiring --with-dl-dld or --with-sgi-dl was removed - (see PEP 11). - -- Tests for sizeof(char) were removed since ANSI C mandates that - sizeof(char) must be 1. - -C API ------ - -- Thanks to Anthony Tuininga, the datetime module now supplies a C API - containing type-check macros and constructors. See new docs in the - Python/C API Reference Manual for details. - -- Private function _PyTime_DoubleToTimet added, to convert a Python - timestamp (C double) to platform time_t with some out-of-bounds - checking. Declared in new header file timefuncs.h. It would be - good to expose some other internal timemodule.c functions there. - -- New public functions PyEval_EvaluateFrame and PyGen_New to expose - generator objects. - -- New public functions Py_IncRef() and Py_DecRef(), exposing the - functionality of the Py_XINCREF() and Py_XDECREF macros. Useful for - runtime dynamic embedding of Python. See patch #938302, by Bob - Ippolito. - -- Added a new macro, PySequence_Fast_ITEMS, which retrieves a fast sequence's - underlying array of PyObject pointers. Useful for high speed looping. - -- Created a new method flag, METH_COEXIST, which causes a method to be loaded - even if already defined by a slot wrapper. This allows a __contains__ - method, for example, to co-exist with a defined sq_contains slot. This - is helpful because the PyCFunction can take advantage of optimized calls - whenever METH_O or METH_NOARGS flags are defined. - -- Added a new function, PyDict_Contains(d, k) which is like - PySequence_Contains() but is specific to dictionaries and executes - about 10% faster. - -- Added three new macros: Py_RETURN_NONE, Py_RETURN_TRUE, and Py_RETURN_FALSE. - Each return the singleton they mention after Py_INCREF()ing them. - -- Added a new function, PyTuple_Pack(n, ...) for constructing tuples from a - variable length argument list of Python objects without having to invoke - the more complex machinery of Py_BuildValue(). PyTuple_Pack(3, a, b, c) - is equivalent to Py_BuildValue("(OOO)", a, b, c). - -Windows -------- - -- The _winreg module could segfault when reading very large registry - values, due to unchecked alloca() calls (SF bug 851056). The fix is - uses either PyMem_Malloc(n) or PyString_FromStringAndSize(NULL, n), - as appropriate, followed by a size check. - -- file.truncate() could misbehave if the file was open for update - (modes r+, rb+, w+, wb+), and the most recent file operation before - the truncate() call was an input operation. SF bug 801631. - - -What's New in Python 2.3 final? -=============================== - -*Release date: 29-Jul-2003* - -IDLE ----- - -- Bug 778400: IDLE hangs when selecting "Edit with IDLE" from explorer. - This was unique to Windows, and was fixed by adding an -n switch to - the command the Windows installer creates to execute "Edit with IDLE" - context-menu actions. - -- IDLE displays a new message upon startup: some "personal firewall" - kinds of programs (for example, ZoneAlarm) open a dialog of their - own when any program opens a socket. IDLE does use sockets, talking - on the computer's internal loopback interface. This connection is not - visible on any external interface and no data is sent to or received - from the Internet. So, if you get such a dialog when opening IDLE, - asking whether to let pythonw.exe talk to address 127.0.0.1, say yes, - and rest assured no communication external to your machine is taking - place. If you don't allow it, IDLE won't be able to start. - - -What's New in Python 2.3 release candidate 2? -============================================= - -*Release date: 24-Jul-2003* - -Core and builtins ------------------ - -- It is now possible to import from zipfiles containing additional - data bytes before the zip compatible archive. Zipfiles containing a - comment at the end are still unsupported. - -Extension modules ------------------ - -- A longstanding bug in the parser module's initialization could cause - fatal internal refcount confusion when the module got initialized more - than once. This has been fixed. - -- Fixed memory leak in pyexpat; using the parser's ParseFile() method - with open files that aren't instances of the standard file type - caused an instance of the bound .read() method to be leaked on every - call. - -- Fixed some leaks in the locale module. - -Library -------- - -- Lib/encodings/rot_13.py when used as a script, now more properly - uses the first Python interpreter on your path. - -- Removed caching of TimeRE (and thus LocaleTime) in _strptime.py to - fix a locale related bug in the test suite. Although another patch - was needed to actually fix the problem, the cache code was not - restored. - -IDLE ----- - -- Calltips patches. - -Build ------ - -- For MacOSX, added -mno-fused-madd to BASECFLAGS to fix test_coercion - on Panther (OSX 10.3). - -C API ------ - -Windows -------- - -- The tempfile module could do insane imports on Windows if PYTHONCASEOK - was set, making temp file creation impossible. Repaired. - -- Add a patch to workaround pthread_sigmask() bugs in Cygwin. - -Mac ---- - -- Various fixes to pimp. - -- Scripts runs with pythonw no longer had full window manager access. - -- Don't force boot-disk-only install, for reasons unknown it causes - more problems than it solves. - - -What's New in Python 2.3 release candidate 1? -============================================= - -*Release date: 18-Jul-2003* - -Core and builtins ------------------ - -- The new function sys.getcheckinterval() returns the last value set - by sys.setcheckinterval(). - -- Several bugs in the symbol table phase of the compiler have been - fixed. Errors could be lost and compilation could fail without - reporting an error. SF patch 763201. - -- The interpreter is now more robust about importing the warnings - module. In an executable generated by freeze or similar programs, - earlier versions of 2.3 would fail if the warnings module could - not be found on the file system. Fixes SF bug 771097. - -- A warning about assignments to module attributes that shadow - builtins, present in earlier releases of 2.3, has been removed. - -- It is not possible to create subclasses of built-in types like str - and tuple that define an itemsize. Earlier releases of Python 2.3 - allowed this by mistake, leading to crashes and other problems. - -- The thread_id is now initialized to 0 in a non-thread build. SF bug - 770247. - -- SF bug 762891: "del p[key]" on proxy object no longer raises SystemError. - -Extension modules ------------------ - -- weakref.proxy() can now handle "del obj[i]" for proxy objects - defining __delitem__. Formerly, it generated a SystemError. - -- SSL no longer crashes the interpreter when the remote side disconnects. - -- On Unix the mmap module can again be used to map device files. - -- time.strptime now exclusively uses the Python implementation - contained within the _strptime module. - -- The print slot of weakref proxy objects was removed, because it was - not consistent with the object's repr slot. - -- The mmap module only checks file size for regular files, not - character or block devices. SF patch 708374. - -- The cPickle Pickler garbage collection support was fixed to traverse - the find_class attribute, if present. - -- There are several fixes for the bsddb3 wrapper module. - - bsddb3 no longer crashes if an environment is closed before a cursor - (SF bug 763298). - - The DB and DBEnv set_get_returns_none function was extended to take - a level instead of a boolean flag. The new level 2 means that in - addition, cursor.set()/.get() methods return None instead of raising - an exception. - - A typo was fixed in DBCursor.join_item(), preventing a crash. - -Library -------- - -- distutils now supports MSVC 7.1 - -- doctest now examines all docstrings by default. Previously, it would - skip over functions with private names (as indicated by the underscore - naming convention). The old default created too much of a risk that - user tests were being skipped inadvertently. Note, this change could - break code in the unlikely case that someone had intentionally put - failing tests in the docstrings of private functions. The breakage - is easily fixable by specifying the old behavior when calling testmod() - or Tester(). - -- There were several fixes to the way dumbdbms are closed. It's vital - that a dumbdbm database be closed properly, else the on-disk data - and directory files can be left in mutually inconsistent states. - dumbdbm.py's _Database.__del__() method attempted to close the - database properly, but a shutdown race in _Database._commit() could - prevent this from working, so that a program trusting __del__() to - get the on-disk files in synch could be badly surprised. The race - has been repaired. A sync() method was also added so that shelve - can guarantee data is written to disk. - - The close() method can now be called more than once without complaint. - -- The classes in threading.py are now new-style classes. That they - weren't before was an oversight. - -- The urllib2 digest authentication handlers now define the correct - auth_header. The earlier versions would fail at runtime. - -- SF bug 763023: fix uncaught ZeroDivisionError in difflib ratio methods - when there are no lines. - -- SF bug 763637: fix exception in Tkinter with after_cancel - which could occur with Tk 8.4 - -- SF bug 770601: CGIHTTPServer.py now passes the entire environment - to child processes. - -- SF bug 765238: add filter to fnmatch's __all__. - -- SF bug 748201: make time.strptime() error messages more helpful. - -- SF patch 764470: Do not dump the args attribute of a Fault object in - xmlrpclib. - -- SF patch 549151: urllib and urllib2 now redirect POSTs on 301 - responses. - -- SF patch 766650: The whichdb module was fixed to recognize dbm files - generated by gdbm on OS/2 EMX. - -- SF bugs 763047 and 763052: fixes bug of timezone value being left as - -1 when ``time.tzname[0] == time.tzname[1] and not time.daylight`` - is true when it should only when time.daylight is true. - -- SF bug 764548: re now allows subclasses of str and unicode to be - used as patterns. - -- SF bug 763637: In Tkinter, change after_cancel() to handle tuples - of varying sizes. Tk 8.4 returns a different number of values - than Tk 8.3. - -- SF bug 763023: difflib.ratio() did not catch zero division. - -- The Queue module now has an __all__ attribute. - -Tools/Demos ------------ - -- See Lib/idlelib/NEWS.txt for IDLE news. - -- SF bug 753592: webchecker/wsgui now handles user supplied directories. - -- The trace.py script has been removed. It is now in the standard library. - -Build ------ - -- Python now compiles with -fno-strict-aliasing if possible (SF bug 766696). - -- The socket module compiles on IRIX 6.5.10. - -- An irix64 system is treated the same way as an irix6 system (SF - patch 764560). - -- Several definitions were missing on FreeBSD 5.x unless the - __BSD_VISIBLE symbol was defined. configure now defines it as - needed. - -C API ------ - -- Unicode objects now support mbcs as a built-in encoding, so the C - API can use it without deferring to the encodings package. - -Windows -------- - -- The Windows implementation of PyThread_start_new_thread() never - checked error returns from Windows functions correctly. As a result, - it could claim to start a new thread even when the Microsoft - _beginthread() function failed (due to "too many threads" -- this is - on the order of thousands when it happens). In these cases, the - Python exception :: - - thread.error: can't start new thread - - is raised now. - -- SF bug 766669: Prevent a GPF on interpreter exit when sockets are in - use. The interpreter now calls WSACleanup() from Py_Finalize() - instead of from DLL teardown. - -Mac ---- - -- Bundlebuilder now inherits default values in the right way. It was - previously possible for app bundles to get a type of "BNDL" instead - of "APPL." Other improvements include, a --build-id option to - specify the CFBundleIdentifier and using the --python option to set - the executable in the bundle. - -- Fixed two bugs in MacOSX framework handling. - -- pythonw did not allow user interaction in 2.3rc1, this has been fixed. - -- Python is now compiled with -mno-fused-madd, making all tests pass - on Panther. - -What's New in Python 2.3 beta 2? -================================ - -*Release date: 29-Jun-2003* - -Core and builtins ------------------ - -- A program can now set the environment variable PYTHONINSPECT to some - string value in Python, and cause the interpreter to enter the - interactive prompt at program exit, as if Python had been invoked - with the -i option. - -- list.index() now accepts optional start and stop arguments. Similar - changes were made to UserList.index(). SF feature request 754014. - -- SF patch 751998 fixes an unwanted side effect of the previous fix - for SF bug 742860 (the next item). - -- SF bug 742860: "WeakKeyDictionary __delitem__ uses iterkeys". This - wasn't threadsafe, was very inefficient (expected time O(len(dict)) - instead of O(1)), and could raise a spurious RuntimeError if another - thread mutated the dict during __delitem__, or if a comparison function - mutated it. It also neglected to raise KeyError when the key wasn't - present; didn't raise TypeError when the key wasn't of a weakly - referencable type; and broke various more-or-less obscure dict - invariants by using a sequence of equality comparisons over the whole - set of dict keys instead of computing the key's hash code to narrow - the search to those keys with the same hash code. All of these are - considered to be bugs. A new implementation of __delitem__ repairs all - that, but note that fixing these bugs may change visible behavior in - code relying (whether intentionally or accidentally) on old behavior. - -- SF bug 734869: Fixed a compiler bug that caused a fatal error when - compiling a list comprehension that contained another list comprehension - embedded in a lambda expression. - -- SF bug 705231: builtin pow() no longer lets the platform C pow() - raise -1.0 to integer powers, because (at least) glibc gets it wrong - in some cases. The result should be -1.0 if the power is odd and 1.0 - if the power is even, and any float with a sufficiently large exponent - is (mathematically) an exact even integer. - -- SF bug 759227: A new-style class that implements __nonzero__() must - return a bool or int (but not an int subclass) from that method. This - matches the restriction on classic classes. - -- The encoding attribute has been added for file objects, and set to - the terminal encoding on Unix and Windows. - -- The softspace attribute of file objects became read-only by oversight. - It's writable again. - -- Reverted a 2.3 beta 1 change to iterators for subclasses of list and - tuple. By default, the iterators now access data elements directly - instead of going through __getitem__. If __getitem__ access is - preferred, then __iter__ can be overridden. - -- SF bug 735247: The staticmethod and super types participate in - garbage collection. Before this change, it was possible for leaks to - occur in functions with non-global free variables that used these types. - -Extension modules ------------------ - -- the socket module has a new exception, socket.timeout, to allow - timeouts to be handled separately from other socket errors. - -- SF bug 751276: cPickle has fixed to propagate exceptions raised in - user code. In earlier versions, cPickle caught and ignored any - exception when it performed operations that it expected to raise - specific exceptions like AttributeError. - -- cPickle Pickler and Unpickler objects now participate in garbage - collection. - -- mimetools.choose_boundary() could return duplicate strings at times, - especially likely on Windows. The strings returned are now guaranteed - unique within a single program run. - -- thread.interrupt_main() raises KeyboardInterrupt in the main thread. - dummy_thread has also been modified to try to simulate the behavior. - -- array.array.insert() now treats negative indices as being relative - to the end of the array, just like list.insert() does. (SF bug #739313) - -- The datetime module classes datetime, time, and timedelta are now - properly subclassable. - -- _tkinter.{get|set}busywaitinterval was added. - -- itertools.islice() now accepts stop=None as documented. - Fixes SF bug #730685. - -- the bsddb185 module is built in one restricted instance - - /usr/include/db.h exists and defines HASHVERSION to be 2. This is true - for many BSD-derived systems. - - -Library -------- - -- Some happy doctest extensions from Jim Fulton have been added to - doctest.py. These are already being used in Zope3. The two - primary ones: - - doctest.debug(module, name) extracts the doctests from the named object - in the given module, puts them in a temp file, and starts pdb running - on that file. This is great when a doctest fails. - - doctest.DocTestSuite(module=None) returns a synthesized unittest - TestSuite instance, to be run by the unittest framework, which - runs all the doctests in the module. This allows writing tests in - doctest style (which can be clearer and shorter than writing tests - in unittest style), without losing unittest's powerful testing - framework features (which doctest lacks). - -- For compatibility with doctests created before 2.3, if an expected - output block consists solely of "1" and the actual output block - consists solely of "True", it's accepted as a match; similarly - for "0" and "False". This is quite un-doctest-like, but is practical. - The behavior can be disabled by passing the new doctest module - constant DONT_ACCEPT_TRUE_FOR_1 to the new optionflags optional - argument. - -- ZipFile.testzip() now only traps BadZipfile exceptions. Previously, - a bare except caught to much and reported all errors as a problem - in the archive. - -- The logging module now has a new function, makeLogRecord() making - LogHandler easier to interact with DatagramHandler and SocketHandler. - -- The cgitb module has been extended to support plain text display (SF patch - 569574). - -- A brand new version of IDLE (from the IDLEfork project at - SourceForge) is now included as Lib/idlelib. The old Tools/idle is - no more. - -- Added a new module: trace (documentation missing). This module used - to be distributed in Tools/scripts. It uses sys.settrace() to trace - code execution -- either function calls or individual lines. It can - generate tracing output during execution or a post-mortem report of - code coverage. - -- The threading module has new functions settrace() and setprofile() - that cooperate with the functions of the same name in the sys - module. A function registered with the threading module will - be used for all threads it creates. The new trace module uses this - to provide tracing for code running in threads. - -- copy.py: applied SF patch 707900, fixing bug 702858, by Steven - Taschuk. Copying a new-style class that had a reference to itself - didn't work. (The same thing worked fine for old-style classes.) - Builtin functions are now treated as atomic, fixing bug #746304. - -- difflib.py has two new functions: context_diff() and unified_diff(). - -- More fixes to urllib (SF 549151): (a) When redirecting, always use - GET. This is common practice and more-or-less sanctioned by the - HTTP standard. (b) Add a handler for 307 redirection, which becomes - an error for POST, but a regular redirect for GET and HEAD - -- Added optional 'onerror' argument to os.walk(), to control error - handling. - -- inspect.is{method|data}descriptor was added, to allow pydoc display - __doc__ of data descriptors. - -- Fixed socket speed loss caused by use of the _socketobject wrapper class - in socket.py. - -- timeit.py now checks the current directory for imports. - -- urllib2.py now knows how to order proxy classes, so the user doesn't - have to insert it in front of other classes, nor do dirty tricks like - inserting a "dummy" HTTPHandler after a ProxyHandler when building an - opener with proxy support. - -- Iterators have been added for dbm keys. - -- random.Random objects can now be pickled. - -Tools/Demos ------------ - -- pydoc now offers help on keywords and topics. - -- Tools/idle is gone; long live Lib/idlelib. - -- diff.py prints file diffs in context, unified, or ndiff formats, - providing a command line interface to difflib.py. - -- texcheck.py is a new script for making a rough validation of Python LaTeX - files. - -Build ------ - -- Setting DESTDIR during 'make install' now allows specifying a - different root directory. - -C API ------ - -- PyType_Ready(): If a type declares that it participates in gc - (Py_TPFLAGS_HAVE_GC), and its base class does not, and its base class's - tp_free slot is the default _PyObject_Del, and type does not define - a tp_free slot itself, _PyObject_GC_Del is assigned to type->tp_free. - Previously _PyObject_Del was inherited, which could at best lead to a - segfault. In addition, if even after this magic the type's tp_free - slot is _PyObject_Del or NULL, and the type is a base type - (Py_TPFLAGS_BASETYPE), TypeError is raised: since the type is a base - type, its dealloc function must call type->tp_free, and since the type - is gc'able, tp_free must not be NULL or _PyObject_Del. - -- PyThreadState_SetAsyncExc(): A new API (deliberately accessible only - from C) to interrupt a thread by sending it an exception. It is - intentional that you have to write your own C extension to call it - from Python. - - -New platforms -------------- - -None this time. - -Tests ------ - -- test_imp rewritten so that it doesn't raise RuntimeError if run as a - side effect of being imported ("import test.autotest"). - -Windows -------- - -- The Windows installer ships with Tcl/Tk 8.4.3 (upgraded from 8.4.1). - -- The installer always suggested that Python be installed on the C: - drive, due to a hardcoded "C:" generated by the Wise installation - wizard. People with machines where C: is not the system drive - usually want Python installed on whichever drive is their system drive - instead. We removed the hardcoded "C:", and two testers on machines - where C: is not the system drive report that the installer now - suggests their system drive. Note that you can always select the - directory you want in the "Select Destination Directory" dialog -- - that's what it's for. - -Mac ---- - -- There's a new module called "autoGIL", which offers a mechanism to - automatically release the Global Interpreter Lock when an event loop - goes to sleep, allowing other threads to run. It's currently only - supported on OSX, in the Mach-O version. -- The OSA modules now allow direct access to properties of the - toplevel application class (in AppleScript terminology). -- The Package Manager can now update itself. - -SourceForge Bugs and Patches Applied ------------------------------------- - -430160, 471893, 501716, 542562, 549151, 569574, 595837, 596434, -598163, 604210, 604716, 610332, 612627, 614770, 620190, 621891, -622042, 639139, 640236, 644345, 649742, 649742, 658233, 660022, -661318, 661676, 662807, 662923, 666219, 672855, 678325, 682347, -683486, 684981, 685773, 686254, 692776, 692959, 693094, 696777, -697989, 700827, 703666, 708495, 708604, 708901, 710733, 711902, -713722, 715782, 718286, 719359, 719367, 723136, 723831, 723962, -724588, 724767, 724767, 725942, 726150, 726446, 726869, 727051, -727719, 727719, 727805, 728277, 728563, 728656, 729096, 729103, -729293, 729297, 729300, 729317, 729395, 729622, 729817, 730170, -730296, 730594, 730685, 730826, 730963, 731209, 731403, 731504, -731514, 731626, 731635, 731643, 731644, 731644, 731689, 732124, -732143, 732234, 732284, 732284, 732479, 732761, 732783, 732951, -733667, 733781, 734118, 734231, 734869, 735051, 735293, 735527, -735613, 735694, 736962, 736962, 737970, 738066, 739313, 740055, -740234, 740301, 741806, 742126, 742741, 742860, 742860, 742911, -744041, 744104, 744238, 744687, 744877, 745055, 745478, 745525, -745620, 746012, 746304, 746366, 746801, 746953, 747348, 747667, -747954, 748846, 748849, 748973, 748975, 749191, 749210, 749759, -749831, 749911, 750008, 750092, 750542, 750595, 751038, 751107, -751276, 751451, 751916, 751941, 751956, 751998, 752671, 753451, -753602, 753617, 753845, 753925, 754014, 754340, 754447, 755031, -755087, 755147, 755245, 755683, 755987, 756032, 756996, 757058, -757229, 757818, 757821, 757822, 758112, 758910, 759227, 759889, -760257, 760703, 760792, 761104, 761337, 761519, 761830, 762455 - - -What's New in Python 2.3 beta 1? -================================ - -*Release date: 25-Apr-2003* - -Core and builtins ------------------ - -- New format codes B, H, I, k and K have been implemented for - PyArg_ParseTuple and PyBuild_Value. - -- New built-in function sum(seq, start=0) returns the sum of all the - items in iterable object seq, plus start (items are normally numbers, - and cannot be strings). - -- bool() called without arguments now returns False rather than - raising an exception. This is consistent with calling the - constructors for the other built-in types -- called without argument - they all return the false value of that type. (SF patch #724135) - -- In support of PEP 269 (making the pgen parser generator accessible - from Python), some changes to the pgen code structure were made; a - few files that used to be linked only with pgen are now linked with - Python itself. - -- The repr() of a weakref object now shows the __name__ attribute of - the referenced object, if it has one. - -- super() no longer ignores data descriptors, except __class__. See - the thread started at - http://mail.python.org/pipermail/python-dev/2003-April/034338.html - -- list.insert(i, x) now interprets negative i as it would be - interpreted by slicing, so negative values count from the end of the - list. This was the only place where such an interpretation was not - placed on a list index. - -- range() now works even if the arguments are longs with magnitude - larger than sys.maxint, as long as the total length of the sequence - fits. E.g., range(2**100, 2**101, 2**100) is the following list: - [1267650600228229401496703205376L]. (SF patch #707427.) - -- Some horridly obscure problems were fixed involving interaction - between garbage collection and old-style classes with "ambitious" - getattr hooks. If an old-style instance didn't have a __del__ method, - but did have a __getattr__ hook, and the instance became reachable - only from an unreachable cycle, and the hook resurrected or deleted - unreachable objects when asked to resolve "__del__", anything up to - a segfault could happen. That's been repaired. - -- dict.pop now takes an optional argument specifying a default - value to return if the key is not in the dict. If a default is not - given and the key is not found, a KeyError will still be raised. - Parallel changes were made to UserDict.UserDict and UserDict.DictMixin. - [SF patch #693753] (contributed by Michael Stone.) - -- sys.getfilesystemencoding() was added to expose - Py_FileSystemDefaultEncoding. - -- New function sys.exc_clear() clears the current exception. This is - rarely needed, but can sometimes be useful to release objects - referenced by the traceback held in sys.exc_info()[2]. (SF patch - #693195.) - -- On 64-bit systems, a dictionary could contain duplicate long/int keys - if the key value was larger than 2**32. See SF bug #689659. - -- Fixed SF bug #663074. The codec system was using global static - variables to store internal data. As a result, any attempts to use the - unicode system with multiple active interpreters, or successive - interpreter executions, would fail. - -- "%c" % u"a" now returns a unicode string instead of raising a - TypeError. u"%c" % 0xffffffff now raises an OverflowError instead - of a ValueError to be consistent with "%c" % 256. See SF patch #710127. - -Extension modules ------------------ - -- The socket module now provides the functions inet_pton and inet_ntop - for converting between string and packed representation of IP - addresses. There is also a new module variable, has_ipv6, which is - True iff the current Python has IPv6 support. See SF patch #658327. - -- Tkinter wrappers around Tcl variables now pass objects directly - to Tcl, instead of first converting them to strings. - -- The .*? pattern in the re module is now special-cased to avoid the - recursion limit. (SF patch #720991 -- many thanks to Gary Herron - and Greg Chapman.) - -- New function sys.call_tracing() allows pdb to debug code - recursively. - -- New function gc.get_referents(obj) returns a list of objects - directly referenced by obj. In effect, it exposes what the object's - tp_traverse slot does, and can be helpful when debugging memory - leaks. - -- The iconv module has been removed from this release. - -- The platform-independent routines for packing floats in IEEE formats - (struct.pack's f, d codes; pickle and cPickle's protocol 1 - pickling of floats) ignored that rounding can cause a carry to - propagate. The worst consequence was that, in rare cases, f - could produce strings that, when unpacked again, were a factor of 2 - away from the original float. This has been fixed. See SF bug - #705836. - -- New function time.tzset() provides access to the C library tzset() - function, if supported. (SF patch #675422.) - -- Using createfilehandler, deletefilehandler, createtimerhandler functions - on Tkinter.tkinter (_tkinter module) no longer crashes the interpreter. - See SF bug #692416. - -- Modified the fcntl.ioctl() function to allow modification of a passed - mutable buffer (for details see the reference documentation). - -- Made user requested changes to the itertools module. - Subsumed the times() function into repeat(). - Added chain() and cycle(). - -- The rotor module is now deprecated; the encryption algorithm it uses - is not believed to be secure, and including crypto code with Python - has implications for exporting and importing it in various countries. - -- The socket module now always uses the _socketobject wrapper class, even on - platforms which have dup(2). The makefile() method is built directly - on top of the socket without duplicating the file descriptor, allowing - timeouts to work properly. - -Library -------- - -- New generator function os.walk() is an easy-to-use alternative to - os.path.walk(). See os module docs for details. os.path.walk() - isn't deprecated at this time, but may become deprecated in a - future release. - -- Added new module "platform" which provides a wide range of tools - for querying platform dependent features. - -- netrc now allows ASCII punctuation characters in passwords. - -- shelve now supports the optional writeback argument, and exposes - pickle protocol versions. - -- Several methods of nntplib.NNTP have grown an optional file argument - which specifies a file where to divert the command's output - (already supported by the body() method). (SF patch #720468) - -- The self-documenting XML server library DocXMLRPCServer was added. - -- Support for internationalized domain names has been added through - the 'idna' and 'punycode' encodings, the 'stringprep' module, the - 'mkstringprep' tool, and enhancements to the socket and httplib - modules. - -- htmlentitydefs has two new dictionaries: name2codepoint maps - HTML entity names to Unicode codepoints (as integers). - codepoint2name is the reverse mapping. See SF patch #722017. - -- pdb has a new command, "debug", which lets you step through - arbitrary code from the debugger's (pdb) prompt. - -- unittest.failUnlessEqual and its equivalent unittest.assertEqual now - return 'not a == b' rather than 'a != b'. This gives the desired - result for classes that define __eq__ without defining __ne__. - -- sgmllib now supports SGML marked sections, in particular the - MS Office extensions. - -- The urllib module now offers support for the iterator protocol. - SF patch 698520 contributed by Brett Cannon. - -- New module timeit provides a simple framework for timing the - execution speed of expressions and statements. - -- sets.Set objects now support mixed-type __eq__ and __ne__, instead - of raising TypeError. If x is a Set object and y is a non-Set object, - x == y is False, and x != y is True. This is akin to the change made - for mixed-type comparisons of datetime objects in 2.3a2; more info - about the rationale is in the NEWS entry for that. See also SF bug - report . - -- On Unix platforms, if os.listdir() is called with a Unicode argument, - it now returns Unicode strings. (This behavior was added earlier - to the Windows NT/2k/XP version of os.listdir().) - -- Distutils: both 'py_modules' and 'packages' keywords can now be specified - in core.setup(). Previously you could supply one or the other, but - not both of them. (SF patch #695090 from Bernhard Herzog) - -- New csv package makes it easy to read/write CSV files. - -- Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and - commands. An iterator interface was also implemented. - -Tools/Demos ------------ - -- New script combinerefs.py helps analyze new PYTHONDUMPREFS output. - See the module docstring for details. - -Build ------ - -- Fix problem building on OSF1 because the compiler only accepted - preprocessor directives that start in column 1. (SF bug #691793.) - -C API ------ - -- Added PyGC_Collect(), equivalent to calling gc.collect(). - -- PyThreadState_GetDict() was changed not to raise an exception or - issue a fatal error when no current thread state is available. This - makes it possible to print dictionaries when no thread is active. - -- LONG_LONG was renamed to PY_LONG_LONG. Extensions that use this and - need compatibility with previous versions can use this: - - #ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG - #endif - -- Added PyObject_SelfIter() to fill the tp_iter slot for the - typical case where the method returns its self argument. - -- The extended type structure used for heap types (new-style - classes defined by Python code using a class statement) is now - exported from object.h as PyHeapTypeObject. (SF patch #696193.) - -New platforms -------------- - -None this time. - -Tests ------ - -- test_timeout now requires -u network to be passed to regrtest to run. - See SF bug #692988. - -Windows -------- - -- os.fsync() now exists on Windows, and calls the Microsoft _commit() - function. - -- New function winsound.MessageBeep() wraps the Win32 API - MessageBeep(). - -Mac ---- - -- os.listdir() now returns Unicode strings on MacOS X when called with - a Unicode argument. See the general news item under "Library". - -- A new method MacOS.WMAvailable() returns true if it is safe to access - the window manager, false otherwise. - -- EasyDialogs dialogs are now movable-modal, and if the application is - currently in the background they will ask to be moved to the foreground - before displaying. - -- OSA Scripting support has improved a lot, and gensuitemodule.py can now - be used by mere mortals. The documentation is now also more or less - complete. - -- The IDE (in a framework build) now includes introductory documentation - in Apple Help Viewer format. - - -What's New in Python 2.3 alpha 2? -================================= - -*Release date: 19-Feb-2003* - -Core and builtins ------------------ - -- Negative positions returned from PEP 293 error callbacks are now - treated as being relative to the end of the input string. Positions - that are out of bounds raise an IndexError. - -- sys.path[0] (the directory from which the script is loaded) is now - turned into an absolute pathname, unless it is the empty string. - (SF patch #664376.) - -- Finally fixed the bug in compile() and exec where a string ending - with an indented code block but no newline would raise SyntaxError. - This would have been a four-line change in parsetok.c... Except - codeop.py depends on this behavior, so a compilation flag had to be - invented that causes the tokenizer to revert to the old behavior; - this required extra changes to 2 .h files, 2 .c files, and 2 .py - files. (Fixes SF bug #501622.) - -- If a new-style class defines neither __new__ nor __init__, its - constructor would ignore all arguments. This is changed now: the - constructor refuses arguments in this case. This might break code - that worked under Python 2.2. The simplest fix is to add a no-op - __init__: ``def __init__(self, *args, **kw): pass``. - -- Through a bytecode optimizer bug (and I bet you didn't even know - Python *had* a bytecode optimizer :-), "unsigned" hex/oct constants - with a leading minus sign would come out with the wrong sign. - ("Unsigned" hex/oct constants are those with a face value in the - range sys.maxint+1 through sys.maxint*2+1, inclusive; these have - always been interpreted as negative numbers through sign folding.) - E.g. 0xffffffff is -1, and -(0xffffffff) is 1, but -0xffffffff would - come out as -4294967295. This was the case in Python 2.2 through - 2.2.2 and 2.3a1, and in Python 2.4 it will once again have that - value, but according to PEP 237 it really needs to be 1 now. This - will be backported to Python 2.2.3 a well. (SF #660455) - -- int(s, base) sometimes sign-folds hex and oct constants; it only - does this when base is 0 and s.strip() starts with a '0'. When the - sign is actually folded, as in int("0xffffffff", 0) on a 32-bit - machine, which returns -1, a FutureWarning is now issued; in Python - 2.4, this will return 4294967295L, as do int("+0xffffffff", 0) and - int("0xffffffff", 16) right now. (PEP 347) - -- super(X, x): x may now be a proxy for an X instance, i.e. - issubclass(x.__class__, X) but not issubclass(type(x), X). - -- isinstance(x, X): if X is a new-style class, this is now equivalent - to issubclass(type(x), X) or issubclass(x.__class__, X). Previously - only type(x) was tested. (For classic classes this was already the - case.) - -- compile(), eval() and the exec statement now fully support source code - passed as unicode strings. - -- int subclasses can be initialized with longs if the value fits in an int. - See SF bug #683467. - -- long(string, base) takes time linear in len(string) when base is a power - of 2 now. It used to take time quadratic in len(string). - -- filter returns now Unicode results for Unicode arguments. - -- raw_input can now return Unicode objects. - -- List objects' sort() method now accepts None as the comparison function. - Passing None is semantically identical to calling sort() with no - arguments. - -- Fixed crash when printing a subclass of str and __str__ returned self. - See SF bug #667147. - -- Fixed an invalid RuntimeWarning and an undetected error when trying - to convert a long integer into a float which couldn't fit. - See SF bug #676155. - -- Function objects now have a __module__ attribute that is bound to - the name of the module in which the function was defined. This - applies for C functions and methods as well as functions and methods - defined in Python. This attribute is used by pickle.whichmodule(), - which changes the behavior of whichmodule slightly. In Python 2.2 - whichmodule() returns "__main__" for functions that are not defined - at the top-level of a module (examples: methods, nested functions). - Now whichmodule() will return the proper module name. - -Extension modules ------------------ - -- operator.isNumberType() now checks that the object has a nb_int or - nb_float slot, rather than simply checking whether it has a non-NULL - tp_as_number pointer. - -- The imp module now has ways to acquire and release the "import - lock": imp.acquire_lock() and imp.release_lock(). Note: this is a - reentrant lock, so releasing the lock only truly releases it when - this is the last release_lock() call. You can check with - imp.lock_held(). (SF bug #580952 and patch #683257.) - -- Change to cPickle to match pickle.py (see below and PEP 307). - -- Fix some bugs in the parser module. SF bug #678518. - -- Thanks to Scott David Daniels, a subtle bug in how the zlib - extension implemented flush() was fixed. Scott also rewrote the - zlib test suite using the unittest module. (SF bug #640230 and - patch #678531.) - -- Added an itertools module containing high speed, memory efficient - looping constructs inspired by tools from Haskell and SML. - -- The SSL module now handles sockets with a timeout set correctly (SF - patch #675750, fixing SF bug #675552). - -- os/posixmodule has grown the sysexits.h constants (EX_OK and friends). - -- Fixed broken threadstate swap in readline that could cause fatal - errors when a readline hook was being invoked while a background - thread was active. (SF bugs #660476 and #513033.) - -- fcntl now exposes the strops.h I_* constants. - -- Fix a crash on Solaris that occurred when calling close() on - an mmap'ed file which was already closed. (SF patch #665913) - -- Fixed several serious bugs in the zipimport implementation. - -- datetime changes: - - The date class is now properly subclassable. (SF bug #720908) - - The datetime and datetimetz classes have been collapsed into a single - datetime class, and likewise the time and timetz classes into a single - time class. Previously, a datetimetz object with tzinfo=None acted - exactly like a datetime object, and similarly for timetz. This wasn't - enough of a difference to justify distinct classes, and life is simpler - now. - - today() and now() now round system timestamps to the closest - microsecond . This repairs an - irritation most likely seen on Windows systems. - - In dt.astimezone(tz), if tz.utcoffset(dt) returns a duration, - ValueError is raised if tz.dst(dt) returns None (2.3a1 treated it - as 0 instead, but a tzinfo subclass wishing to participate in - time zone conversion has to take a stand on whether it supports - DST; if you don't care about DST, then code dst() to return 0 minutes, - meaning that DST is never in effect). - - The tzinfo methods utcoffset() and dst() must return a timedelta object - (or None) now. In 2.3a1 they could also return an int or long, but that - was an unhelpfully redundant leftover from an earlier version wherein - they couldn't return a timedelta. TOOWTDI. - - The example tzinfo class for local time had a bug. It was replaced - by a later example coded by Guido. - - datetime.astimezone(tz) no longer raises an exception when the - input datetime has no UTC equivalent in tz. For typical "hybrid" time - zones (a single tzinfo subclass modeling both standard and daylight - time), this case can arise one hour per year, at the hour daylight time - ends. See new docs for details. In short, the new behavior mimics - the local wall clock's behavior of repeating an hour in local time. - - dt.astimezone() can no longer be used to convert between naive and aware - datetime objects. If you merely want to attach, or remove, a tzinfo - object, without any conversion of date and time members, use - dt.replace(tzinfo=whatever) instead, where "whatever" is None or a - tzinfo subclass instance. - - A new method tzinfo.fromutc(dt) can be overridden in tzinfo subclasses - to give complete control over how a UTC time is to be converted to - a local time. The default astimezone() implementation calls fromutc() - as its last step, so a tzinfo subclass can affect that too by overriding - fromutc(). It's expected that the default fromutc() implementation will - be suitable as-is for "almost all" time zone subclasses, but the - creativity of political time zone fiddling appears unbounded -- fromutc() - allows the highly motivated to emulate any scheme expressible in Python. - - datetime.now(): The optional tzinfo argument was undocumented (that's - repaired), and its name was changed to tz ("tzinfo" is overloaded enough - already). With a tz argument, now(tz) used to return the local date - and time, and attach tz to it, without any conversion of date and time - members. This was less than useful. Now now(tz) returns the current - date and time as local time in tz's time zone, akin to :: - - tz.fromutc(datetime.utcnow().replace(tzinfo=utc)) - - where "utc" is an instance of a tzinfo subclass modeling UTC. Without - a tz argument, now() continues to return the current local date and time, - as a naive datetime object. - - datetime.fromtimestamp(): Like datetime.now() above, this had less than - useful behavior when the optional tinzo argument was specified. See - also SF bug report . - - date and datetime comparison: In order to prevent comparison from - falling back to the default compare-object-addresses strategy, these - raised TypeError whenever they didn't understand the other object type. - They still do, except when the other object has a "timetuple" attribute, - in which case they return NotImplemented now. This gives other - datetime objects (e.g., mxDateTime) a chance to intercept the - comparison. - - date, time, datetime and timedelta comparison: When the exception - for mixed-type comparisons in the last paragraph doesn't apply, if - the comparison is == then False is returned, and if the comparison is - != then True is returned. Because dict lookup and the "in" operator - only invoke __eq__, this allows, for example, :: - - if some_datetime in some_sequence: - - and :: - - some_dict[some_timedelta] = whatever - - to work as expected, without raising TypeError just because the - sequence is heterogeneous, or the dict has mixed-type keys. [This - seems like a good idea to implement for all mixed-type comparisons - that don't want to allow falling back to address comparison.] - - The constructors building a datetime from a timestamp could raise - ValueError if the platform C localtime()/gmtime() inserted "leap - seconds". Leap seconds are ignored now. On such platforms, it's - possible to have timestamps that differ by a second, yet where - datetimes constructed from them are equal. - - The pickle format of date, time and datetime objects has changed - completely. The undocumented pickler and unpickler functions no - longer exist. The undocumented __setstate__() and __getstate__() - methods no longer exist either. - -Library -------- - -- The logging module was updated slightly; the WARN level was renamed - to WARNING, and the matching function/method warn() to warning(). - -- The pickle and cPickle modules were updated with a new pickling - protocol (documented by pickletools.py, see below) and several - extensions to the pickle customization API (__reduce__, __setstate__ - etc.). The copy module now uses more of the pickle customization - API to copy objects that don't implement __copy__ or __deepcopy__. - See PEP 307 for details. - -- The distutils "register" command now uses http://www.python.org/pypi - as the default repository. (See PEP 301.) - -- the platform dependent path related variables sep, altsep, - pathsep, curdir, pardir and defpath are now defined in the platform - dependent path modules (e.g. ntpath.py) rather than os.py, so these - variables are now available via os.path. They continue to be - available from the os module. - (see ). - -- array.array was added to the types repr.py knows about (see - ). - -- The new pickletools.py contains lots of documentation about pickle - internals, and supplies some helpers for working with pickles, such as - a symbolic pickle disassembler. - -- xmlrpclib.py now supports the built-in boolean type. - -- py_compile has a new 'doraise' flag and a new PyCompileError - exception. - -- SimpleXMLRPCServer now supports CGI through the CGIXMLRPCRequestHandler - class. - -- The sets module now raises TypeError in __cmp__, to clarify that - sets are not intended to be three-way-compared; the comparison - operators are overloaded as subset/superset tests. - -- Bastion.py and rexec.py are disabled. These modules are not safe in - Python 2.2. or 2.3. - -- realpath is now exported when doing ``from poxixpath import *``. - It is also exported for ntpath, macpath, and os2emxpath. - See SF bug #659228. - -- New module tarfile from Lars Gustäbel provides a comprehensive interface - to tar archive files with transparent gzip and bzip2 compression. - See SF patch #651082. - -- urlparse can now parse imap:// URLs. See SF feature request #618024. - -- Tkinter.Canvas.scan_dragto() provides an optional parameter to support - the gain value which is passed to Tk. SF bug# 602259. - -- Fix logging.handlers.SysLogHandler protocol when using UNIX domain sockets. - See SF patch #642974. - -- The dospath module was deleted. Use the ntpath module when manipulating - DOS paths from other platforms. - -Tools/Demos ------------ - -- Two new scripts (db2pickle.py and pickle2db.py) were added to the - Tools/scripts directory to facilitate conversion from the old bsddb module - to the new one. While the user-visible API of the new module is - compatible with the old one, it's likely that the version of the - underlying database library has changed. To convert from the old library, - run the db2pickle.py script using the old version of Python to convert it - to a pickle file. After upgrading Python, run the pickle2db.py script - using the new version of Python to reconstitute your database. For - example: - - % python2.2 db2pickle.py -h some.db > some.pickle - % python2.3 pickle2db.py -h some.db.new < some.pickle - - Run the scripts without any args to get a usage message. - - -Build ------ - -- The audio driver tests (test_ossaudiodev.py and - test_linuxaudiodev.py) are no longer run by default. This is - because they don't always work, depending on your hardware and - software. To run these tests, you must use an invocation like :: - - ./python Lib/test/regrtest.py -u audio test_ossaudiodev - -- On systems which build using the configure script, compiler flags which - used to be lumped together using the OPT flag have been split into two - groups, OPT and BASECFLAGS. OPT is meant to carry just optimization- and - debug-related flags like "-g" and "-O3". BASECFLAGS is meant to carry - compiler flags that are required to get a clean compile. On some - platforms (many Linux flavors in particular) BASECFLAGS will be empty by - default. On others, such as Mac OS X and SCO, it will contain required - flags. This change allows people building Python to override OPT without - fear of clobbering compiler flags which are required to get a clean build. - -- On Darwin/Mac OS X platforms, /sw/lib and /sw/include are added to the - relevant search lists in setup.py. This allows users building Python to - take advantage of the many packages available from the fink project - . - -- A new Makefile target, scriptsinstall, installs a number of useful scripts - from the Tools/scripts directory. - -C API ------ - -- PyEval_GetFrame() is now declared to return a ``PyFrameObject *`` - instead of a plain ``PyObject *``. (SF patch #686601.) - -- PyNumber_Check() now checks that the object has a nb_int or nb_float - slot, rather than simply checking whether it has a non-NULL - tp_as_number pointer. - -- A C type that inherits from a base type that defines tp_as_buffer - will now inherit the tp_as_buffer pointer if it doesn't define one. - (SF #681367) - -- The PyArg_Parse functions now issue a DeprecationWarning if a float - argument is provided when an integer is specified (this affects the 'b', - 'B', 'h', 'H', 'i', and 'l' codes). Future versions of Python will - raise a TypeError. - -Tests ------ - -- Several tests weren't being run from regrtest.py (test_timeout.py, - test_tarfile.py, test_netrc.py, test_multifile.py, - test_importhooks.py and test_imp.py). Now they are. (Note to - developers: please read Lib/test/README when creating a new test, to - make sure to do it right! All tests need to use either unittest or - pydoc.) - -- Added test_posix.py, a test suite for the posix module. - -- Added test_hexoct.py, a test suite for hex/oct constant folding. - -Windows -------- - -- The timeout code for socket connect() didn't work right; this has - now been fixed. test_timeout.py should pass (at least most of the - time). - -- distutils' msvccompiler class now passes the preprocessor options to - the resource compiler. See SF patch #669198. - -- The bsddb module now ships with Sleepycat's 4.1.25.NC, the latest - release without strong cryptography. - -- sys.path[0], if it contains a directory name, is now always an - absolute pathname. (SF patch #664376.) - -- The new logging package is now installed by the Windows installer. It - wasn't in 2.3a1 due to oversight. - -Mac ---- - -- There are new dialogs EasyDialogs.AskFileForOpen, AskFileForSave - and AskFolder. The old macfs.StandardGetFile and friends are deprecated. - -- Most of the standard library now uses pathnames or FSRefs in preference - of FSSpecs, and use the underlying Carbon.File and Carbon.Folder modules - in stead of macfs. macfs will probably be deprecated in the future. - -- Type Carbon.File.FSCatalogInfo and supporting methods have been implemented. - This also makes macfs.FSSpec.SetDates() work again. - -- There is a new module pimp, the package install manager for Python, and - accompanying applet PackageManager. These allow you to easily download - and install pretested extension packages either in source or binary - form. Only in MacPython-OSX. - -- Applets are now built with bundlebuilder in MacPython-OSX, which should make - them more robust and also provides a path towards BuildApplication. The - downside of this change is that applets can no longer be run from the - Terminal window, this will hopefully be fixed in the 2.3b1. - - -What's New in Python 2.3 alpha 1? -================================= - -*Release date: 31-Dec-2002* - -Type/class unification and new-style classes --------------------------------------------- - -- One can now assign to __bases__ and __name__ of new-style classes. - -- dict() now accepts keyword arguments so that dict(one=1, two=2) - is the equivalent of {"one": 1, "two": 2}. Accordingly, - the existing (but undocumented) 'items' keyword argument has - been eliminated. This means that dict(items=someMapping) now has - a different meaning than before. - -- int() now returns a long object if the argument is outside the - integer range, so int("4" * 1000), int(1e200) and int(1L<<1000) will - all return long objects instead of raising an OverflowError. - -- Assignment to __class__ is disallowed if either the old or the new - class is a statically allocated type object (such as defined by an - extension module). This prevents anomalies like 2.__class__ = bool. - -- New-style object creation and deallocation have been sped up - significantly; they are now faster than classic instance creation - and deallocation. - -- The __slots__ variable can now mention "private" names, and the - right thing will happen (e.g. __slots__ = ["__foo"]). - -- The built-ins slice() and buffer() are now callable types. The - types classobj (formerly class), code, function, instance, and - instancemethod (formerly instance-method), which have no built-in - names but are accessible through the types module, are now also - callable. The type dict-proxy is renamed to dictproxy. - -- Cycles going through the __class__ link of a new-style instance are - now detected by the garbage collector. - -- Classes using __slots__ are now properly garbage collected. - [SF bug 519621] - -- Tightened the __slots__ rules: a slot name must be a valid Python - identifier. - -- The constructor for the module type now requires a name argument and - takes an optional docstring argument. Previously, this constructor - ignored its arguments. As a consequence, deriving a class from a - module (not from the module type) is now illegal; previously this - created an unnamed module, just like invoking the module type did. - [SF bug 563060] - -- A new type object, 'basestring', is added. This is a common base type - for 'str' and 'unicode', and can be used instead of - types.StringTypes, e.g. to test whether something is "a string": - isinstance(x, basestring) is True for Unicode and 8-bit strings. This - is an abstract base class and cannot be instantiated directly. - -- Changed new-style class instantiation so that when C's __new__ - method returns something that's not a C instance, its __init__ is - not called. [SF bug #537450] - -- Fixed super() to work correctly with class methods. [SF bug #535444] - -- If you try to pickle an instance of a class that has __slots__ but - doesn't define or override __getstate__, a TypeError is now raised. - This is done by adding a bozo __getstate__ to the class that always - raises TypeError. (Before, this would appear to be pickled, but the - state of the slots would be lost.) - -Core and builtins ------------------ - -- Import from zipfiles is now supported. The name of a zipfile placed - on sys.path causes the import statement to look for importable Python - modules (with .py, pyc and .pyo extensions) and packages inside the - zipfile. The zipfile import follows the specification (though not - the sample implementation) of PEP 273. The semantics of __path__ are - compatible with those that have been implemented in Jython since - Jython 2.1. - -- PEP 302 has been accepted. Although it was initially developed to - support zipimport, it offers a new, general import hook mechanism. - Several new variables have been added to the sys module: - sys.meta_path, sys.path_hooks, and sys.path_importer_cache; these - make extending the import statement much more convenient than - overriding the __import__ built-in function. For a description of - these, see PEP 302. - -- A frame object's f_lineno attribute can now be written to from a - trace function to change which line will execute next. A command to - exploit this from pdb has been added. [SF patch #643835] - -- The _codecs support module for codecs.py was turned into a built-in - module to assure that at least the built-in codecs are available - to the Python parser for source code decoding according to PEP 263. - -- issubclass now supports a tuple as the second argument, just like - isinstance does. ``issubclass(X, (A, B))`` is equivalent to - ``issubclass(X, A) or issubclass(X, B)``. - -- Thanks to Armin Rigo, the last known way to provoke a system crash - by cleverly arranging for a comparison function to mutate a list - during a list.sort() operation has been fixed. The effect of - attempting to mutate a list, or even to inspect its contents or - length, while a sort is in progress, is not defined by the language. - The C implementation of Python 2.3 attempts to detect mutations, - and raise ValueError if one occurs, but there's no guarantee that - all mutations will be caught, or that any will be caught across - releases or implementations. - -- Unicode file name processing for Windows (PEP 277) is implemented. - All platforms now have an os.path.supports_unicode_filenames attribute, - which is set to True on Windows NT/2000/XP, and False elsewhere. - -- Codec error handling callbacks (PEP 293) are implemented. - Error handling in unicode.encode or str.decode can now be customized. - -- A subtle change to the semantics of the built-in function intern(): - interned strings are no longer immortal. You must keep a reference - to the return value intern() around to get the benefit. - -- Use of 'None' as a variable, argument or attribute name now - issues a SyntaxWarning. In the future, None may become a keyword. - -- SET_LINENO is gone. co_lnotab is now consulted to determine when to - call the trace function. C code that accessed f_lineno should call - PyCode_Addr2Line instead (f_lineno is still there, but only kept up - to date when there is a trace function set). - -- There's a new warning category, FutureWarning. This is used to warn - about a number of situations where the value or sign of an integer - result will change in Python 2.4 as a result of PEP 237 (integer - unification). The warnings implement stage B0 mentioned in that - PEP. The warnings are about the following situations: - - - Octal and hex literals without 'L' prefix in the inclusive range - [0x80000000..0xffffffff]; these are currently negative ints, but - in Python 2.4 they will be positive longs with the same bit - pattern. - - - Left shifts on integer values that cause the outcome to lose - bits or have a different sign than the left operand. To be - precise: x< -*-" in the first - or second line of a Python source file indicates the encoding. - -- list.sort() has a new implementation. While cross-platform results - may vary, and in data-dependent ways, this is much faster on many - kinds of partially ordered lists than the previous implementation, - and reported to be just as fast on randomly ordered lists on - several major platforms. This sort is also stable (if A==B and A - precedes B in the list at the start, A precedes B after the sort too), - although the language definition does not guarantee stability. A - potential drawback is that list.sort() may require temp space of - len(list)*2 bytes (``*4`` on a 64-bit machine). It's therefore possible - for list.sort() to raise MemoryError now, even if a comparison function - does not. See for full details. - -- All standard iterators now ensure that, once StopIteration has been - raised, all future calls to next() on the same iterator will also - raise StopIteration. There used to be various counterexamples to - this behavior, which could have caused confusion or subtle program - breakage, without any benefits. (Note that this is still an - iterator's responsibility; the iterator framework does not enforce - this.) - -- Ctrl+C handling on Windows has been made more consistent with - other platforms. KeyboardInterrupt can now reliably be caught, - and Ctrl+C at an interactive prompt no longer terminates the - process under NT/2k/XP (it never did under Win9x). Ctrl+C will - interrupt time.sleep() in the main thread, and any child processes - created via the popen family (on win2k; we can't make win9x work - reliably) are also interrupted (as generally happens on for Linux/Unix.) - [SF bugs 231273, 439992 and 581232] - -- sys.getwindowsversion() has been added on Windows. This - returns a tuple with information about the version of Windows - currently running. - -- Slices and repetitions of buffer objects now consistently return - a string. Formerly, strings would be returned most of the time, - but a buffer object would be returned when the repetition count - was one or when the slice range was all inclusive. - -- Unicode objects in sys.path are no longer ignored but treated - as directory names. - -- Fixed string.startswith and string.endswith built-in methods - so they accept negative indices. [SF bug 493951] - -- Fixed a bug with a continue inside a try block and a yield in the - finally clause. [SF bug 567538] - -- Most built-in sequences now support "extended slices", i.e. slices - with a third "stride" parameter. For example, "hello world"[::-1] - gives "dlrow olleh". - -- A new warning PendingDeprecationWarning was added to provide - direction on features which are in the process of being deprecated. - The warning will not be printed by default. To see the pending - deprecations, use -Walways::PendingDeprecationWarning:: - as a command line option or warnings.filterwarnings() in code. - -- Deprecated features of xrange objects have been removed as - promised. The start, stop, and step attributes and the tolist() - method no longer exist. xrange repetition and slicing have been - removed. - -- New built-in function enumerate(x), from PEP 279. Example: - enumerate("abc") is an iterator returning (0,"a"), (1,"b"), (2,"c"). - The argument can be an arbitrary iterable object. - -- The assert statement no longer tests __debug__ at runtime. This means - that assert statements cannot be disabled by assigning a false value - to __debug__. - -- A method zfill() was added to str and unicode, that fills a numeric - string to the left with zeros. For example, - "+123".zfill(6) -> "+00123". - -- Complex numbers supported divmod() and the // and % operators, but - these make no sense. Since this was documented, they're being - deprecated now. - -- String and unicode methods lstrip(), rstrip() and strip() now take - an optional argument that specifies the characters to strip. For - example, "Foo!!!?!?!?".rstrip("?!") -> "Foo". - -- There's a new dictionary constructor (a class method of the dict - class), dict.fromkeys(iterable, value=None). It constructs a - dictionary with keys taken from the iterable and all values set to a - single value. It can be used for building sets and for removing - duplicates from sequences. - -- Added a new dict method pop(key). This removes and returns the - value corresponding to key. [SF patch #539949] - -- A new built-in type, bool, has been added, as well as built-in - names for its two values, True and False. Comparisons and sundry - other operations that return a truth value have been changed to - return a bool instead. Read PEP 285 for an explanation of why this - is backward compatible. - -- Fixed two bugs reported as SF #535905: under certain conditions, - deallocating a deeply nested structure could cause a segfault in the - garbage collector, due to interaction with the "trashcan" code; - access to the current frame during destruction of a local variable - could access a pointer to freed memory. - -- The optional object allocator ("pymalloc") has been enabled by - default. The recommended practice for memory allocation and - deallocation has been streamlined. A header file is included, - Misc/pymemcompat.h, which can be bundled with 3rd party extensions - and lets them use the same API with Python versions from 1.5.2 - onwards. - -- PyErr_Display will provide file and line information for all exceptions - that have an attribute print_file_and_line, not just SyntaxErrors. - -- The UTF-8 codec will now encode and decode Unicode surrogates - correctly and without raising exceptions for unpaired ones. - -- Universal newlines (PEP 278) is implemented. Briefly, using 'U' - instead of 'r' when opening a text file for reading changes the line - ending convention so that any of '\r', '\r\n', and '\n' is - recognized (even mixed in one file); all three are converted to - '\n', the standard Python line end character. - -- file.xreadlines() now raises a ValueError if the file is closed: - Previously, an xreadlines object was returned which would raise - a ValueError when the xreadlines.next() method was called. - -- sys.exit() inadvertently allowed more than one argument. - An exception will now be raised if more than one argument is used. - -- Changed evaluation order of dictionary literals to conform to the - general left to right evaluation order rule. Now {f1(): f2()} will - evaluate f1 first. - -- Fixed bug #521782: when a file was in non-blocking mode, file.read() - could silently lose data or wrongly throw an unknown error. - -- The sq_repeat, sq_inplace_repeat, sq_concat and sq_inplace_concat - slots are now always tried after trying the corresponding nb_* slots. - This fixes a number of minor bugs (see bug #624807). - -- Fix problem with dynamic loading on 64-bit AIX (see bug #639945). - -Extension modules ------------------ - -- Added three operators to the operator module: - operator.pow(a,b) which is equivalent to: a**b. - operator.is_(a,b) which is equivalent to: a is b. - operator.is_not(a,b) which is equivalent to: a is not b. - -- posix.openpty now works on all systems that have /dev/ptmx. - -- A module zipimport exists to support importing code from zip - archives. - -- The new datetime module supplies classes for manipulating dates and - times. The basic design came from the Zope "fishbowl process", and - favors practical commercial applications over calendar esoterica. See - - http://www.zope.org/Members/fdrake/DateTimeWiki/FrontPage - -- _tkinter now returns Tcl objects, instead of strings. Objects which - have Python equivalents are converted to Python objects, other objects - are wrapped. This can be configured through the wantobjects method, - or Tkinter.wantobjects. - -- The PyBSDDB wrapper around the Sleepycat Berkeley DB library has - been added as the package bsddb. The traditional bsddb module is - still available in source code, but not built automatically anymore, - and is now named bsddb185. This supports Berkeley DB versions from - 3.0 to 4.1. For help converting your databases from the old module (which - probably used an obsolete version of Berkeley DB) to the new module, see - the db2pickle.py and pickle2db.py scripts described in the Tools/Demos - section above. - -- unicodedata was updated to Unicode 3.2. It supports normalization - and names for Hangul syllables and CJK unified ideographs. - -- resource.getrlimit() now returns longs instead of ints. - -- readline now dynamically adjusts its input/output stream if - sys.stdin/stdout changes. - -- The _tkinter module (and hence Tkinter) has dropped support for - Tcl/Tk 8.0 and 8.1. Only Tcl/Tk versions 8.2, 8.3 and 8.4 are - supported. - -- cPickle.BadPickleGet is now a class. - -- The time stamps in os.stat_result are floating point numbers - after stat_float_times has been called. - -- If the size passed to mmap.mmap() is larger than the length of the - file on non-Windows platforms, a ValueError is raised. [SF bug 585792] - -- The xreadlines module is slated for obsolescence. - -- The strptime function in the time module is now always available (a - Python implementation is used when the C library doesn't define it). - -- The 'new' module is no longer an extension, but a Python module that - only exists for backwards compatibility. Its contents are no longer - functions but callable type objects. - -- The bsddb.*open functions can now take 'None' as a filename. - This will create a temporary in-memory bsddb that won't be - written to disk. - -- posix.getloadavg, posix.lchown, posix.killpg, posix.mknod, and - posix.getpgid have been added where available. - -- The locale module now exposes the C library's gettext interface. It - also has a new function getpreferredencoding. - -- A security hole ("double free") was found in zlib-1.1.3, a popular - third party compression library used by some Python modules. The - hole was quickly plugged in zlib-1.1.4, and the Windows build of - Python now ships with zlib-1.1.4. - -- pwd, grp, and resource return enhanced tuples now, with symbolic - field names. - -- array.array is now a type object. A new format character - 'u' indicates Py_UNICODE arrays. For those, .tounicode and - .fromunicode methods are available. Arrays now support __iadd__ - and __imul__. - -- dl now builds on every system that has dlfcn.h. Failure in case - of sizeof(int)!=sizeof(long)!=sizeof(void*) is delayed until dl.open - is called. - -- The sys module acquired a new attribute, api_version, which evaluates - to the value of the PYTHON_API_VERSION macro with which the - interpreter was compiled. - -- Fixed bug #470582: sre module would return a tuple (None, 'a', 'ab') - when applying the regular expression '^((a)c)?(ab)$' on 'ab'. It now - returns (None, None, 'ab'), as expected. Also fixed handling of - lastindex/lastgroup match attributes in similar cases. For example, - when running the expression r'(a)(b)?b' over 'ab', lastindex must be - 1, not 2. - -- Fixed bug #581080: sre scanner was not checking the buffer limit - before increasing the current pointer. This was creating an infinite - loop in the search function, once the pointer exceeded the buffer - limit. - -- The os.fdopen function now enforces a file mode starting with the - letter 'r', 'w' or 'a', otherwise a ValueError is raised. This fixes - bug #623464. - -- The linuxaudiodev module is now deprecated; it is being replaced by - ossaudiodev. The interface has been extended to cover a lot more of - OSS (see www.opensound.com), including most DSP ioctls and the - OSS mixer API. Documentation forthcoming in 2.3a2. - -Library -------- - -- imaplib.py now supports SSL (Tino Lange and Piers Lauder). - -- Freeze's modulefinder.py has been moved to the standard library; - slightly improved so it will issue less false missing submodule - reports (see sf path #643711 for details). Documentation will follow - with Python 2.3a2. - -- os.path exposes getctime. - -- unittest.py now has two additional methods called assertAlmostEqual() - and failIfAlmostEqual(). They implement an approximate comparison - by rounding the difference between the two arguments and comparing - the result to zero. Approximate comparison is essential for - unit tests of floating point results. - -- calendar.py now depends on the new datetime module rather than - the time module. As a result, the range of allowable dates - has been increased. - -- pdb has a new 'j(ump)' command to select the next line to be - executed. - -- The distutils created windows installers now can run a - postinstallation script. - -- doctest.testmod can now be called without argument, which means to - test the current module. - -- When canceling a server that implemented threading with a keyboard - interrupt, the server would shut down but not terminate (waiting on - client threads). A new member variable, daemon_threads, was added to - the ThreadingMixIn class in SocketServer.py to make it explicit that - this behavior needs to be controlled. - -- A new module, optparse, provides a fancy alternative to getopt for - command line parsing. It is a slightly modified version of Greg - Ward's Optik package. - -- UserDict.py now defines a DictMixin class which defines all dictionary - methods for classes that already have a minimum mapping interface. - This greatly simplifies writing classes that need to be substitutable - for dictionaries (such as the shelve module). - -- shelve.py now subclasses from UserDict.DictMixin. Now shelve supports - all dictionary methods. This eases the transition to persistent - storage for scripts originally written with dictionaries in mind. - -- shelve.open and the various classes in shelve.py now accept an optional - binary flag, which defaults to False. If True, the values stored in the - shelf are binary pickles. - -- A new package, logging, implements the logging API defined by PEP - 282. The code is written by Vinay Sajip. - -- StreamReader, StreamReaderWriter and StreamRecoder in the codecs - modules are iterators now. - -- gzip.py now handles files exceeding 2GB. Files over 4GB also work - now (provided the OS supports it, and Python is configured with large - file support), but in that case the underlying gzip file format can - record only the least-significant 32 bits of the file size, so that - some tools working with gzipped files may report an incorrect file - size. - -- xml.sax.saxutils.unescape has been added, to replace entity references - with their entity value. - -- Queue.Queue.{put,get} now support an optional timeout argument. - -- Various features of Tk 8.4 are exposed in Tkinter.py. The multiple - option of tkFileDialog is exposed as function askopenfile{,name}s. - -- Various configure methods of Tkinter have been stream-lined, so that - tag_configure, image_configure, window_configure now return a - dictionary when invoked with no argument. - -- Importing the readline module now no longer has the side effect of - calling setlocale(LC_CTYPE, ""). The initial "C" locale, or - whatever locale is explicitly set by the user, is preserved. If you - want repr() of 8-bit strings in your preferred encoding to preserve - all printable characters of that encoding, you have to add the - following code to your $PYTHONSTARTUP file or to your application's - main(): - - import locale - locale.setlocale(locale.LC_CTYPE, "") - -- shutil.move was added. shutil.copytree now reports errors as an - exception at the end, instead of printing error messages. - -- Encoding name normalization was generalized to not only - replace hyphens with underscores, but also all other non-alphanumeric - characters (with the exception of the dot which is used for Python - package names during lookup). The aliases.py mapping was updated - to the new standard. - -- mimetypes has two new functions: guess_all_extensions() which - returns a list of all known extensions for a mime type, and - add_type() which adds one mapping between a mime type and - an extension to the database. - -- New module: sets, defines the class Set that implements a mutable - set type using the keys of a dict to represent the set. There's - also a class ImmutableSet which is useful when you need sets of sets - or when you need to use sets as dict keys, and a class BaseSet which - is the base class of the two. - -- Added random.sample(population,k) for random sampling without replacement. - Returns a k length list of unique elements chosen from the population. - -- random.randrange(-sys.maxint-1, sys.maxint) no longer raises - OverflowError. That is, it now accepts any combination of 'start' - and 'stop' arguments so long as each is in the range of Python's - bounded integers. - -- Thanks to Raymond Hettinger, random.random() now uses a new core - generator. The Mersenne Twister algorithm is implemented in C, - threadsafe, faster than the previous generator, has an astronomically - large period (2**19937-1), creates random floats to full 53-bit - precision, and may be the most widely tested random number generator - in existence. - - The random.jumpahead(n) method has different semantics for the new - generator. Instead of jumping n steps ahead, it uses n and the - existing state to create a new state. This means that jumpahead() - continues to support multi-threaded code needing generators of - non-overlapping sequences. However, it will break code which relies - on jumpahead moving a specific number of steps forward. - - The attributes random.whseed and random.__whseed have no meaning for - the new generator. Code using these attributes should switch to a - new class, random.WichmannHill which is provided for backward - compatibility and to make an alternate generator available. - -- New "algorithms" module: heapq, implements a heap queue. Thanks to - Kevin O'Connor for the code and François Pinard for an entertaining - write-up explaining the theory and practical uses of heaps. - -- New encoding for the Palm OS character set: palmos. - -- binascii.crc32() and the zipfile module had problems on some 64-bit - platforms. These have been fixed. On a platform with 8-byte C longs, - crc32() now returns a signed-extended 4-byte result, so that its value - as a Python int is equal to the value computed a 32-bit platform. - -- xml.dom.minidom.toxml and toprettyxml now take an optional encoding - argument. - -- Some fixes in the copy module: when an object is copied through its - __reduce__ method, there was no check for a __setstate__ method on - the result [SF patch 565085]; deepcopy should treat instances of - custom metaclasses the same way it treats instances of type 'type' - [SF patch 560794]. - -- Sockets now support timeout mode. After s.settimeout(T), where T is - a float expressing seconds, subsequent operations raise an exception - if they cannot be completed within T seconds. To disable timeout - mode, use s.settimeout(None). There's also a module function, - socket.setdefaulttimeout(T), which sets the default for all sockets - created henceforth. - -- getopt.gnu_getopt was added. This supports GNU-style option - processing, where options can be mixed with non-option arguments. - -- Stop using strings for exceptions. String objects used for - exceptions are now classes deriving from Exception. The objects - changed were: Tkinter.TclError, bdb.BdbQuit, macpath.norm_error, - tabnanny.NannyNag, and xdrlib.Error. - -- Constants BOM_UTF8, BOM_UTF16, BOM_UTF16_LE, BOM_UTF16_BE, - BOM_UTF32, BOM_UTF32_LE and BOM_UTF32_BE that represent the Byte - Order Mark in UTF-8, UTF-16 and UTF-32 encodings for little and - big endian systems were added to the codecs module. The old names - BOM32_* and BOM64_* were off by a factor of 2. - -- Added conversion functions math.degrees() and math.radians(). - -- math.log() now takes an optional argument: math.log(x[, base]). - -- ftplib.retrlines() now tests for callback is None rather than testing - for False. Was causing an error when given a callback object which - was callable but also returned len() as zero. The change may - create new breakage if the caller relied on the undocumented behavior - and called with callback set to [] or some other False value not - identical to None. - -- random.gauss() uses a piece of hidden state used by nothing else, - and the .seed() and .whseed() methods failed to reset it. In other - words, setting the seed didn't completely determine the sequence of - results produced by random.gauss(). It does now. Programs repeatedly - mixing calls to a seed method with calls to gauss() may see different - results now. - -- The pickle.Pickler class grew a clear_memo() method to mimic that - provided by cPickle.Pickler. - -- difflib's SequenceMatcher class now does a dynamic analysis of - which elements are so frequent as to constitute noise. For - comparing files as sequences of lines, this generally works better - than the IS_LINE_JUNK function, and function ndiff's linejunk - argument defaults to None now as a result. A happy benefit is - that SequenceMatcher may run much faster now when applied - to large files with many duplicate lines (for example, C program - text with lots of repeated "}" and "return NULL;" lines). - -- New Text.dump() method in Tkinter module. - -- New distutils commands for building packagers were added to - support pkgtool on Solaris and swinstall on HP-UX. - -- distutils now has a new abstract binary packager base class - command/bdist_packager, which simplifies writing packagers. - This will hopefully provide the missing bits to encourage - people to submit more packagers, e.g. for Debian, FreeBSD - and other systems. - -- The UTF-16, -LE and -BE stream readers now raise a - NotImplementedError for all calls to .readline(). Previously, they - used to just produce garbage or fail with an encoding error -- - UTF-16 is a 2-byte encoding and the C lib's line reading APIs don't - work well with these. - -- compileall now supports quiet operation. - -- The BaseHTTPServer now implements optional HTTP/1.1 persistent - connections. - -- socket module: the SSL support was broken out of the main - _socket module C helper and placed into a new _ssl helper - which now gets imported by socket.py if available and working. - -- encodings package: added aliases for all supported IANA character - sets - -- ftplib: to safeguard the user's privacy, anonymous login will use - "anonymous@" as default password, rather than the real user and host - name. - -- webbrowser: tightened up the command passed to os.system() so that - arbitrary shell code can't be executed because a bogus URL was - passed in. - -- gettext.translation has an optional fallback argument, and - gettext.find an optional all argument. Translations will now fallback - on a per-message basis. The module supports plural forms, by means - of gettext.[d]ngettext and Translation.[u]ngettext. - -- distutils bdist commands now offer a --skip-build option. - -- warnings.warn now accepts a Warning instance as first argument. - -- The xml.sax.expatreader.ExpatParser class will no longer create - circular references by using itself as the locator that gets passed - to the content handler implementation. [SF bug #535474] - -- The email.Parser.Parser class now properly parses strings regardless - of their line endings, which can be any of \r, \n, or \r\n (CR, LF, - or CRLF). Also, the Header class's constructor default arguments - has changed slightly so that an explicit maxlinelen value is always - honored, and so unicode conversion error handling can be specified. - -- distutils' build_ext command now links C++ extensions with the C++ - compiler available in the Makefile or CXX environment variable, if - running under \*nix. - -- New module bz2: provides a comprehensive interface for the bz2 compression - library. It implements a complete file interface, one-shot (de)compression - functions, and types for sequential (de)compression. - -- New pdb command 'pp' which is like 'p' except that it pretty-prints - the value of its expression argument. - -- Now bdist_rpm distutils command understands a verify_script option in - the config file, including the contents of the referred filename in - the "%verifyscript" section of the rpm spec file. - -- Fixed bug #495695: webbrowser module would run graphic browsers in a - unix environment even if DISPLAY was not set. Also, support for - skipstone browser was included. - -- Fixed bug #636769: rexec would run unallowed code if subclasses of - strings were used as parameters for certain functions. - -Tools/Demos ------------ - -- pygettext.py now supports globbing on Windows, and accepts module - names in addition to accepting file names. - -- The SGI demos (Demo/sgi) have been removed. Nobody thought they - were interesting any more. (The SGI library modules and extensions - are still there; it is believed that at least some of these are - still used and useful.) - -- IDLE supports the new encoding declarations (PEP 263); it can also - deal with legacy 8-bit files if they use the locale's encoding. It - allows non-ASCII strings in the interactive shell and executes them - in the locale's encoding. - -- freeze.py now produces binaries which can import shared modules, - unlike before when this failed due to missing symbol exports in - the generated binary. - -Build ------ - -- On Unix, IDLE is now installed automatically. - -- The fpectl module is not built by default; it's dangerous or useless - except in the hands of experts. - -- The public Python C API will generally be declared using PyAPI_FUNC - and PyAPI_DATA macros, while Python extension module init functions - will be declared with PyMODINIT_FUNC. DL_EXPORT/DL_IMPORT macros - are deprecated. - -- A bug was fixed that could cause COUNT_ALLOCS builds to segfault, or - get into infinite loops, when a new-style class got garbage-collected. - Unfortunately, to avoid this, the way COUNT_ALLOCS works requires - that new-style classes be immortal in COUNT_ALLOCS builds. Note that - COUNT_ALLOCS is not enabled by default, in either release or debug - builds, and that new-style classes are immortal only in COUNT_ALLOCS - builds. - -- Compiling out the cyclic garbage collector is no longer an option. - The old symbol WITH_CYCLE_GC is now ignored, and Python.h arranges - that it's always defined (for the benefit of any extension modules - that may be conditionalizing on it). A bonus is that any extension - type participating in cyclic gc can choose to participate in the - Py_TRASHCAN mechanism now too; in the absence of cyclic gc, this used - to require editing the core to teach the trashcan mechanism about the - new type. - -- According to Annex F of the current C standard, - - The Standard C macro HUGE_VAL and its float and long double analogs, - HUGE_VALF and HUGE_VALL, expand to expressions whose values are - positive infinities. - - Python only uses the double HUGE_VAL, and only to #define its own symbol - Py_HUGE_VAL. Some platforms have incorrect definitions for HUGE_VAL. - pyport.h used to try to worm around that, but the workarounds triggered - other bugs on other platforms, so we gave up. If your platform defines - HUGE_VAL incorrectly, you'll need to #define Py_HUGE_VAL to something - that works on your platform. The only instance of this I'm sure about - is on an unknown subset of Cray systems, described here: - - http://www.cray.com/swpubs/manuals/SN-2194_2.0/html-SN-2194_2.0/x3138.htm - - Presumably 2.3a1 breaks such systems. If anyone uses such a system, help! - -- The configure option --without-doc-strings can be used to remove the - doc strings from the built-in functions and modules; this reduces the - size of the executable. - -- The universal newlines option (PEP 278) is on by default. On Unix - it can be disabled by passing --without-universal-newlines to the - configure script. On other platforms, remove - WITH_UNIVERSAL_NEWLINES from pyconfig.h. - -- On Unix, a shared libpython2.3.so can be created with --enable-shared. - -- All uses of the CACHE_HASH, INTERN_STRINGS, and DONT_SHARE_SHORT_STRINGS - preprocessor symbols were eliminated. The internal decisions they - controlled stopped being experimental long ago. - -- The tools used to build the documentation now work under Cygwin as - well as Unix. - -- The bsddb and dbm module builds have been changed to try and avoid version - skew problems and disable linkage with Berkeley DB 1.85 unless the - installer knows what s/he's doing. See the section on building these - modules in the README file for details. - -C API ------ - -- PyNumber_Check() now returns true for string and unicode objects. - This is a result of these types having a partially defined - tp_as_number slot. (This is not a feature, but an indication that - PyNumber_Check() is not very useful to determine numeric behavior. - It may be deprecated.) - -- The string object's layout has changed: the pointer member - ob_sinterned has been replaced by an int member ob_sstate. On some - platforms (e.g. most 64-bit systems) this may change the offset of - the ob_sval member, so as a precaution the API_VERSION has been - incremented. The apparently unused feature of "indirect interned - strings", supported by the ob_sinterned member, is gone. Interned - strings are now usually mortal; there is a new API, - PyString_InternImmortal() that creates immortal interned strings. - (The ob_sstate member can only take three values; however, while - making it a char saves a few bytes per string object on average, in - it also slowed things down a bit because ob_sval was no longer - aligned.) - -- The Py_InitModule*() functions now accept NULL for the 'methods' - argument. Modules without global functions are becoming more common - now that factories can be types rather than functions. - -- New C API PyUnicode_FromOrdinal() which exposes unichr() at C - level. - -- New functions PyErr_SetExcFromWindowsErr() and - PyErr_SetExcFromWindowsErrWithFilename(). Similar to - PyErr_SetFromWindowsErrWithFilename() and - PyErr_SetFromWindowsErr(), but they allow specifying - the exception type to raise. Available on Windows. - -- Py_FatalError() is now declared as taking a const char* argument. It - was previously declared without const. This should not affect working - code. - -- Added new macro PySequence_ITEM(o, i) that directly calls - sq_item without rechecking that o is a sequence and without - adjusting for negative indices. - -- PyRange_New() now raises ValueError if the fourth argument is not 1. - This is part of the removal of deprecated features of the xrange - object. - -- PyNumber_Coerce() and PyNumber_CoerceEx() now also invoke the type's - coercion if both arguments have the same type but this type has the - CHECKTYPES flag set. This is to better support proxies. - -- The type of tp_free has been changed from "``void (*)(PyObject *)``" to - "``void (*)(void *)``". - -- PyObject_Del, PyObject_GC_Del are now functions instead of macros. - -- A type can now inherit its metatype from its base type. Previously, - when PyType_Ready() was called, if ob_type was found to be NULL, it - was always set to &PyType_Type; now it is set to base->ob_type, - where base is tp_base, defaulting to &PyObject_Type. - -- PyType_Ready() accidentally did not inherit tp_is_gc; now it does. - -- The PyCore_* family of APIs have been removed. - -- The "u#" parser marker will now pass through Unicode objects as-is - without going through the buffer API. - -- The enumerators of cmp_op have been renamed to use the prefix ``PyCmp_``. - -- An old #define of ANY as void has been removed from pyport.h. This - hasn't been used since Python's pre-ANSI days, and the #define has - been marked as obsolete since then. SF bug 495548 says it created - conflicts with other packages, so keeping it around wasn't harmless. - -- Because Python's magic number scheme broke on January 1st, we decided - to stop Python development. Thanks for all the fish! - -- Some of us don't like fish, so we changed Python's magic number - scheme to a new one. See Python/import.c for details. - -New platforms -------------- - -- OpenVMS is now supported. - -- AtheOS is now supported. - -- the EMX runtime environment on OS/2 is now supported. - -- GNU/Hurd is now supported. - -Tests ------ - -- The regrtest.py script's -u option now provides a way to say "allow - all resources except this one." For example, to allow everything - except bsddb, give the option '-uall,-bsddb'. - -Windows -------- - -- The Windows distribution now ships with version 4.0.14 of the - Sleepycat Berkeley database library. This should be a huge - improvement over the previous Berkeley DB 1.85, which had many - bugs. - XXX What are the licensing issues here? - XXX If a user has a database created with a previous version of - XXX Python, what must they do to convert it? - XXX I'm still not sure how to link this thing (see PCbuild/readme.txt). - XXX The version # is likely to change before 2.3a1. - -- The Windows distribution now ships with a Secure Sockets Library (SLL) - module (_ssl.pyd) - -- The Windows distribution now ships with Tcl/Tk version 8.4.1 (it - previously shipped with Tcl/Tk 8.3.2). - -- When Python is built under a Microsoft compiler, sys.version now - includes the compiler version number (_MSC_VER). For example, under - MSVC 6, sys.version contains the substring "MSC v.1200 ". 1200 is - the value of _MSC_VER under MSVC 6. - -- Sometimes the uninstall executable (UNWISE.EXE) vanishes. One cause - of that has been fixed in the installer (disabled Wise's "delete in- - use files" uninstall option). - -- Fixed a bug in urllib's proxy handling in Windows. [SF bug #503031] - -- The installer now installs Start menu shortcuts under (the local - equivalent of) "All Users" when doing an Admin install. - -- file.truncate([newsize]) now works on Windows for all newsize values. - It used to fail if newsize didn't fit in 32 bits, reflecting a - limitation of MS _chsize (which is no longer used). - -- os.waitpid() is now implemented for Windows, and can be used to block - until a specified process exits. This is similar to, but not exactly - the same as, os.waitpid() on POSIX systems. If you're waiting for - a specific process whose pid was obtained from one of the spawn() - functions, the same Python os.waitpid() code works across platforms. - See the docs for details. The docs were changed to clarify that - spawn functions return, and waitpid requires, a process handle on - Windows (not the same thing as a Windows process id). - -- New tempfile.TemporaryFile implementation for Windows: this doesn't - need a TemporaryFileWrapper wrapper anymore, and should be immune - to a nasty problem: before 2.3, if you got a temp file on Windows, it - got wrapped in an object whose close() method first closed the - underlying file, then deleted the file. This usually worked fine. - However, the spawn family of functions on Windows create (at a low C - level) the same set of open files in the spawned process Q as were - open in the spawning process P. If a temp file f was among them, then - doing f.close() in P first closed P's C-level file handle on f, but Q's - C-level file handle on f remained open, so the attempt in P to delete f - blew up with a "Permission denied" error (Windows doesn't allow - deleting open files). This was surprising, subtle, and difficult to - work around. - -- The os module now exports all the symbolic constants usable with the - low-level os.open() on Windows: the new constants in 2.3 are - O_NOINHERIT, O_SHORT_LIVED, O_TEMPORARY, O_RANDOM and O_SEQUENTIAL. - The others were also available in 2.2: O_APPEND, O_BINARY, O_CREAT, - O_EXCL, O_RDONLY, O_RDWR, O_TEXT, O_TRUNC and O_WRONLY. Contrary - to Microsoft docs, O_SHORT_LIVED does not seem to imply O_TEMPORARY - (so specify both if you want both; note that neither is useful unless - specified with O_CREAT too). - -Mac ----- - -- Mac/Relnotes is gone, the release notes are now here. - -- Python (the OSX-only, unix-based version, not the OS9-compatible CFM - version) now fully supports unicode strings as arguments to various file - system calls, eg. open(), file(), os.stat() and os.listdir(). - -- The current naming convention for Python on the Macintosh is that MacPython - refers to the unix-based OSX-only version, and MacPython-OS9 refers to the - CFM-based version that runs on both OS9 and OSX. - -- All MacPython-OS9 functionality is now available in an OSX unix build, - including the Carbon modules, the IDE, OSA support, etc. A lot of this - will only work correctly in a framework build, though, because you cannot - talk to the window manager unless your application is run from a .app - bundle. There is a command line tool "pythonw" that runs your script - with an interpreter living in such a .app bundle, this interpreter should - be used to run any Python script using the window manager (including - Tkinter or wxPython scripts). - -- Most of Mac/Lib has moved to Lib/plat-mac, which is again used both in - MacPython-OSX and MacPython-OS9. The only modules remaining in Mac/Lib - are specifically for MacPython-OS9 (CFM support, preference resources, etc). - -- A new utility PythonLauncher will start a Python interpreter when a .py or - .pyw script is double-clicked in the Finder. By default .py scripts are - run with a normal Python interpreter in a Terminal window and .pyw - files are run with a window-aware pythonw interpreter without a Terminal - window, but all this can be customized. - -- MacPython-OS9 is now Carbon-only, so it runs on Mac OS 9 or Mac OS X and - possibly on Mac OS 8.6 with the right CarbonLib installed, but not on earlier - releases. - -- Many tools such as BuildApplet.py and gensuitemodule.py now support a command - line interface too. - -- All the Carbon classes are now PEP253 compliant, meaning that you can - subclass them from Python. Most of the attributes have gone, you should - now use the accessor function call API, which is also what Apple's - documentation uses. Some attributes such as grafport.visRgn are still - available for convenience. - -- New Carbon modules File (implementing the APIs in Files.h and Aliases.h) - and Folder (APIs from Folders.h). The old macfs built-in module is - gone, and replaced by a Python wrapper around the new modules. - -- Pathname handling should now be fully consistent: MacPython-OSX always uses - unix pathnames and MacPython-OS9 always uses colon-separated Mac pathnames - (also when running on Mac OS X). - -- New Carbon modules Help and AH give access to the Carbon Help Manager. - There are hooks in the IDE to allow accessing the Python documentation - (and Apple's Carbon and Cocoa documentation) through the Help Viewer. - See Mac/OSX/README for converting the Python documentation to a - Help Viewer compatible form and installing it. - -- OSA support has been redesigned and the generated Python classes now - mirror the inheritance defined by the underlying OSA classes. - -- MacPython no longer maps both \r and \n to \n on input for any text file. - This feature has been replaced by universal newline support (PEP278). - -- The default encoding for Python sourcefiles in MacPython-OS9 is no longer - mac-roman (or whatever your local Mac encoding was) but "ascii", like on - other platforms. If you really need sourcefiles with Mac characters in them - you can change this in site.py. - - -What's New in Python 2.2 final? -=============================== - -*Release date: 21-Dec-2001* - -Type/class unification and new-style classes --------------------------------------------- - -- pickle.py, cPickle: allow pickling instances of new-style classes - with a custom metaclass. - -Core and builtins ------------------ - -- weakref proxy object: when comparing, unwrap both arguments if both - are proxies. - -Extension modules ------------------ - -- binascii.b2a_base64(): fix a potential buffer overrun when encoding - very short strings. - -- cPickle: the obscure "fast" mode was suspected of causing stack - overflows on the Mac. Hopefully fixed this by setting the recursion - limit much smaller. If the limit is too low (it only affects - performance), you can change it by defining PY_CPICKLE_FAST_LIMIT - when compiling cPickle.c (or in pyconfig.h). - -Library -------- - -- dumbdbm.py: fixed a dumb old bug (the file didn't get synched at - close or delete time). - -- rfc822.py: fixed a bug where the address '<>' was converted to None - instead of an empty string (also fixes the email.Utils module). - -- xmlrpclib.py: version 1.0.0; uses precision for doubles. - -- test suite: the pickle and cPickle tests were not executing any code - when run from the standard regression test. - -Tools/Demos ------------ - -Build ------ - -C API ------ - -New platforms -------------- - -Tests ------ - -Windows -------- - -- distutils package: fixed broken Windows installers (bdist_wininst). - -- tempfile.py: prevent mysterious warnings when TemporaryFileWrapper - instances are deleted at process exit time. - -- socket.py: prevent mysterious warnings when socket instances are - deleted at process exit time. - -- posixmodule.c: fix a Windows crash with stat() of a filename ending - in backslash. - -Mac ----- - -- The Carbon toolbox modules have been upgraded to Universal Headers - 3.4, and experimental CoreGraphics and CarbonEvents modules have - been added. All only for framework-enabled MacOSX. - - -What's New in Python 2.2c1? -=========================== - -*Release date: 14-Dec-2001* - -Type/class unification and new-style classes --------------------------------------------- - -- Guido's tutorial introduction to the new type/class features has - been extensively updated. See - - http://www.python.org/2.2/descrintro.html - - That remains the primary documentation in this area. - -- Fixed a leak: instance variables declared with __slots__ were never - deleted! - -- The "delete attribute" method of descriptor objects is called - __delete__, not __del__. In previous releases, it was mistakenly - called __del__, which created an unfortunate overloading condition - with finalizers. (The "get attribute" and "set attribute" methods - are still called __get__ and __set__, respectively.) - -- Some subtle issues with the super built-in were fixed: - - (a) When super itself is subclassed, its __get__ method would still - return an instance of the base class (i.e., of super). - - (b) super(C, C()).__class__ would return C rather than super. This - is confusing. To fix this, I decided to change the semantics of - super so that it only applies to code attributes, not to data - attributes. After all, overriding data attributes is not - supported anyway. - - (c) The __get__ method didn't check whether the argument was an - instance of the type used in creation of the super instance. - -- Previously, hash() of an instance of a subclass of a mutable type - (list or dictionary) would return some value, rather than raising - TypeError. This has been fixed. Also, directly calling - dict.__hash__ and list.__hash__ now raises the same TypeError - (previously, these were the same as object.__hash__). - -- New-style objects now support deleting their __dict__. This is for - all intents and purposes equivalent to assigning a brand new empty - dictionary, but saves space if the object is not used further. - -Core and builtins ------------------ - -- -Qnew now works as documented in PEP 238: when -Qnew is passed on - the command line, all occurrences of "/" use true division instead - of classic division. See the PEP for details. Note that "all" - means all instances in library and 3rd-party modules, as well as in - your own code. As the PEP says, -Qnew is intended for use only in - educational environments with control over the libraries in use. - Note that test_coercion.py in the standard Python test suite fails - under -Qnew; this is expected, and won't be repaired until true - division becomes the default (in the meantime, test_coercion is - testing the current rules). - -- complex() now only allows the first argument to be a string - argument, and raises TypeError if either the second arg is a string - or if the second arg is specified when the first is a string. - -Extension modules ------------------ - -- gc.get_referents was renamed to gc.get_referrers. - -Library -------- - -- Functions in the os.spawn() family now release the global interpreter - lock around calling the platform spawn. They should always have done - this, but did not before 2.2c1. Multithreaded programs calling - an os.spawn function with P_WAIT will no longer block all Python threads - until the spawned program completes. It's possible that some programs - relies on blocking, although more likely by accident than by design. - -- webbrowser defaults to netscape.exe on OS/2 now. - -- Tix.ResizeHandle exposes detach_widget, hide, and show. - -- The charset alias windows_1252 has been added. - -- types.StringTypes is a tuple containing the defined string types; - usually this will be (str, unicode), but if Python was compiled - without Unicode support it will be just (str,). - -- The pulldom and minidom modules were synchronized to PyXML. - -Tools/Demos ------------ - -- A new script called Tools/scripts/google.py was added, which fires - off a search on Google. - -Build ------ - -- Note that release builds of Python should arrange to define the - preprocessor symbol NDEBUG on the command line (or equivalent). - In the 2.2 pre-release series we tried to define this by magic in - Python.h instead, but it proved to cause problems for extension - authors. The Unix, Windows and Mac builds now all define NDEBUG in - release builds via cmdline (or equivalent) instead. Ports to - other platforms should do likewise. - -- It is no longer necessary to use --with-suffix when building on a - case-insensitive file system (such as Mac OS X HFS+). In the build - directory an extension is used, but not in the installed python. - -C API ------ - -- New function PyDict_MergeFromSeq2() exposes the built-in dict - constructor's logic for updating a dictionary from an iterable object - producing key-value pairs. - -- PyArg_ParseTupleAndKeywords() requires that the number of entries in - the keyword list equal the number of argument specifiers. This - wasn't checked correctly, and PyArg_ParseTupleAndKeywords could even - dump core in some bad cases. This has been repaired. As a result, - PyArg_ParseTupleAndKeywords may raise RuntimeError in bad cases that - previously went unchallenged. - -New platforms -------------- - -Tests ------ - -Windows -------- - -Mac ----- - -- In unix-Python on Mac OS X (and darwin) sys.platform is now "darwin", - without any trailing digits. - -- Changed logic for finding python home in Mac OS X framework Pythons. - Now sys.executable points to the executable again, in stead of to - the shared library. The latter is used only for locating the python - home. - - -What's New in Python 2.2b2? -=========================== - -*Release date: 16-Nov-2001* - -Type/class unification and new-style classes --------------------------------------------- - -- Multiple inheritance mixing new-style and classic classes in the - list of base classes is now allowed, so this works now: - - class Classic: pass - class Mixed(Classic, object): pass - - The MRO (method resolution order) for each base class is respected - according to its kind, but the MRO for the derived class is computed - using new-style MRO rules if any base class is a new-style class. - This needs to be documented. - -- The new built-in dictionary() constructor, and dictionary type, have - been renamed to dict. This reflects a decade of common usage. - -- dict() now accepts an iterable object producing 2-sequences. For - example, dict(d.items()) == d for any dictionary d. The argument, - and the elements of the argument, can be any iterable objects. - -- New-style classes can now have a __del__ method, which is called - when the instance is deleted (just like for classic classes). - -- Assignment to object.__dict__ is now possible, for objects that are - instances of new-style classes that have a __dict__ (unless the base - class forbids it). - -- Methods of built-in types now properly check for keyword arguments - (formerly these were silently ignored). The only built-in methods - that take keyword arguments are __call__, __init__ and __new__. - -- The socket function has been converted to a type; see below. - -Core and builtins ------------------ - -- Assignment to __debug__ raises SyntaxError at compile-time. This - was promised when 2.1c1 was released as "What's New in Python 2.1c1" - (see below) says. - -- Clarified the error messages for unsupported operands to an operator - (like 1 + ''). - -Extension modules ------------------ - -- mmap has a new keyword argument, "access", allowing a uniform way for - both Windows and Unix users to create read-only, write-through and - copy-on-write memory mappings. This was previously possible only on - Unix. A new keyword argument was required to support this in a - uniform way because the mmap() signatures had diverged across - platforms. Thanks to Jay T Miller for repairing this! - -- By default, the gc.garbage list now contains only those instances in - unreachable cycles that have __del__ methods; in 2.1 it contained all - instances in unreachable cycles. "Instances" here has been generalized - to include instances of both new-style and old-style classes. - -- The socket module defines a new method for socket objects, - sendall(). This is like send() but may make multiple calls to - send() until all data has been sent. Also, the socket function has - been converted to a subclassable type, like list and tuple (etc.) - before it; socket and SocketType are now the same thing. - -- Various bugfixes to the curses module. There is now a test suite - for the curses module (you have to run it manually). - -- binascii.b2a_base64 no longer places an arbitrary restriction of 57 - bytes on its input. - -Library -------- - -- tkFileDialog exposes a Directory class and askdirectory - convenience function. - -- Symbolic group names in regular expressions must be unique. For - example, the regexp r'(?P)(?P)' is not allowed, because a - single name can't mean both "group 1" and "group 2" simultaneously. - Python 2.2 detects this error at regexp compilation time; - previously, the error went undetected, and results were - unpredictable. Also in sre, the pattern.split(), pattern.sub(), and - pattern.subn() methods have been rewritten in C. Also, an - experimental function/method finditer() has been added, which works - like findall() but returns an iterator. - -- Tix exposes more commands through the classes DirSelectBox, - DirSelectDialog, ListNoteBook, Meter, CheckList, and the - methods tix_addbitmapdir, tix_cget, tix_configure, tix_filedialog, - tix_getbitmap, tix_getimage, tix_option_get, and tix_resetoptions. - -- Traceback objects are now scanned by cyclic garbage collection, so - cycles created by casual use of sys.exc_info() no longer cause - permanent memory leaks (provided garbage collection is enabled). - -- mimetypes.py has optional support for non-standard, but commonly - found types. guess_type() and guess_extension() now accept an - optional 'strict' flag, defaulting to true, which controls whether - recognize non-standard types or not. A few non-standard types we - know about have been added. Also, when run as a script, there are - new -l and -e options. - -- statcache is now deprecated. - -- email.Utils.formatdate() now produces the preferred RFC 2822 style - dates with numeric timezones (it used to produce obsolete dates - hard coded to "GMT" timezone). An optional 'localtime' flag is - added to produce dates in the local timezone, with daylight savings - time properly taken into account. - -- In pickle and cPickle, instead of masking errors in load() by - transforming them into SystemError, we let the original exception - propagate out. Also, implement support for __safe_for_unpickling__ - in pickle, as it already was supported in cPickle. - -Tools/Demos ------------ - -Build ------ - -- The dbm module is built using libdb1 if available. The bsddb module - is built with libdb3 if available. - -- Misc/Makefile.pre.in has been removed by BDFL pronouncement. - -C API ------ - -- New function PySequence_Fast_GET_SIZE() returns the size of a non- - NULL result from PySequence_Fast(), more quickly than calling - PySequence_Size(). - -- New argument unpacking function PyArg_UnpackTuple() added. - -- New functions PyObject_CallFunctionObjArgs() and - PyObject_CallMethodObjArgs() have been added to make it more - convenient and efficient to call functions and methods from C. - -- PyArg_ParseTupleAndKeywords() no longer masks errors, so it's - possible that this will propagate errors it didn't before. - -- New function PyObject_CheckReadBuffer(), which returns true if its - argument supports the single-segment readable buffer interface. - -New platforms -------------- - -- We've finally confirmed that this release builds on HP-UX 11.00, - *with* threads, and passes the test suite. - -- Thanks to a series of patches from Michael Muller, Python may build - again under OS/2 Visual Age C++. - -- Updated RISCOS port by Dietmar Schwertberger. - -Tests ------ - -- Added a test script for the curses module. It isn't run automatically; - regrtest.py must be run with '-u curses' to enable it. - -Windows -------- - -Mac ----- - -- PythonScript has been moved to unsupported and is slated to be - removed completely in the next release. - -- It should now be possible to build applets that work on both OS9 and - OSX. - -- The core is now linked with CoreServices not Carbon; as a side - result, default 8bit encoding on OSX is now ASCII. - -- Python should now build on OSX 10.1.1 - - -What's New in Python 2.2b1? -=========================== - -*Release date: 19-Oct-2001* - -Type/class unification and new-style classes --------------------------------------------- - -- New-style classes are now always dynamic (except for built-in and - extension types). There is no longer a performance penalty, and I - no longer see another reason to keep this baggage around. One relic - remains: the __dict__ of a new-style class is a read-only proxy; you - must set the class's attribute to modify it. As a consequence, the - __defined__ attribute of new-style types no longer exists, for lack - of need: there is once again only one __dict__ (although in the - future a __cache__ may be resurrected with a similar function, if I - can prove that it actually speeds things up). - -- C.__doc__ now works as expected for new-style classes (in 2.2a4 it - always returned None, even when there was a class docstring). - -- doctest now finds and runs docstrings attached to new-style classes, - class methods, static methods, and properties. - -Core and builtins ------------------ - -- A very subtle syntactical pitfall in list comprehensions was fixed. - For example: [a+b for a in 'abc', for b in 'def']. The comma in - this example is a mistake. Previously, this would silently let 'a' - iterate over the singleton tuple ('abc',), yielding ['abcd', 'abce', - 'abcf'] rather than the intended ['ad', 'ae', 'af', 'bd', 'be', - 'bf', 'cd', 'ce', 'cf']. Now, this is flagged as a syntax error. - Note that [a for a in ] is a convoluted way to say - [] anyway, so it's not like any expressiveness is lost. - -- getattr(obj, name, default) now only catches AttributeError, as - documented, rather than returning the default value for all - exceptions (which could mask bugs in a __getattr__ hook, for - example). - -- Weak reference objects are now part of the core and offer a C API. - A bug which could allow a core dump when binary operations involved - proxy reference has been fixed. weakref.ReferenceError is now a - built-in exception. - -- unicode(obj) now behaves more like str(obj), accepting arbitrary - objects, and calling a __unicode__ method if it exists. - unicode(obj, encoding) and unicode(obj, encoding, errors) still - require an 8-bit string or character buffer argument. - -- isinstance() now allows any object as the first argument and a - class, a type or something with a __bases__ tuple attribute for the - second argument. The second argument may also be a tuple of a - class, type, or something with __bases__, in which case isinstance() - will return true if the first argument is an instance of any of the - things contained in the second argument tuple. E.g. - - isinstance(x, (A, B)) - - returns true if x is an instance of A or B. - -Extension modules ------------------ - -- thread.start_new_thread() now returns the thread ID (previously None). - -- binascii has now two quopri support functions, a2b_qp and b2a_qp. - -- readline now supports setting the startup_hook and the - pre_event_hook, and adds the add_history() function. - -- os and posix supports chroot(), setgroups() and unsetenv() where - available. The stat(), fstat(), statvfs() and fstatvfs() functions - now return "pseudo-sequences" -- the various fields can now be - accessed as attributes (e.g. os.stat("/").st_mtime) but for - backwards compatibility they also behave as a fixed-length sequence. - Some platform-specific fields (e.g. st_rdev) are only accessible as - attributes. - -- time: localtime(), gmtime() and strptime() now return a - pseudo-sequence similar to the os.stat() return value, with - attributes like tm_year etc. - -- Decompression objects in the zlib module now accept an optional - second parameter to decompress() that specifies the maximum amount - of memory to use for the uncompressed data. - -- optional SSL support in the socket module now exports OpenSSL - functions RAND_add(), RAND_egd(), and RAND_status(). These calls - are useful on platforms like Solaris where OpenSSL does not - automatically seed its PRNG. Also, the keyfile and certfile - arguments to socket.ssl() are now optional. - -- posixmodule (and by extension, the os module on POSIX platforms) now - exports O_LARGEFILE, O_DIRECT, O_DIRECTORY, and O_NOFOLLOW. - -Library -------- - -- doctest now excludes functions and classes not defined by the module - being tested, thanks to Tim Hochberg. - -- HotShot, a new profiler implemented using a C-based callback, has - been added. This substantially reduces the overhead of profiling, - but it is still quite preliminary. Support modules and - documentation will be added in upcoming releases (before 2.2 final). - -- profile now produces correct output in situations where an exception - raised in Python is cleared by C code (e.g. hasattr()). This used - to cause wrong output, including spurious claims of recursive - functions and attribution of time spent to the wrong function. - - The code and documentation for the derived OldProfile and HotProfile - profiling classes was removed. The code hasn't worked for years (if - you tried to use them, they raised exceptions). OldProfile - intended to reproduce the behavior of the profiler Python used more - than 7 years ago, and isn't interesting anymore. HotProfile intended - to provide a faster profiler (but producing less information), and - that's a worthy goal we intend to meet via a different approach (but - without losing information). - -- Profile.calibrate() has a new implementation that should deliver - a much better system-specific calibration constant. The constant can - now be specified in an instance constructor, or as a Profile class or - instance variable, instead of by editing profile.py's source code. - Calibration must still be done manually (see the docs for the profile - module). - - Note that Profile.calibrate() must be overridden by subclasses. - Improving the accuracy required exploiting detailed knowledge of - profiler internals; the earlier method abstracted away the details - and measured a simplified model instead, but consequently computed - a constant too small by a factor of 2 on some modern machines. - -- quopri's encode and decode methods take an optional header parameter, - which indicates whether output is intended for the header 'Q' - encoding. - -- The SocketServer.ThreadingMixIn class now closes the request after - finish_request() returns. (Not when it errors out though.) - -- The nntplib module's NNTP.body() method has grown a 'file' argument - to allow saving the message body to a file. - -- The email package has added a class email.Parser.HeaderParser which - only parses headers and does not recurse into the message's body. - Also, the module/class MIMEAudio has been added for representing - audio data (contributed by Anthony Baxter). - -- ftplib should be able to handle files > 2GB. - -- ConfigParser.getboolean() now also interprets TRUE, FALSE, YES, NO, - ON, and OFF. - -- xml.dom.minidom NodeList objects now support the length attribute - and item() method as required by the DOM specifications. - -Tools/Demos ------------ - -- Demo/dns was removed. It no longer serves any purpose; a package - derived from it is now maintained by Anthony Baxter, see - http://PyDNS.SourceForge.net. - -- The freeze tool has been made more robust, and two new options have - been added: -X and -E. - -Build ------ - -- configure will use CXX in LINKCC if CXX is used to build main() and - the system requires to link a C++ main using the C++ compiler. - -C API ------ - -- The documentation for the tp_compare slot is updated to require that - the return value must be -1, 0, 1; an arbitrary number <0 or >0 is - not correct. This is not yet enforced but will be enforced in - Python 2.3; even later, we may use -2 to indicate errors and +2 for - "NotImplemented". Right now, -1 should be used for an error return. - -- PyLong_AsLongLong() now accepts int (as well as long) arguments. - Consequently, PyArg_ParseTuple's 'L' code also accepts int (as well - as long) arguments. - -- PyThread_start_new_thread() now returns a long int giving the thread - ID, if one can be calculated; it returns -1 for error, 0 if no - thread ID is calculated (this is an incompatible change, but only - the thread module used this API). This code has only really been - tested on Linux and Windows; other platforms please beware (and - report any bugs or strange behavior). - -- PyUnicode_FromEncodedObject() no longer accepts Unicode objects as - input. - -New platforms -------------- - -Tests ------ - -Windows -------- - -- Installer: If you install IDLE, and don't disable file-extension - registration, a new "Edit with IDLE" context (right-click) menu entry - is created for .py and .pyw files. - -- The signal module now supports SIGBREAK on Windows, thanks to Steven - Scott. Note that SIGBREAK is unique to Windows. The default SIGBREAK - action remains to call Win32 ExitProcess(). This can be changed via - signal.signal(). For example:: - - # Make Ctrl+Break raise KeyboardInterrupt, like Python's default Ctrl+C - # (SIGINT) behavior. - import signal - signal.signal(signal.SIGBREAK, signal.default_int_handler) - - try: - while 1: - pass - except KeyboardInterrupt: - # We get here on Ctrl+C or Ctrl+Break now; if we had not changed - # SIGBREAK, only on Ctrl+C (and Ctrl+Break would terminate the - # program without the possibility for any Python-level cleanup). - print "Clean exit" - - -What's New in Python 2.2a4? -=========================== - -*Release date: 28-Sep-2001* - -Type/class unification and new-style classes --------------------------------------------- - -- pydoc and inspect are now aware of new-style classes; - e.g. help(list) at the interactive prompt now shows proper - documentation for all operations on list objects. - -- Applications using Jim Fulton's ExtensionClass module can now safely - be used with Python 2.2. In particular, Zope 2.4.1 now works with - Python 2.2 (as well as with Python 2.1.1). The Demo/metaclass - examples also work again. It is hoped that Gtk and Boost also work - with 2.2a4 and beyond. (If you can confirm this, please write - webmaster@python.org; if there are still problems, please open a bug - report on SourceForge.) - -- property() now takes 4 keyword arguments: fget, fset, fdel and doc. - These map to read-only attributes 'fget', 'fset', 'fdel', and '__doc__' - in the constructed property object. fget, fset and fdel weren't - discoverable from Python in 2.2a3. __doc__ is new, and allows - associating a docstring with a property. - -- Comparison overloading is now more completely implemented. For - example, a str subclass instance can properly be compared to a str - instance, and it can properly overload comparison. Ditto for most - other built-in object types. - -- The repr() of new-style classes has changed; instead of a new-style class is now rendered as , - *except* for built-in types, which are still rendered as (to avoid upsetting existing code that might parse or - otherwise rely on repr() of certain type objects). - -- The repr() of new-style objects is now always ; - previously, it was sometimes . - -- For new-style classes, what was previously called __getattr__ is now - called __getattribute__. This method, if defined, is called for - *every* attribute access. A new __getattr__ hook more similar to the - one in classic classes is defined which is called only if regular - attribute access raises AttributeError; to catch *all* attribute - access, you can use __getattribute__ (for new-style classes). If - both are defined, __getattribute__ is called first, and if it raises - AttributeError, __getattr__ is called. - -- The __class__ attribute of new-style objects can be assigned to. - The new class must have the same C-level object layout as the old - class. - -- The built-in file type can be subclassed now. In the usual pattern, - "file" is the name of the built-in type, and file() is a new built-in - constructor, with the same signature as the built-in open() function. - file() is now the preferred way to open a file. - -- Previously, __new__ would only see sequential arguments passed to - the type in a constructor call; __init__ would see both sequential - and keyword arguments. This made no sense whatsoever any more, so - now both __new__ and __init__ see all arguments. - -- Previously, hash() applied to an instance of a subclass of str or - unicode always returned 0. This has been repaired. - -- Previously, an operation on an instance of a subclass of an - immutable type (int, long, float, complex, tuple, str, unicode), - where the subtype didn't override the operation (and so the - operation was handled by the built-in type), could return that - instance instead a value of the base type. For example, if s was of - a str subclass type, s[:] returned s as-is. Now it returns a str - with the same value as s. - -- Provisional support for pickling new-style objects has been added. - -Core ----- - -- file.writelines() now accepts any iterable object producing strings. - -- PyUnicode_FromEncodedObject() now works very much like - PyObject_Str(obj) in that it tries to use __str__/tp_str - on the object if the object is not a string or buffer. This - makes unicode() behave like str() when applied to non-string/buffer - objects. - -- PyFile_WriteObject now passes Unicode objects to the file's write - method. As a result, all file-like objects which may be the target - of a print statement must support Unicode objects, i.e. they must - at least convert them into ASCII strings. - -- Thread scheduling on Solaris should be improved; it is no longer - necessary to insert a small sleep at the start of a thread in order - to let other runnable threads be scheduled. - -Library -------- - -- StringIO.StringIO instances and cStringIO.StringIO instances support - read character buffer compatible objects for their .write() methods. - These objects are converted to strings and then handled as such - by the instances. - -- The "email" package has been added. This is basically a port of the - mimelib package with API changes - and some implementations updated to use iterators and generators. - -- difflib.ndiff() and difflib.Differ.compare() are generators now. This - restores the ability of Tools/scripts/ndiff.py to start producing output - before the entire comparison is complete. - -- StringIO.StringIO instances and cStringIO.StringIO instances support - iteration just like file objects (i.e. their .readline() method is - called for each iteration until it returns an empty string). - -- The codecs module has grown four new helper APIs to access - built-in codecs: getencoder(), getdecoder(), getreader(), - getwriter(). - -- SimpleXMLRPCServer: a new module (based upon SimpleHTMLServer) - simplifies writing XML RPC servers. - -- os.path.realpath(): a new function that returns the absolute pathname - after interpretation of symbolic links. On non-Unix systems, this - is an alias for os.path.abspath(). - -- operator.indexOf() (PySequence_Index() in the C API) now works with any - iterable object. - -- smtplib now supports various authentication and security features of - the SMTP protocol through the new login() and starttls() methods. - -- hmac: a new module implementing keyed hashing for message - authentication. - -- mimetypes now recognizes more extensions and file types. At the - same time, some mappings not sanctioned by IANA were removed. - -- The "compiler" package has been brought up to date to the state of - Python 2.2 bytecode generation. It has also been promoted from a - Tool to a standard library package. (Tools/compiler still exists as - a sample driver.) - -Build ------ - -- Large file support (LFS) is now automatic when the platform supports - it; no more manual configuration tweaks are needed. On Linux, at - least, it's possible to have a system whose C library supports large - files but whose kernel doesn't; in this case, large file support is - still enabled but doesn't do you any good unless you upgrade your - kernel or share your Python executable with another system whose - kernel has large file support. - -- The configure script now supplies plausible defaults in a - cross-compilation environment. This doesn't mean that the supplied - values are always correct, or that cross-compilation now works - flawlessly -- but it's a first step (and it shuts up most of - autoconf's warnings about AC_TRY_RUN). - -- The Unix build is now a bit less chatty, courtesy of the parser - generator. The build is completely silent (except for errors) when - using "make -s", thanks to a -q option to setup.py. - -C API ------ - -- The "structmember" API now supports some new flag bits to deny read - and/or write access to attributes in restricted execution mode. - -New platforms -------------- - -- Compaq's iPAQ handheld, running the "familiar" Linux distribution - (http://familiar.handhelds.org). - -Tests ------ - -- The "classic" standard tests, which work by comparing stdout to - an expected-output file under Lib/test/output/, no longer stop at - the first mismatch. Instead the test is run to completion, and a - variant of ndiff-style comparison is used to report all differences. - This is much easier to understand than the previous style of reporting. - -- The unittest-based standard tests now use regrtest's test_main() - convention, instead of running as a side-effect of merely being - imported. This allows these tests to be run in more natural and - flexible ways as unittests, outside the regrtest framework. - -- regrtest.py is much better integrated with unittest and doctest now, - especially in regard to reporting errors. - -Windows -------- - -- Large file support now also works for files > 4GB, on filesystems - that support it (NTFS under Windows 2000). See "What's New in - Python 2.2a3" for more detail. - - -What's New in Python 2.2a3? -=========================== - -*Release Date: 07-Sep-2001* - -Core ----- - -- Conversion of long to float now raises OverflowError if the long is too - big to represent as a C double. - -- The 3-argument builtin pow() no longer allows a third non-None argument - if either of the first two arguments is a float, or if both are of - integer types and the second argument is negative (in which latter case - the arguments are converted to float, so this is really the same - restriction). - -- The builtin dir() now returns more information, and sometimes much - more, generally naming all attributes of an object, and all attributes - reachable from the object via its class, and from its class's base - classes, and so on from them too. Example: in 2.2a2, dir([]) returned - an empty list. In 2.2a3, - - >>> dir([]) - ['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', - '__eq__', '__ge__', '__getattr__', '__getitem__', '__getslice__', - '__gt__', '__hash__', '__iadd__', '__imul__', '__init__', '__le__', - '__len__', '__lt__', '__mul__', '__ne__', '__new__', '__repr__', - '__rmul__', '__setattr__', '__setitem__', '__setslice__', '__str__', - 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort'] - - dir(module) continues to return only the module's attributes, though. - -- Overflowing operations on plain ints now return a long int rather - than raising OverflowError. This is a partial implementation of PEP - 237. You can use -Wdefault::OverflowWarning to enable a warning for - this situation, and -Werror::OverflowWarning to revert to the old - OverflowError exception. - -- A new command line option, -Q, is added to control run-time - warnings for the use of classic division. (See PEP 238.) Possible - values are -Qold, -Qwarn, -Qwarnall, and -Qnew. The default is - -Qold, meaning the / operator has its classic meaning and no - warnings are issued. Using -Qwarn issues a run-time warning about - all uses of classic division for int and long arguments; -Qwarnall - also warns about classic division for float and complex arguments - (for use with fixdiv.py). - [Note: the remainder of this item (preserved below) became - obsolete in 2.2c1 -- -Qnew has global effect in 2.2] :: - - Using -Qnew is questionable; it turns on new division by default, but - only in the __main__ module. You can usefully combine -Qwarn or - -Qwarnall and -Qnew: this gives the __main__ module new division, and - warns about classic division everywhere else. - -- Many built-in types can now be subclassed. This applies to int, - long, float, str, unicode, and tuple. (The types complex, list and - dictionary can also be subclassed; this was introduced earlier.) - Note that restrictions apply when subclassing immutable built-in - types: you can only affect the value of the instance by overloading - __new__. You can add mutable attributes, and the subclass instances - will have a __dict__ attribute, but you cannot change the "value" - (as implemented by the base class) of an immutable subclass instance - once it is created. - -- The dictionary constructor now takes an optional argument, a - mapping-like object, and initializes the dictionary from its - (key, value) pairs. - -- A new built-in type, super, has been added. This facilitates making - "cooperative super calls" in a multiple inheritance setting. For an - explanation, see http://www.python.org/2.2/descrintro.html#cooperation - -- A new built-in type, property, has been added. This enables the - creation of "properties". These are attributes implemented by - getter and setter functions (or only one of these for read-only or - write-only attributes), without the need to override __getattr__. - See http://www.python.org/2.2/descrintro.html#property - -- The syntax of floating-point and imaginary literals has been - liberalized, to allow leading zeroes. Examples of literals now - legal that were SyntaxErrors before: - - 00.0 0e3 0100j 07.5 00000000000000000008. - -- An old tokenizer bug allowed floating point literals with an incomplete - exponent, such as 1e and 3.1e-. Such literals now raise SyntaxError. - -Library -------- - -- telnetlib includes symbolic names for the options, and support for - setting an option negotiation callback. It also supports processing - of suboptions. - -- The new C standard no longer requires that math libraries set errno to - ERANGE on overflow. For platform libraries that exploit this new - freedom, Python's overflow-checking was wholly broken. A new overflow- - checking scheme attempts to repair that, but may not be reliable on all - platforms (C doesn't seem to provide anything both useful and portable - in this area anymore). - -- Asynchronous timeout actions are available through the new class - threading.Timer. - -- math.log and math.log10 now return sensible results for even huge - long arguments. For example, math.log10(10 ** 10000) ~= 10000.0. - -- A new function, imp.lock_held(), returns 1 when the import lock is - currently held. See the docs for the imp module. - -- pickle, cPickle and marshal on 32-bit platforms can now correctly read - dumps containing ints written on platforms where Python ints are 8 bytes. - When read on a box where Python ints are 4 bytes, such values are - converted to Python longs. - -- In restricted execution mode (using the rexec module), unmarshalling - code objects is no longer allowed. This plugs a security hole. - -- unittest.TestResult instances no longer store references to tracebacks - generated by test failures. This prevents unexpected dangling references - to objects that should be garbage collected between tests. - -Tools ------ - -- Tools/scripts/fixdiv.py has been added which can be used to fix - division operators as per PEP 238. - -Build ------ - -- If you are an adventurous person using Mac OS X you may want to look at - Mac/OSX. There is a Makefile there that will build Python as a real Mac - application, which can be used for experimenting with Carbon or Cocoa. - Discussion of this on pythonmac-sig, please. - -C API ------ - -- New function PyObject_Dir(obj), like Python __builtin__.dir(obj). - -- Note that PyLong_AsDouble can fail! This has always been true, but no - callers checked for it. It's more likely to fail now, because overflow - errors are properly detected now. The proper way to check:: - - double x = PyLong_AsDouble(some_long_object); - if (x == -1.0 && PyErr_Occurred()) { - /* The conversion failed. */ - } - -- The GC API has been changed. Extensions that use the old API will still - compile but will not participate in GC. To upgrade an extension - module: - - - rename Py_TPFLAGS_GC to PyTPFLAGS_HAVE_GC - - - use PyObject_GC_New or PyObject_GC_NewVar to allocate objects and - PyObject_GC_Del to deallocate them - - - rename PyObject_GC_Init to PyObject_GC_Track and PyObject_GC_Fini - to PyObject_GC_UnTrack - - - remove PyGC_HEAD_SIZE from object size calculations - - - remove calls to PyObject_AS_GC and PyObject_FROM_GC - -- Two new functions: PyString_FromFormat() and PyString_FromFormatV(). - These can be used safely to construct string objects from a - sprintf-style format string (similar to the format string supported - by PyErr_Format()). - -New platforms -------------- - -- Stephen Hansen contributed patches sufficient to get a clean compile - under Borland C (Windows), but he reports problems running it and ran - out of time to complete the port. Volunteers? Expect a MemoryError - when importing the types module; this is probably shallow, and - causing later failures too. - -Tests ------ - -Windows -------- - -- Large file support is now enabled on Win32 platforms as well as on - Win64. This means that, for example, you can use f.tell() and f.seek() - to manipulate files larger than 2 gigabytes (provided you have enough - disk space, and are using a Windows filesystem that supports large - partitions). Windows filesystem limits: FAT has a 2GB (gigabyte) - filesize limit, and large file support makes no difference there. - FAT32's limit is 4GB, and files >= 2GB are easier to use from Python now. - NTFS has no practical limit on file size, and files of any size can be - used from Python now. - -- The w9xpopen hack is now used on Windows NT and 2000 too when COMPSPEC - points to command.com (patch from Brian Quinlan). - - -What's New in Python 2.2a2? -=========================== - -*Release Date: 22-Aug-2001* - -Build ------ - -- Tim Peters developed a brand new Windows installer using Wise 8.1, - generously donated to us by Wise Solutions. - -- configure supports a new option --enable-unicode, with the values - ucs2 and ucs4 (new in 2.2a1). With --disable-unicode, the Unicode - type and supporting code is completely removed from the interpreter. - -- A new configure option --enable-framework builds a Mac OS X framework, - which "make frameworkinstall" will install. This provides a starting - point for more mac-like functionality, join pythonmac-sig@python.org - if you are interested in helping. - -- The NeXT platform is no longer supported. - -- The 'new' module is now statically linked. - -Tools ------ - -- The new Tools/scripts/cleanfuture.py can be used to automatically - edit out obsolete future statements from Python source code. See - the module docstring for details. - -Tests ------ - -- regrtest.py now knows which tests are expected to be skipped on some - platforms, allowing clearer test result output to be given. regrtest - also has optional --use/-u switch to run normally disabled tests - which require network access or consume significant disk resources. - -- Several new tests in the standard test suite, with special thanks to - Nick Mathewson. - -Core ----- - -- The floor division operator // has been added as outlined in PEP - 238. The / operator still provides classic division (and will until - Python 3.0) unless "from __future__ import division" is included, in - which case the / operator will provide true division. The operator - module provides truediv() and floordiv() functions. Augmented - assignment variants are included, as are the equivalent overloadable - methods and C API methods. See the PEP for a full discussion: - - -- Future statements are now effective in simulated interactive shells - (like IDLE). This should "just work" by magic, but read Michael - Hudson's "Future statements in simulated shells" PEP 264 for full - details: . - -- The type/class unification (PEP 252-253) was integrated into the - trunk and is not so tentative any more (the exact specification of - some features is still tentative). A lot of work has done on fixing - bugs and adding robustness and features (performance still has to - come a long way). - -- Warnings about a mismatch in the Python API during extension import - now use the Python warning framework (which makes it possible to - write filters for these warnings). - -- A function's __dict__ (aka func_dict) will now always be a - dictionary. It used to be possible to delete it or set it to None, - but now both actions raise TypeErrors. It is still legal to set it - to a dictionary object. Getting func.__dict__ before any attributes - have been assigned now returns an empty dictionary instead of None. - -- A new command line option, -E, was added which disables the use of - all environment variables, or at least those that are specifically - significant to Python. Usually those have a name starting with - "PYTHON". This was used to fix a problem where the tests fail if - the user happens to have PYTHONHOME or PYTHONPATH pointing to an - older distribution. - -Library -------- - -- New class Differ and new functions ndiff() and restore() in difflib.py. - These package the algorithms used by the popular Tools/scripts/ndiff.py, - for programmatic reuse. - -- New function xml.sax.saxutils.quoteattr(): Quote an XML attribute - value using the minimal quoting required for the value; more - reliable than using xml.sax.saxutils.escape() for attribute values. - -- Readline completion support for cmd.Cmd was added. - -- Calling os.tempnam() or os.tmpnam() generate RuntimeWarnings. - -- Added function threading.BoundedSemaphore() - -- Added Ka-Ping Yee's cgitb.py module. - -- The 'new' module now exposes the CO_xxx flags. - -- The gc module offers the get_referents function. - -New platforms -------------- - -C API ------ - -- Two new APIs PyOS_snprintf() and PyOS_vsnprintf() were added - which provide a cross-platform implementations for the - relatively new snprintf()/vsnprintf() C lib APIs. In contrast to - the standard sprintf() and vsprintf() C lib APIs, these versions - apply bounds checking on the used buffer which enhances protection - against buffer overruns. - -- Unicode APIs now use name mangling to assure that mixing interpreters - and extensions using different Unicode widths is rendered next to - impossible. Trying to import an incompatible Unicode-aware extension - will result in an ImportError. Unicode extensions writers must make - sure to check the Unicode width compatibility in their extensions by - using at least one of the mangled Unicode APIs in the extension. - -- Two new flags METH_NOARGS and METH_O are available in method definition - tables to simplify implementation of methods with no arguments and a - single untyped argument. Calling such methods is more efficient than - calling corresponding METH_VARARGS methods. METH_OLDARGS is now - deprecated. - -Windows -------- - -- "import module" now compiles module.pyw if it exists and nothing else - relevant is found. - - -What's New in Python 2.2a1? -=========================== - -*Release date: 18-Jul-2001* - -Core ----- - -- TENTATIVELY, a large amount of code implementing much of what's - described in PEP 252 (Making Types Look More Like Classes) and PEP - 253 (Subtyping Built-in Types) was added. This will be released - with Python 2.2a1. Documentation will be provided separately - through http://www.python.org/2.2/. The purpose of releasing this - with Python 2.2a1 is to test backwards compatibility. It is - possible, though not likely, that a decision is made not to release - this code as part of 2.2 final, if any serious backwards - incompatibilities are found during alpha testing that cannot be - repaired. - -- Generators were added; this is a new way to create an iterator (see - below) using what looks like a simple function containing one or - more 'yield' statements. See PEP 255. Since this adds a new - keyword to the language, this feature must be enabled by including a - future statement: "from __future__ import generators" (see PEP 236). - Generators will become a standard feature in a future release - (probably 2.3). Without this future statement, 'yield' remains an - ordinary identifier, but a warning is issued each time it is used. - (These warnings currently don't conform to the warnings framework of - PEP 230; we intend to fix this in 2.2a2.) - -- The UTF-16 codec was modified to be more RFC compliant. It will now - only remove BOM characters at the start of the string and then - only if running in native mode (UTF-16-LE and -BE won't remove a - leading BMO character). - -- Strings now have a new method .decode() to complement the already - existing .encode() method. These two methods provide direct access - to the corresponding decoders and encoders of the registered codecs. - - To enhance the usability of the .encode() method, the special - casing of Unicode object return values was dropped (Unicode objects - were auto-magically converted to string using the default encoding). - - Both methods will now return whatever the codec in charge of the - requested encoding returns as object, e.g. Unicode codecs will - return Unicode objects when decoding is requested ("äöü".decode("latin-1") - will return u"äöü"). This enables codec writer to create codecs - for various simple to use conversions. - - New codecs were added to demonstrate these new features (the .encode() - and .decode() columns indicate the type of the returned objects): - - +---------+-----------+-----------+-----------------------------+ - |Name | .encode() | .decode() | Description | - +=========+===========+===========+=============================+ - |uu | string | string | UU codec (e.g. for email) | - +---------+-----------+-----------+-----------------------------+ - |base64 | string | string | base64 codec | - +---------+-----------+-----------+-----------------------------+ - |quopri | string | string | quoted-printable codec | - +---------+-----------+-----------+-----------------------------+ - |zlib | string | string | zlib compression | - +---------+-----------+-----------+-----------------------------+ - |hex | string | string | 2-byte hex codec | - +---------+-----------+-----------+-----------------------------+ - |rot-13 | string | Unicode | ROT-13 Unicode charmap codec| - +---------+-----------+-----------+-----------------------------+ - -- Some operating systems now support the concept of a default Unicode - encoding for file system operations. Notably, Windows supports 'mbcs' - as the default. The Macintosh will also adopt this concept in the medium - term, although the default encoding for that platform will be other than - 'mbcs'. - - On operating system that support non-ASCII filenames, it is common for - functions that return filenames (such as os.listdir()) to return Python - string objects pre-encoded using the default file system encoding for - the platform. As this encoding is likely to be different from Python's - default encoding, converting this name to a Unicode object before passing - it back to the Operating System would result in a Unicode error, as Python - would attempt to use its default encoding (generally ASCII) rather than - the default encoding for the file system. - - In general, this change simply removes surprises when working with - Unicode and the file system, making these operations work as you expect, - increasing the transparency of Unicode objects in this context. - See [????] for more details, including examples. - -- Float (and complex) literals in source code were evaluated to full - precision only when running from a .py file; the same code loaded from a - .pyc (or .pyo) file could suffer numeric differences starting at about the - 12th significant decimal digit. For example, on a machine with IEEE-754 - floating arithmetic, - - x = 9007199254740992.0 - print long(x) - - printed 9007199254740992 if run directly from .py, but 9007199254740000 - if from a compiled (.pyc or .pyo) file. This was due to marshal using - str(float) instead of repr(float) when building code objects. marshal - now uses repr(float) instead, which should reproduce floats to full - machine precision (assuming the platform C float<->string I/O conversion - functions are of good quality). - - This may cause floating-point results to change in some cases, and - usually for the better, but may also cause numerically unstable - algorithms to break. - -- The implementation of dicts suffers fewer collisions, which has speed - benefits. However, the order in which dict entries appear in dict.keys(), - dict.values() and dict.items() may differ from previous releases for a - given dict. Nothing is defined about this order, so no program should - rely on it. Nevertheless, it's easy to write test cases that rely on the - order by accident, typically because of printing the str() or repr() of a - dict to an "expected results" file. See Lib/test/test_support.py's new - sortdict(dict) function for a simple way to display a dict in sorted - order. - -- Many other small changes to dicts were made, resulting in faster - operation along the most common code paths. - -- Dictionary objects now support the "in" operator: "x in dict" means - the same as dict.has_key(x). - -- The update() method of dictionaries now accepts generic mapping - objects. Specifically the argument object must support the .keys() - and __getitem__() methods. This allows you to say, for example, - {}.update(UserDict()) - -- Iterators were added; this is a generalized way of providing values - to a for loop. See PEP 234. There's a new built-in function iter() - to return an iterator. There's a new protocol to get the next value - from an iterator using the next() method (in Python) or the - tp_iternext slot (in C). There's a new protocol to get iterators - using the __iter__() method (in Python) or the tp_iter slot (in C). - Iterating (i.e. a for loop) over a dictionary generates its keys. - Iterating over a file generates its lines. - -- The following functions were generalized to work nicely with iterator - arguments:: - - map(), filter(), reduce(), zip() - list(), tuple() (PySequence_Tuple() and PySequence_Fast() in C API) - max(), min() - join() method of strings - extend() method of lists - 'x in y' and 'x not in y' (PySequence_Contains() in C API) - operator.countOf() (PySequence_Count() in C API) - right-hand side of assignment statements with multiple targets, such as :: - x, y, z = some_iterable_object_returning_exactly_3_values - -- Accessing module attributes is significantly faster (for example, - random.random or os.path or yourPythonModule.yourAttribute). - -- Comparing dictionary objects via == and != is faster, and now works even - if the keys and values don't support comparisons other than ==. - -- Comparing dictionaries in ways other than == and != is slower: there were - insecurities in the dict comparison implementation that could cause Python - to crash if the element comparison routines for the dict keys and/or - values mutated the dicts. Making the code bulletproof slowed it down. - -- Collisions in dicts are resolved via a new approach, which can help - dramatically in bad cases. For example, looking up every key in a dict - d with d.keys() == [i << 16 for i in range(20000)] is approximately 500x - faster now. Thanks to Christian Tismer for pointing out the cause and - the nature of an effective cure (last December! better late than never). - -- repr() is much faster for large containers (dict, list, tuple). - - -Library -------- - -- The constants ascii_letters, ascii_lowercase. and ascii_uppercase - were added to the string module. These a locale-independent - constants, unlike letters, lowercase, and uppercase. These are now - use in appropriate locations in the standard library. - -- The flags used in dlopen calls can now be configured using - sys.setdlopenflags and queried using sys.getdlopenflags. - -- Fredrik Lundh's xmlrpclib is now a standard library module. This - provides full client-side XML-RPC support. In addition, - Demo/xmlrpc/ contains two server frameworks (one SocketServer-based, - one asyncore-based). Thanks to Eric Raymond for the documentation. - -- The xrange() object is simplified: it no longer supports slicing, - repetition, comparisons, efficient 'in' checking, the tolist() - method, or the start, stop and step attributes. See PEP 260. - -- A new function fnmatch.filter to filter lists of file names was added. - -- calendar.py uses month and day names based on the current locale. - -- strop is now *really* obsolete (this was announced before with 1.6), - and issues DeprecationWarning when used (except for the four items - that are still imported into string.py). - -- Cookie.py now sorts key+value pairs by key in output strings. - -- pprint.isrecursive(object) didn't correctly identify recursive objects. - Now it does. - -- pprint functions now much faster for large containers (tuple, list, dict). - -- New 'q' and 'Q' format codes in the struct module, corresponding to C - types "long long" and "unsigned long long" (on Windows, __int64). In - native mode, these can be used only when the platform C compiler supports - these types (when HAVE_LONG_LONG is #define'd by the Python config - process), and then they inherit the sizes and alignments of the C types. - In standard mode, 'q' and 'Q' are supported on all platforms, and are - 8-byte integral types. - -- The site module installs a new built-in function 'help' that invokes - pydoc.help. It must be invoked as 'help()'; when invoked as 'help', - it displays a message reminding the user to use 'help()' or - 'help(object)'. - -Tests ------ - -- New test_mutants.py runs dict comparisons where the key and value - comparison operators mutate the dicts randomly during comparison. This - rapidly causes Python to crash under earlier releases (not for the faint - of heart: it can also cause Win9x to freeze or reboot!). - -- New test_pprint.py verifies that pprint.isrecursive() and - pprint.isreadable() return sensible results. Also verifies that simple - cases produce correct output. - -C API ------ - -- Removed the unused last_is_sticky argument from the internal - _PyTuple_Resize(). If this affects you, you were cheating. - -What's New in Python 2.1 (final)? -================================= - -We only changed a few things since the last release candidate, all in -Python library code: - -- A bug in the locale module was fixed that affected locales which - define no grouping for numeric formatting. - -- A few bugs in the weakref module's implementations of weak - dictionaries (WeakValueDictionary and WeakKeyDictionary) were fixed, - and the test suite was updated to check for these bugs. - -- An old bug in the os.path.walk() function (introduced in Python - 2.0!) was fixed: a non-existent file would cause an exception - instead of being ignored. - -- Fixed a few bugs in the new symtable module found by Neil Norwitz's - PyChecker. - - -What's New in Python 2.1c2? -=========================== - -A flurry of small changes, and one showstopper fixed in the nick of -time made it necessary to release another release candidate. The list -here is the *complete* list of patches (except version updates): - -Core - -- Tim discovered a nasty bug in the dictionary code, caused by - PyDict_Next() calling dict_resize(), and the GC code's use of - PyDict_Next() violating an assumption in dict_items(). This was - fixed with considerable amounts of band-aid, but the net effect is a - saner and more robust implementation. - -- Made a bunch of symbols static that were accidentally global. - -Build and Ports - -- The setup.py script didn't check for a new enough version of zlib - (1.1.3 is needed). Now it does. - -- Changed "make clean" target to also remove shared libraries. - -- Added a more general warning about the SGI Irix optimizer to README. - -Library - -- Fix a bug in urllib.basejoin("http://host", "../file.html") which - omitted the slash between host and file.html. - -- The mailbox module's _Mailbox class contained a completely broken - and undocumented seek() method. Ripped it out. - -- Fixed a bunch of typos in various library modules (urllib2, smtpd, - sgmllib, netrc, chunk) found by Neil Norwitz's PyChecker. - -- Fixed a few last-minute bugs in unittest. - -Extensions - -- Reverted the patch to the OpenSSL code in socketmodule.c to support - RAND_status() and the EGD, and the subsequent patch that tried to - fix it for pre-0.9.5 versions; the problem with the patch is that on - some systems it issues a warning whenever socket is imported, and - that's unacceptable. - -Tests - -- Fixed the pickle tests to work with "import test.test_pickle". - -- Tweaked test_locale.py to actually run the test Windows. - -- In distutils/archive_util.py, call zipfile.ZipFile() with mode "w", - not "wb" (which is not a valid mode at all). - -- Fix pstats browser crashes. Import readline if it exists to make - the user interface nicer. - -- Add "import thread" to the top of test modules that import the - threading module (test_asynchat and test_threadedtempfile). This - prevents test failures caused by a broken threading module resulting - from a previously caught failed import. - -- Changed test_asynchat.py to set the SO_REUSEADDR option; this was - needed on some platforms (e.g. Solaris 8) when the tests are run - twice in succession. - -- Skip rather than fail test_sunaudiodev if no audio device is found. - - -What's New in Python 2.1c1? -=========================== - -This list was significantly updated when 2.1c2 was released; the 2.1c1 -release didn't mention most changes that were actually part of 2.1c1: - -Legal - -- Copyright was assigned to the Python Software Foundation (PSF) and a - PSF license (very similar to the CNRI license) was added. - -- The CNRI copyright notice was updated to include 2001. - -Core - -- After a public outcry, assignment to __debug__ is no longer illegal; - instead, a warning is issued. It will become illegal in 2.2. - -- Fixed a core dump with "%#x" % 0, and changed the semantics so that - "%#x" now always prepends "0x", even if the value is zero. - -- Fixed some nits in the bytecode compiler. - -- Fixed core dumps when calling certain kinds of non-functions. - -- Fixed various core dumps caused by reference count bugs. - -Build and Ports - -- Use INSTALL_SCRIPT to install script files. - -- New port: SCO Unixware 7, by Billy G. Allie. - -- Updated RISCOS port. - -- Updated BeOS port and notes. - -- Various other porting problems resolved. - -Library - -- The TERMIOS and SOCKET modules are now truly obsolete and - unnecessary. Their symbols are incorporated in the termios and - socket modules. - -- Fixed some 64-bit bugs in pickle, cPickle, and struct, and added - better tests for pickling. - -- threading: make Condition.wait() robust against KeyboardInterrupt. - -- zipfile: add support to zipfile to support opening an archive - represented by an open file rather than a file name. Fix bug where - the archive was not properly closed. Fixed a bug in this bugfix - where flush() was called for a read-only file. - -- imputil: added an uninstall() method to the ImportManager. - -- Canvas: fixed bugs in lower() and tkraise() methods. - -- SocketServer: API change (added overridable close_request() method) - so that the TCP server can explicitly close the request. - -- pstats: Eric Raymond added a simple interactive statistics browser, - invoked when the module is run as a script. - -- locale: fixed a problem in format(). - -- webbrowser: made it work when the BROWSER environment variable has a - value like "/usr/bin/netscape". Made it auto-detect Konqueror for - KDE 2. Fixed some other nits. - -- unittest: changes to allow using a different exception than - AssertionError, and added a few more function aliases. Some other - small changes. - -- urllib, urllib2: fixed redirect problems and a coupleof other nits. - -- asynchat: fixed a critical bug in asynchat that slipped through the - 2.1b2 release. Fixed another rare bug. - -- Fix some unqualified except: clauses (always a bad code example). - -XML - -- pyexpat: new API get_version_string(). - -- Fixed some minidom bugs. - -Extensions - -- Fixed a core dump in _weakref. Removed the weakref.mapping() - function (it adds nothing to the API). - -- Rationalized the use of header files in the readline module, to make - it compile (albeit with some warnings) with the very recent readline - 4.2, without breaking for earlier versions. - -- Hopefully fixed a buffering problem in linuxaudiodev. - -- Attempted a fix to make the OpenSSL support in the socket module - work again with pre-0.9.5 versions of OpenSSL. - -Tests - -- Added a test case for asynchat and asyncore. - -- Removed coupling between tests where one test failing could break - another. - -Tools - -- Ping added an interactive help browser to pydoc, fixed some nits - in the rest of the pydoc code, and added some features to his - inspect module. - -- An updated python-mode.el version 4.1 which integrates Ken - Manheimer's pdbtrack.el. This makes debugging Python code via pdb - much nicer in XEmacs and Emacs. When stepping through your program - with pdb, in either the shell window or the *Python* window, the - source file and line will be tracked by an arrow. Very cool! - -- IDLE: syntax warnings in interactive mode are changed into errors. - -- Some improvements to Tools/webchecker (ignore some more URL types, - follow some more links). - -- Brought the Tools/compiler package up to date. - - -What's New in Python 2.1 beta 2? -================================ - -(Unlisted are many fixed bugs, more documentation, etc.) - -Core language, builtins, and interpreter - -- The nested scopes work (enabled by "from __future__ import - nested_scopes") is completed; in particular, the future now extends - into code executed through exec, eval() and execfile(), and into the - interactive interpreter. - -- When calling a base class method (e.g. BaseClass.__init__(self)), - this is now allowed even if self is not strictly spoken a class - instance (e.g. when using metaclasses or the Don Beaudry hook). - -- Slice objects are now comparable but not hashable; this prevents - dict[:] from being accepted but meaningless. - -- Complex division is now calculated using less braindead algorithms. - This doesn't change semantics except it's more likely to give useful - results in extreme cases. Complex repr() now uses full precision - like float repr(). - -- sgmllib.py now calls handle_decl() for simple declarations. - -- It is illegal to assign to the name __debug__, which is set when the - interpreter starts. It is effectively a compile-time constant. - -- A warning will be issued if a global statement for a variable - follows a use or assignment of that variable. - -Standard library - -- unittest.py, a unit testing framework by Steve Purcell (PyUNIT, - inspired by JUnit), is now part of the standard library. You now - have a choice of two testing frameworks: unittest requires you to - write testcases as separate code, doctest gathers them from - docstrings. Both approaches have their advantages and - disadvantages. - -- A new module Tix was added, which wraps the Tix extension library - for Tk. With that module, it is not necessary to statically link - Tix with _tkinter, since Tix will be loaded with Tcl's "package - require" command. See Demo/tix/. - -- tzparse.py is now obsolete. - -- In gzip.py, the seek() and tell() methods are removed -- they were - non-functional anyway, and it's better if callers can test for their - existence with hasattr(). - -Python/C API - -- PyDict_Next(): it is now safe to call PyDict_SetItem() with a key - that's already in the dictionary during a PyDict_Next() iteration. - This used to fail occasionally when a dictionary resize operation - could be triggered that would rehash all the keys. All other - modifications to the dictionary are still off-limits during a - PyDict_Next() iteration! - -- New extended APIs related to passing compiler variables around. - -- New abstract APIs PyObject_IsInstance(), PyObject_IsSubclass() - implement isinstance() and issubclass(). - -- Py_BuildValue() now has a "D" conversion to create a Python complex - number from a Py_complex C value. - -- Extensions types which support weak references must now set the - field allocated for the weak reference machinery to NULL themselves; - this is done to avoid the cost of checking each object for having a - weakly referencable type in PyObject_INIT(), since most types are - not weakly referencable. - -- PyFrame_FastToLocals() and PyFrame_LocalsToFast() copy bindings for - free variables and cell variables to and from the frame's f_locals. - -- Variants of several functions defined in pythonrun.h have been added - to support the nested_scopes future statement. The variants all end - in Flags and take an extra argument, a PyCompilerFlags *; examples: - PyRun_AnyFileExFlags(), PyRun_InteractiveLoopFlags(). These - variants may be removed in Python 2.2, when nested scopes are - mandatory. - -Distutils - -- the sdist command now writes a PKG-INFO file, as described in PEP 241, - into the release tree. - -- several enhancements to the bdist_wininst command from Thomas Heller - (an uninstaller, more customization of the installer's display) - -- from Jack Jansen: added Mac-specific code to generate a dialog for - users to specify the command-line (because providing a command-line with - MacPython is awkward). Jack also made various fixes for the Mac - and the Metrowerks compiler. - -- added 'platforms' and 'keywords' to the set of metadata that can be - specified for a distribution. - -- applied patches from Jason Tishler to make the compiler class work with - Cygwin. - - -What's New in Python 2.1 beta 1? -================================ - -Core language, builtins, and interpreter - -- Following an outcry from the community about the amount of code - broken by the nested scopes feature introduced in 2.1a2, we decided - to make this feature optional, and to wait until Python 2.2 (or at - least 6 months) to make it standard. The option can be enabled on a - per-module basis by adding "from __future__ import nested_scopes" at - the beginning of a module (before any other statements, but after - comments and an optional docstring). See PEP 236 (Back to the - __future__) for a description of the __future__ statement. PEP 227 - (Statically Nested Scopes) has been updated to reflect this change, - and to clarify the semantics in a number of endcases. - -- The nested scopes code, when enabled, has been hardened, and most - bugs and memory leaks in it have been fixed. - -- Compile-time warnings are now generated for a number of conditions - that will break or change in meaning when nested scopes are enabled: - - - Using "from...import *" or "exec" without in-clause in a function - scope that also defines a lambda or nested function with one or - more free (non-local) variables. The presence of the import* or - bare exec makes it impossible for the compiler to determine the - exact set of local variables in the outer scope, which makes it - impossible to determine the bindings for free variables in the - inner scope. To avoid the warning about import *, change it into - an import of explicitly name object, or move the import* statement - to the global scope; to avoid the warning about bare exec, use - exec...in... (a good idea anyway -- there's a possibility that - bare exec will be deprecated in the future). - - - Use of a global variable in a nested scope with the same name as a - local variable in a surrounding scope. This will change in - meaning with nested scopes: the name in the inner scope will - reference the variable in the outer scope rather than the global - of the same name. To avoid the warning, either rename the outer - variable, or use a global statement in the inner function. - -- An optional object allocator has been included. This allocator is - optimized for Python objects and should be faster and use less memory - than the standard system allocator. It is not enabled by default - because of possible thread safety problems. The allocator is only - protected by the Python interpreter lock and it is possible that some - extension modules require a thread safe allocator. The object - allocator can be enabled by providing the "--with-pymalloc" option to - configure. - -Standard library - -- pyexpat now detects the expat version if expat.h defines it. A - number of additional handlers are provided, which are only available - since expat 1.95. In addition, the methods SetParamEntityParsing and - GetInputContext of Parser objects are available with 1.95.x - only. Parser objects now provide the ordered_attributes and - specified_attributes attributes. A new module expat.model was added, - which offers a number of additional constants if 1.95.x is used. - -- xml.dom offers the new functions registerDOMImplementation and - getDOMImplementation. - -- xml.dom.minidom offers a toprettyxml method. A number of DOM - conformance issues have been resolved. In particular, Element now - has a hasAttributes method, and the handling of namespaces was - improved. - -- Ka-Ping Yee contributed two new modules: inspect.py, a module for - getting information about live Python code, and pydoc.py, a module - for interactively converting docstrings to HTML or text. - Tools/scripts/pydoc, which is now automatically installed into - /bin, uses pydoc.py to display documentation; try running - "pydoc -h" for instructions. "pydoc -g" pops up a small GUI that - lets you browse the module docstrings using a web browser. - -- New library module difflib.py, primarily packaging the SequenceMatcher - class at the heart of the popular ndiff.py file-comparison tool. - -- doctest.py (a framework for verifying Python code examples in docstrings) - is now part of the std library. - -Windows changes - -- A new entry in the Start menu, "Module Docs", runs "pydoc -g" -- a - small GUI that lets you browse the module docstrings using your - default web browser. - -- Import is now case-sensitive. PEP 235 (Import on Case-Insensitive - Platforms) is implemented. See - - http://python.sourceforge.net/peps/pep-0235.html - - for full details, especially the "Current Lower-Left Semantics" section. - The new Windows import rules are simpler than before: - - A. If the PYTHONCASEOK environment variable exists, same as - before: silently accept the first case-insensitive match of any - kind; raise ImportError if none found. - - B. Else search sys.path for the first case-sensitive match; raise - ImportError if none found. - - The same rules have been implemented on other platforms with case- - insensitive but case-preserving filesystems too (including Cygwin, and - several flavors of Macintosh operating systems). - -- winsound module: Under Win9x, winsound.Beep() now attempts to simulate - what it's supposed to do (and does do under NT and 2000) via direct - port manipulation. It's unknown whether this will work on all systems, - but it does work on my Win98SE systems now and was known to be useless on - all Win9x systems before. - -- Build: Subproject _test (effectively) renamed to _testcapi. - -New platforms - -- 2.1 should compile and run out of the box under MacOS X, even using HFS+. - Thanks to Steven Majewski! - -- 2.1 should compile and run out of the box on Cygwin. Thanks to Jason - Tishler! - -- 2.1 contains new files and patches for RISCOS, thanks to Dietmar - Schwertberger! See RISCOS/README for more information -- it seems - that because of the bizarre filename conventions on RISCOS, no port - to that platform is easy. - - -What's New in Python 2.1 alpha 2? -================================= - -Core language, builtins, and interpreter - -- Scopes nest. If a name is used in a function or class, but is not - local, the definition in the nearest enclosing function scope will - be used. One consequence of this change is that lambda statements - could reference variables in the namespaces where the lambda is - defined. In some unusual cases, this change will break code. - - In all previous version of Python, names were resolved in exactly - three namespaces -- the local namespace, the global namespace, and - the builtins namespace. According to this old definition, if a - function A is defined within a function B, the names bound in B are - not visible in A. The new rules make names bound in B visible in A, - unless A contains a name binding that hides the binding in B. - - Section 4.1 of the reference manual describes the new scoping rules - in detail. The test script in Lib/test/test_scope.py demonstrates - some of the effects of the change. - - The new rules will cause existing code to break if it defines nested - functions where an outer function has local variables with the same - name as globals or builtins used by the inner function. Example: - - def munge(str): - def helper(x): - return str(x) - if type(str) != type(''): - str = helper(str) - return str.strip() - - Under the old rules, the name str in helper() is bound to the - built-in function str(). Under the new rules, it will be bound to - the argument named str and an error will occur when helper() is - called. - -- The compiler will report a SyntaxError if "from ... import *" occurs - in a function or class scope. The language reference has documented - that this case is illegal, but the compiler never checked for it. - The recent introduction of nested scope makes the meaning of this - form of name binding ambiguous. In a future release, the compiler - may allow this form when there is no possibility of ambiguity. - -- repr(string) is easier to read, now using hex escapes instead of octal, - and using \t, \n and \r instead of \011, \012 and \015 (respectively): - - >>> "\texample \r\n" + chr(0) + chr(255) - '\texample \r\n\x00\xff' # in 2.1 - '\011example \015\012\000\377' # in 2.0 - -- Functions are now compared and hashed by identity, not by value, since - the func_code attribute is writable. - -- Weak references (PEP 205) have been added. This involves a few - changes in the core, an extension module (_weakref), and a Python - module (weakref). The weakref module is the public interface. It - includes support for "explicit" weak references, proxy objects, and - mappings with weakly held values. - -- A 'continue' statement can now appear in a try block within the body - of a loop. It is still not possible to use continue in a finally - clause. - -Standard library - -- mailbox.py now has a new class, PortableUnixMailbox which is - identical to UnixMailbox but uses a more portable scheme for - determining From_ separators. Also, the constructors for all the - classes in this module have a new optional `factory' argument, which - is a callable used when new message classes must be instantiated by - the next() method. - -- random.py is now self-contained, and offers all the functionality of - the now-deprecated whrandom.py. See the docs for details. random.py - also supports new functions getstate() and setstate(), for saving - and restoring the internal state of the generator; and jumpahead(n), - for quickly forcing the internal state to be the same as if n calls to - random() had been made. The latter is particularly useful for multi- - threaded programs, creating one instance of the random.Random() class for - each thread, then using .jumpahead() to force each instance to use a - non-overlapping segment of the full period. - -- random.py's seed() function is new. For bit-for-bit compatibility with - prior releases, use the whseed function instead. The new seed function - addresses two problems: (1) The old function couldn't produce more than - about 2**24 distinct internal states; the new one about 2**45 (the best - that can be done in the Wichmann-Hill generator). (2) The old function - sometimes produced identical internal states when passed distinct - integers, and there was no simple way to predict when that would happen; - the new one guarantees to produce distinct internal states for all - arguments in [0, 27814431486576L). - -- The socket module now supports raw packets on Linux. The socket - family is AF_PACKET. - -- test_capi.py is a start at running tests of the Python C API. The tests - are implemented by the new Modules/_testmodule.c. - -- A new extension module, _symtable, provides provisional access to the - internal symbol table used by the Python compiler. A higher-level - interface will be added on top of _symtable in a future release. - -- Removed the obsolete soundex module. - -- xml.dom.minidom now uses the standard DOM exceptions. Node supports - the isSameNode method; NamedNodeMap the get method. - -- xml.sax.expatreader supports the lexical handler property; it - generates comment, startCDATA, and endCDATA events. - -Windows changes - -- Build procedure: the zlib project is built in a different way that - ensures the zlib header files used can no longer get out of synch with - the zlib binary used. See PCbuild\readme.txt for details. Your old - zlib-related directories can be deleted; you'll need to download fresh - source for zlib and unpack it into a new directory. - -- Build: New subproject _test for the benefit of test_capi.py (see above). - -- Build: New subproject _symtable, for new DLL _symtable.pyd (a nascent - interface to some Python compiler internals). - -- Build: Subproject ucnhash is gone, since the code was folded into the - unicodedata subproject. - -What's New in Python 2.1 alpha 1? -================================= - -Core language, builtins, and interpreter - -- There is a new Unicode companion to the PyObject_Str() API - called PyObject_Unicode(). It behaves in the same way as the - former, but assures that the returned value is a Unicode object - (applying the usual coercion if necessary). - -- The comparison operators support "rich comparison overloading" (PEP - 207). C extension types can provide a rich comparison function in - the new tp_richcompare slot in the type object. The cmp() function - and the C function PyObject_Compare() first try the new rich - comparison operators before trying the old 3-way comparison. There - is also a new C API PyObject_RichCompare() (which also falls back on - the old 3-way comparison, but does not constrain the outcome of the - rich comparison to a Boolean result). - - The rich comparison function takes two objects (at least one of - which is guaranteed to have the type that provided the function) and - an integer indicating the opcode, which can be Py_LT, Py_LE, Py_EQ, - Py_NE, Py_GT, Py_GE (for <, <=, ==, !=, >, >=), and returns a Python - object, which may be NotImplemented (in which case the tp_compare - slot function is used as a fallback, if defined). - - Classes can overload individual comparison operators by defining one - or more of the methods__lt__, __le__, __eq__, __ne__, __gt__, - __ge__. There are no explicit "reflected argument" versions of - these; instead, __lt__ and __gt__ are each other's reflection, - likewise for__le__ and __ge__; __eq__ and __ne__ are their own - reflection (similar at the C level). No other implications are - made; in particular, Python does not assume that == is the Boolean - inverse of !=, or that < is the Boolean inverse of >=. This makes - it possible to define types with partial orderings. - - Classes or types that want to implement (in)equality tests but not - the ordering operators (i.e. unordered types) should implement == - and !=, and raise an error for the ordering operators. - - It is possible to define types whose rich comparison results are not - Boolean; e.g. a matrix type might want to return a matrix of bits - for A < B, giving elementwise comparisons. Such types should ensure - that any interpretation of their value in a Boolean context raises - an exception, e.g. by defining __nonzero__ (or the tp_nonzero slot - at the C level) to always raise an exception. - -- Complex numbers use rich comparisons to define == and != but raise - an exception for <, <=, > and >=. Unfortunately, this also means - that cmp() of two complex numbers raises an exception when the two - numbers differ. Since it is not mathematically meaningful to compare - complex numbers except for equality, I hope that this doesn't break - too much code. - -- The outcome of comparing non-numeric objects of different types is - not defined by the language, other than that it's arbitrary but - consistent (see the Reference Manual). An implementation detail changed - in 2.1a1 such that None now compares less than any other object. Code - relying on this new behavior (like code that relied on the previous - behavior) does so at its own risk. - -- Functions and methods now support getting and setting arbitrarily - named attributes (PEP 232). Functions have a new __dict__ - (a.k.a. func_dict) which hold the function attributes. Methods get - and set attributes on their underlying im_func. It is a TypeError - to set an attribute on a bound method. - -- The xrange() object implementation has been improved so that - xrange(sys.maxint) can be used on 64-bit platforms. There's still a - limitation that in this case len(xrange(sys.maxint)) can't be - calculated, but the common idiom "for i in xrange(sys.maxint)" will - work fine as long as the index i doesn't actually reach 2**31. - (Python uses regular ints for sequence and string indices; fixing - that is much more work.) - -- Two changes to from...import: - - 1) "from M import X" now works even if (after loading module M) - sys.modules['M'] is not a real module; it's basically a getattr() - operation with AttributeError exceptions changed into ImportError. - - 2) "from M import *" now looks for M.__all__ to decide which names to - import; if M.__all__ doesn't exist, it uses M.__dict__.keys() but - filters out names starting with '_' as before. Whether or not - __all__ exists, there's no restriction on the type of M. - -- File objects have a new method, xreadlines(). This is the fastest - way to iterate over all lines in a file: - - for line in file.xreadlines(): - ...do something to line... - - See the xreadlines module (mentioned below) for how to do this for - other file-like objects. - -- Even if you don't use file.xreadlines(), you may expect a speedup on - line-by-line input. The file.readline() method has been optimized - quite a bit in platform-specific ways: on systems (like Linux) that - support flockfile(), getc_unlocked(), and funlockfile(), those are - used by default. On systems (like Windows) without getc_unlocked(), - a complicated (but still thread-safe) method using fgets() is used by - default. - - You can force use of the fgets() method by #define'ing - USE_FGETS_IN_GETLINE at build time (it may be faster than - getc_unlocked()). - - You can force fgets() not to be used by #define'ing - DONT_USE_FGETS_IN_GETLINE (this is the first thing to try if std test - test_bufio.py fails -- and let us know if it does!). - -- In addition, the fileinput module, while still slower than the other - methods on most platforms, has been sped up too, by using - file.readlines(sizehint). - -- Support for run-time warnings has been added, including a new - command line option (-W) to specify the disposition of warnings. - See the description of the warnings module below. - -- Extensive changes have been made to the coercion code. This mostly - affects extension modules (which can now implement mixed-type - numerical operators without having to use coercion), but - occasionally, in boundary cases the coercion semantics have changed - subtly. Since this was a terrible gray area of the language, this - is considered an improvement. Also note that __rcmp__ is no longer - supported -- instead of calling __rcmp__, __cmp__ is called with - reflected arguments. - -- In connection with the coercion changes, a new built-in singleton - object, NotImplemented is defined. This can be returned for - operations that wish to indicate they are not implemented for a - particular combination of arguments. From C, this is - Py_NotImplemented. - -- The interpreter accepts now bytecode files on the command line even - if they do not have a .pyc or .pyo extension. On Linux, after executing - -import imp,sys,string -magic = string.join(["\\x%.2x" % ord(c) for c in imp.get_magic()],"") -reg = ':pyc:M::%s::%s:' % (magic, sys.executable) -open("/proc/sys/fs/binfmt_misc/register","wb").write(reg) - - any byte code file can be used as an executable (i.e. as an argument - to execve(2)). - -- %[xXo] formats of negative Python longs now produce a sign - character. In 1.6 and earlier, they never produced a sign, - and raised an error if the value of the long was too large - to fit in a Python int. In 2.0, they produced a sign if and - only if too large to fit in an int. This was inconsistent - across platforms (because the size of an int varies across - platforms), and inconsistent with hex() and oct(). Example: - - >>> "%x" % -0x42L - '-42' # in 2.1 - 'ffffffbe' # in 2.0 and before, on 32-bit machines - >>> hex(-0x42L) - '-0x42L' # in all versions of Python - - The behavior of %d formats for negative Python longs remains - the same as in 2.0 (although in 1.6 and before, they raised - an error if the long didn't fit in a Python int). - - %u formats don't make sense for Python longs, but are allowed - and treated the same as %d in 2.1. In 2.0, a negative long - formatted via %u produced a sign if and only if too large to - fit in an int. In 1.6 and earlier, a negative long formatted - via %u raised an error if it was too big to fit in an int. - -- Dictionary objects have an odd new method, popitem(). This removes - an arbitrary item from the dictionary and returns it (in the form of - a (key, value) pair). This can be useful for algorithms that use a - dictionary as a bag of "to do" items and repeatedly need to pick one - item. Such algorithms normally end up running in quadratic time; - using popitem() they can usually be made to run in linear time. - -Standard library - -- In the time module, the time argument to the functions strftime, - localtime, gmtime, asctime and ctime is now optional, defaulting to - the current time (in the local timezone). - -- The ftplib module now defaults to passive mode, which is deemed a - more useful default given that clients are often inside firewalls - these days. Note that this could break if ftplib is used to connect - to a *server* that is inside a firewall, from outside; this is - expected to be a very rare situation. To fix that, you can call - ftp.set_pasv(0). - -- The module site now treats .pth files not only for path configuration, - but also supports extensions to the initialization code: Lines starting - with import are executed. - -- There's a new module, warnings, which implements a mechanism for - issuing and filtering warnings. There are some new built-in - exceptions that serve as warning categories, and a new command line - option, -W, to control warnings (e.g. -Wi ignores all warnings, -We - turns warnings into errors). warnings.warn(message[, category]) - issues a warning message; this can also be called from C as - PyErr_Warn(category, message). - -- A new module xreadlines was added. This exports a single factory - function, xreadlines(). The intention is that this code is the - absolutely fastest way to iterate over all lines in an open - file(-like) object: - - import xreadlines - for line in xreadlines.xreadlines(file): - ...do something to line... - - This is equivalent to the previous the speed record holder using - file.readlines(sizehint). Note that if file is a real file object - (as opposed to a file-like object), this is equivalent: - - for line in file.xreadlines(): - ...do something to line... - -- The bisect module has new functions bisect_left, insort_left, - bisect_right and insort_right. The old names bisect and insort - are now aliases for bisect_right and insort_right. XXX_right - and XXX_left methods differ in what happens when the new element - compares equal to one or more elements already in the list: the - XXX_left methods insert to the left, the XXX_right methods to the - right. Code that doesn't care where equal elements end up should - continue to use the old, short names ("bisect" and "insort"). - -- The new curses.panel module wraps the panel library that forms part - of SYSV curses and ncurses. Contributed by Thomas Gellekum. - -- The SocketServer module now sets the allow_reuse_address flag by - default in the TCPServer class. - -- A new function, sys._getframe(), returns the stack frame pointer of - the caller. This is intended only as a building block for - higher-level mechanisms such as string interpolation. - -- The pyexpat module supports a number of new handlers, which are - available only in expat 1.2. If invocation of a callback fails, it - will report an additional frame in the traceback. Parser objects - participate now in garbage collection. If expat reports an unknown - encoding, pyexpat will try to use a Python codec; that works only - for single-byte charsets. The parser type objects is exposed as - XMLParserObject. - -- xml.dom now offers standard definitions for symbolic node type and - exception code constants, and a hierarchy of DOM exceptions. minidom - was adjusted to use them. - -- The conformance of xml.dom.minidom to the DOM specification was - improved. It detects a number of additional error cases; the - previous/next relationship works even when the tree is modified; - Node supports the normalize() method; NamedNodeMap, DocumentType and - DOMImplementation classes were added; Element supports the - hasAttribute and hasAttributeNS methods; and Text supports the splitText - method. - -Build issues - -- For Unix (and Unix-compatible) builds, configuration and building of - extension modules is now greatly automated. Rather than having to - edit the Modules/Setup file to indicate which modules should be - built and where their include files and libraries are, a - distutils-based setup.py script now takes care of building most - extension modules. All extension modules built this way are built - as shared libraries. Only a few modules that must be linked - statically are still listed in the Setup file; you won't need to - edit their configuration. - -- Python should now build out of the box on Cygwin. If it doesn't, - mail to Jason Tishler (jlt63 at users.sourceforge.net). - -- Python now always uses its own (renamed) implementation of getopt() - -- there's too much variation among C library getopt() - implementations. - -- C++ compilers are better supported; the CXX macro is always set to a - C++ compiler if one is found. - -Windows changes - -- select module: By default under Windows, a select() call - can specify no more than 64 sockets. Python now boosts - this Microsoft default to 512. If you need even more than - that, see the MS docs (you'll need to #define FD_SETSIZE - and recompile Python from source). - -- Support for Windows 3.1, DOS and OS/2 is gone. The Lib/dos-8x3 - subdirectory is no more! - - -What's New in Python 2.0? -========================= - -Below is a list of all relevant changes since release 1.6. Older -changes are in the file HISTORY. If you are making the jump directly -from Python 1.5.2 to 2.0, make sure to read the section for 1.6 in the -HISTORY file! Many important changes listed there. - -Alternatively, a good overview of the changes between 1.5.2 and 2.0 is -the document "What's New in Python 2.0" by Kuchling and Moshe Zadka: -http://www.amk.ca/python/2.0/. - ---Guido van Rossum (home page: http://www.pythonlabs.com/~guido/) - -====================================================================== - -What's new in 2.0 (since release candidate 1)? -============================================== - -Standard library - -- The copy_reg module was modified to clarify its intended use: to - register pickle support for extension types, not for classes. - pickle() will raise a TypeError if it is passed a class. - -- Fixed a bug in gettext's "normalize and expand" code that prevented - it from finding an existing .mo file. - -- Restored support for HTTP/0.9 servers in httplib. - -- The math module was changed to stop raising OverflowError in case of - underflow, and return 0 instead in underflow cases. Whether Python - used to raise OverflowError in case of underflow was platform- - dependent (it did when the platform math library set errno to ERANGE - on underflow). - -- Fixed a bug in StringIO that occurred when the file position was not - at the end of the file and write() was called with enough data to - extend past the end of the file. - -- Fixed a bug that caused Tkinter error messages to get lost on - Windows. The bug was fixed by replacing direct use of - interp->result with Tcl_GetStringResult(interp). - -- Fixed bug in urllib2 that caused it to fail when it received an HTTP - redirect response. - -- Several changes were made to distutils: Some debugging code was - removed from util. Fixed the installer used when an external zip - program (like WinZip) is not found; the source code for this - installer is in Misc/distutils. check_lib() was modified to behave - more like AC_CHECK_LIB by add other_libraries() as a parameter. The - test for whether installed modules are on sys.path was changed to - use both normcase() and normpath(). - -- Several minor bugs were fixed in the xml package (the minidom, - pulldom, expatreader, and saxutils modules). - -- The regression test driver (regrtest.py) behavior when invoked with - -l changed: It now reports a count of objects that are recognized as - garbage but not freed by the garbage collector. - -- The regression test for the math module was changed to test - exceptional behavior when the test is run in verbose mode. Python - cannot yet guarantee consistent exception behavior across platforms, - so the exception part of test_math is run only in verbose mode, and - may fail on your platform. - -Internals - -- PyOS_CheckStack() has been disabled on Win64, where it caused - test_sre to fail. - -Build issues - -- Changed compiler flags, so that gcc is always invoked with -Wall and - -Wstrict-prototypes. Users compiling Python with GCC should see - exactly one warning, except if they have passed configure the - --with-pydebug flag. The expected warning is for getopt() in - Modules/main.c. This warning will be fixed for Python 2.1. - -- Fixed configure to add -threads argument during linking on OSF1. - -Tools and other miscellany - -- The compiler in Tools/compiler was updated to support the new - language features introduced in 2.0: extended print statement, list - comprehensions, and augmented assignments. The new compiler should - also be backwards compatible with Python 1.5.2; the compiler will - always generate code for the version of the interpreter it runs - under. - -What's new in 2.0 release candidate 1 (since beta 2)? -===================================================== - -What is release candidate 1? - -We believe that release candidate 1 will fix all known bugs that we -intend to fix for the 2.0 final release. This release should be a bit -more stable than the previous betas. We would like to see even more -widespread testing before the final release, so we are producing this -release candidate. The final release will be exactly the same unless -any show-stopping (or brown bag) bugs are found by testers of the -release candidate. - -All the changes since the last beta release are bug fixes or changes -to support building Python for specific platforms. - -Core language, builtins, and interpreter - -- A bug that caused crashes when __coerce__ was used with augmented - assignment, e.g. +=, was fixed. - -- Raise ZeroDivisionError when raising zero to a negative number, - e.g. 0.0 ** -2.0. Note that math.pow is unrelated to the built-in - power operator and the result of math.pow(0.0, -2.0) will vary by - platform. On Linux, it raises a ValueError. - -- A bug in Unicode string interpolation was fixed that occasionally - caused errors with formats including "%%". For example, the - following expression "%% %s" % u"abc" no longer raises a TypeError. - -- Compilation of deeply nested expressions raises MemoryError instead - of SyntaxError, e.g. eval("[" * 50 + "]" * 50). - -- In 2.0b2 on Windows, the interpreter wrote .pyc files in text mode, - rendering them useless. They are now written in binary mode again. - -Standard library - -- Keyword arguments are now accepted for most pattern and match object - methods in SRE, the standard regular expression engine. - -- In SRE, fixed error with negative lookahead and lookbehind that - manifested itself as a runtime error in patterns like "(? is now included by Python.h (if it - exists). INT_MAX and LONG_MAX will always be defined, even if - is not available. - -- PyFloat_FromString takes a second argument, pend, that was - effectively useless. It is now officially useless but preserved for - backwards compatibility. If the pend argument is not NULL, *pend is - set to NULL. - -- PyObject_GetAttr() and PyObject_SetAttr() now accept Unicode objects - for the attribute name. See note on getattr() above. - -- A few bug fixes to argument processing for Unicode. - PyArg_ParseTupleAndKeywords() now accepts "es#" and "es". - PyArg_Parse() special cases "s#" for Unicode objects; it returns a - pointer to the default encoded string data instead of to the raw - UTF-16. - -- Py_BuildValue accepts B format (for bgen-generated code). - - -Internals - -- On Unix, fix code for finding Python installation directory so that - it works when argv[0] is a relative path. - -- Added a true unicode_internal_encode() function and fixed the - unicode_internal_decode function() to support Unicode objects directly - rather than by generating a copy of the object. - -- Several of the internal Unicode tables are much smaller now, and - the source code should be much friendlier to weaker compilers. - -- In the garbage collector: Fixed bug in collection of tuples. Fixed - bug that caused some instances to be removed from the container set - while they were still live. Fixed parsing in gc.set_debug() for - platforms where sizeof(long) > sizeof(int). - -- Fixed refcount problem in instance deallocation that only occurred - when Py_REF_DEBUG was defined and Py_TRACE_REFS was not. - -- On Windows, getpythonregpath is now protected against null data in - registry key. - -- On Unix, create .pyc/.pyo files with O_EXCL flag to avoid a race - condition. - - -Build and platform-specific issues - -- Better support of GNU Pth via --with-pth configure option. - -- Python/C API now properly exposed to dynamically-loaded extension - modules on Reliant UNIX. - -- Changes for the benefit of SunOS 4.1.4 (really!). mmapmodule.c: - Don't define MS_SYNC to be zero when it is undefined. Added missing - prototypes in posixmodule.c. - -- Improved support for HP-UX build. Threads should now be correctly - configured (on HP-UX 10.20 and 11.00). - -- Fix largefile support on older NetBSD systems and OpenBSD by adding - define for TELL64. - - -Tools and other miscellany - -- ftpmirror: Call to main() is wrapped in if __name__ == "__main__". - -- freeze: The modulefinder now works with 2.0 opcodes. - -- IDLE: - Move hackery of sys.argv until after the Tk instance has been - created, which allows the application-specific Tkinter - initialization to be executed if present; also pass an explicit - className parameter to the Tk() constructor. - - -What's new in 2.0 beta 1? -========================= - -Source Incompatibilities ------------------------- - -None. Note that 1.6 introduced several incompatibilities with 1.5.2, -such as single-argument append(), connect() and bind(), and changes to -str(long) and repr(float). - - -Binary Incompatibilities ------------------------- - -- Third party extensions built for Python 1.5.x or 1.6 cannot be used -with Python 2.0; these extensions will have to be rebuilt for Python -2.0. - -- On Windows, attempting to import a third party extension built for -Python 1.5.x or 1.6 results in an immediate crash; there's not much we -can do about this. Check your PYTHONPATH environment variable! - -- Python bytecode files (*.pyc and *.pyo) are not compatible between -releases. - - -Overview of Changes Since 1.6 ------------------------------ - -There are many new modules (including brand new XML support through -the xml package, and i18n support through the gettext module); a list -of all new modules is included below. Lots of bugs have been fixed. - -The process for making major new changes to the language has changed -since Python 1.6. Enhancements must now be documented by a Python -Enhancement Proposal (PEP) before they can be accepted. - -There are several important syntax enhancements, described in more -detail below: - - - Augmented assignment, e.g. x += 1 - - - List comprehensions, e.g. [x**2 for x in range(10)] - - - Extended import statement, e.g. import Module as Name - - - Extended print statement, e.g. print >> file, "Hello" - -Other important changes: - - - Optional collection of cyclical garbage - -Python Enhancement Proposal (PEP) ---------------------------------- - -PEP stands for Python Enhancement Proposal. A PEP is a design -document providing information to the Python community, or describing -a new feature for Python. The PEP should provide a concise technical -specification of the feature and a rationale for the feature. - -We intend PEPs to be the primary mechanisms for proposing new -features, for collecting community input on an issue, and for -documenting the design decisions that have gone into Python. The PEP -author is responsible for building consensus within the community and -documenting dissenting opinions. - -The PEPs are available at http://python.sourceforge.net/peps/. - -Augmented Assignment --------------------- - -This must have been the most-requested feature of the past years! -Eleven new assignment operators were added: - - += -= *= /= %= **= <<= >>= &= ^= |= - -For example, - - A += B - -is similar to - - A = A + B - -except that A is evaluated only once (relevant when A is something -like dict[index].attr). - -However, if A is a mutable object, A may be modified in place. Thus, -if A is a number or a string, A += B has the same effect as A = A+B -(except A is only evaluated once); but if a is a list, A += B has the -same effect as A.extend(B)! - -Classes and built-in object types can override the new operators in -order to implement the in-place behavior; the not-in-place behavior is -used automatically as a fallback when an object doesn't implement the -in-place behavior. For classes, the method name is derived from the -method name for the corresponding not-in-place operator by inserting -an 'i' in front of the name, e.g. __iadd__ implements in-place -__add__. - -Augmented assignment was implemented by Thomas Wouters. - - -List Comprehensions -------------------- - -This is a flexible new notation for lists whose elements are computed -from another list (or lists). The simplest form is: - - [ for in ] - -For example, [i**2 for i in range(4)] yields the list [0, 1, 4, 9]. -This is more efficient than a for loop with a list.append() call. - -You can also add a condition: - - [ for in if ] - -For example, [w for w in words if w == w.lower()] would yield the list -of words that contain no uppercase characters. This is more efficient -than a for loop with an if statement and a list.append() call. - -You can also have nested for loops and more than one 'if' clause. For -example, here's a function that flattens a sequence of sequences:: - - def flatten(seq): - return [x for subseq in seq for x in subseq] - - flatten([[0], [1,2,3], [4,5], [6,7,8,9], []]) - -This prints - - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -List comprehensions originated as a patch set from Greg Ewing; Skip -Montanaro and Thomas Wouters also contributed. Described by PEP 202. - - -Extended Import Statement -------------------------- - -Many people have asked for a way to import a module under a different -name. This can be accomplished like this: - - import foo - bar = foo - del foo - -but this common idiom gets old quickly. A simple extension of the -import statement now allows this to be written as follows: - - import foo as bar - -There's also a variant for 'from ... import': - - from foo import bar as spam - -This also works with packages; e.g. you can write this: - - import test.regrtest as regrtest - -Note that 'as' is not a new keyword -- it is recognized only in this -context (this is only possible because the syntax for the import -statement doesn't involve expressions). - -Implemented by Thomas Wouters. Described by PEP 221. - - -Extended Print Statement ------------------------- - -Easily the most controversial new feature, this extension to the print -statement adds an option to make the output go to a different file -than the default sys.stdout. - -For example, to write an error message to sys.stderr, you can now -write: - - print >> sys.stderr, "Error: bad dog!" - -As a special feature, if the expression used to indicate the file -evaluates to None, the current value of sys.stdout is used. Thus: - - print >> None, "Hello world" - -is equivalent to - - print "Hello world" - -Design and implementation by Barry Warsaw. Described by PEP 214. - - -Optional Collection of Cyclical Garbage ---------------------------------------- - -Python is now equipped with a garbage collector that can hunt down -cyclical references between Python objects. It's no replacement for -reference counting; in fact, it depends on the reference counts being -correct, and decides that a set of objects belong to a cycle if all -their reference counts can be accounted for from their references to -each other. This devious scheme was first proposed by Eric Tiedemann, -and brought to implementation by Neil Schemenauer. - -There's a module "gc" that lets you control some parameters of the -garbage collection. There's also an option to the configure script -that lets you enable or disable the garbage collection. In 2.0b1, -it's on by default, so that we (hopefully) can collect decent user -experience with this new feature. There are some questions about its -performance. If it proves to be too much of a problem, we'll turn it -off by default in the final 2.0 release. - - -Smaller Changes ---------------- - -A new function zip() was added. zip(seq1, seq2, ...) is equivalent to -map(None, seq1, seq2, ...) when the sequences have the same length; -i.e. zip([1,2,3], [10,20,30]) returns [(1,10), (2,20), (3,30)]. When -the lists are not all the same length, the shortest list wins: -zip([1,2,3], [10,20]) returns [(1,10), (2,20)]. See PEP 201. - -sys.version_info is a tuple (major, minor, micro, level, serial). - -Dictionaries have an odd new method, setdefault(key, default). -dict.setdefault(key, default) returns dict[key] if it exists; if not, -it sets dict[key] to default and returns that value. Thus: - - dict.setdefault(key, []).append(item) - -does the same work as this common idiom: - - if not dict.has_key(key): - dict[key] = [] - dict[key].append(item) - -There are two new variants of SyntaxError that are raised for -indentation-related errors: IndentationError and TabError. - -Changed \x to consume exactly two hex digits; see PEP 223. Added \U -escape that consumes exactly eight hex digits. - -The limits on the size of expressions and file in Python source code -have been raised from 2**16 to 2**32. Previous versions of Python -were limited because the maximum argument size the Python VM accepted -was 2**16. This limited the size of object constructor expressions, -e.g. [1,2,3] or {'a':1, 'b':2}, and the size of source files. This -limit was raised thanks to a patch by Charles Waldman that effectively -fixes the problem. It is now much more likely that you will be -limited by available memory than by an arbitrary limit in Python. - -The interpreter's maximum recursion depth can be modified by Python -programs using sys.getrecursionlimit and sys.setrecursionlimit. This -limit is the maximum number of recursive calls that can be made by -Python code. The limit exists to prevent infinite recursion from -overflowing the C stack and causing a core dump. The default value is -1000. The maximum safe value for a particular platform can be found -by running Tools/scripts/find_recursionlimit.py. - -New Modules and Packages ------------------------- - -atexit - for registering functions to be called when Python exits. - -imputil - Greg Stein's alternative API for writing custom import -hooks. - -pyexpat - an interface to the Expat XML parser, contributed by Paul -Prescod. - -xml - a new package with XML support code organized (so far) in three -subpackages: xml.dom, xml.sax, and xml.parsers. Describing these -would fill a volume. There's a special feature whereby a -user-installed package named _xmlplus overrides the standard -xmlpackage; this is intended to give the XML SIG a hook to distribute -backwards-compatible updates to the standard xml package. - -webbrowser - a platform-independent API to launch a web browser. - - -Changed Modules ---------------- - -array -- new methods for array objects: count, extend, index, pop, and -remove - -binascii -- new functions b2a_hex and a2b_hex that convert between -binary data and its hex representation - -calendar -- Many new functions that support features including control -over which day of the week is the first day, returning strings instead -of printing them. Also new symbolic constants for days of week, -e.g. MONDAY, ..., SUNDAY. - -cgi -- FieldStorage objects have a getvalue method that works like a -dictionary's get method and returns the value attribute of the object. - -ConfigParser -- The parser object has new methods has_option, -remove_section, remove_option, set, and write. They allow the module -to be used for writing config files as well as reading them. - -ftplib -- ntransfercmd(), transfercmd(), and retrbinary() all now -optionally support the RFC 959 REST command. - -gzip -- readline and readlines now accept optional size arguments - -httplib -- New interfaces and support for HTTP/1.1 by Greg Stein. See -the module doc strings for details. - -locale -- implement getdefaultlocale for Win32 and Macintosh - -marshal -- no longer dumps core when marshaling deeply nested or -recursive data structures - -os -- new functions isatty, seteuid, setegid, setreuid, setregid - -os/popen2 -- popen2/popen3/popen4 support under Windows. popen2/popen3 -support under Unix. - -os/pty -- support for openpty and forkpty - -os.path -- fix semantics of os.path.commonprefix - -smtplib -- support for sending very long messages - -socket -- new function getfqdn() - -readline -- new functions to read, write and truncate history files. -The readline section of the library reference manual contains an -example. - -select -- add interface to poll system call - -shutil -- new copyfileobj function - -SimpleHTTPServer, CGIHTTPServer -- Fix problems with buffering in the -HTTP server. - -Tkinter -- optimization of function flatten - -urllib -- scans environment variables for proxy configuration, -e.g. http_proxy. - -whichdb -- recognizes dumbdbm format - - -Obsolete Modules ----------------- - -None. However note that 1.6 made a whole slew of modules obsolete: -stdwin, soundex, cml, cmpcache, dircache, dump, find, grep, packmail, -poly, zmod, strop, util, whatsound. - - -Changed, New, Obsolete Tools ----------------------------- - -None. - - -C-level Changes ---------------- - -Several cleanup jobs were carried out throughout the source code. - -All C code was converted to ANSI C; we got rid of all uses of the -Py_PROTO() macro, which makes the header files a lot more readable. - -Most of the portability hacks were moved to a new header file, -pyport.h; several other new header files were added and some old -header files were removed, in an attempt to create a more rational set -of header files. (Few of these ever need to be included explicitly; -they are all included by Python.h.) - -Trent Mick ensured portability to 64-bit platforms, under both Linux -and Win64, especially for the new Intel Itanium processor. Mick also -added large file support for Linux64 and Win64. - -The C APIs to return an object's size have been update to consistently -use the form PyXXX_Size, e.g. PySequence_Size and PyDict_Size. In -previous versions, the abstract interfaces used PyXXX_Length and the -concrete interfaces used PyXXX_Size. The old names, -e.g. PyObject_Length, are still available for backwards compatibility -at the API level, but are deprecated. - -The PyOS_CheckStack function has been implemented on Windows by -Fredrik Lundh. It prevents Python from failing with a stack overflow -on Windows. - -The GC changes resulted in creation of two new slots on object, -tp_traverse and tp_clear. The augmented assignment changes result in -the creation of a new slot for each in-place operator. - -The GC API creates new requirements for container types implemented in -C extension modules. See Include/objimpl.h for details. - -PyErr_Format has been updated to automatically calculate the size of -the buffer needed to hold the formatted result string. This change -prevents crashes caused by programmer error. - -New C API calls: PyObject_AsFileDescriptor, PyErr_WriteUnraisable. - -PyRun_AnyFileEx, PyRun_SimpleFileEx, PyRun_FileEx -- New functions -that are the same as their non-Ex counterparts except they take an -extra flag argument that tells them to close the file when done. - -XXX There were other API changes that should be fleshed out here. - - -Windows Changes ---------------- - -New popen2/popen3/peopen4 in os module (see Changed Modules above). - -os.popen is much more usable on Windows 95 and 98. See Microsoft -Knowledge Base article Q150956. The Win9x workaround described there -is implemented by the new w9xpopen.exe helper in the root of your -Python installation. Note that Python uses this internally; it is not -a standalone program. - -Administrator privileges are no longer required to install Python -on Windows NT or Windows 2000. If you have administrator privileges, -Python's registry info will be written under HKEY_LOCAL_MACHINE. -Otherwise the installer backs off to writing Python's registry info -under HKEY_CURRENT_USER. The latter is sufficient for all "normal" -uses of Python, but will prevent some advanced uses from working -(for example, running a Python script as an NT service, or possibly -from CGI). - -[This was new in 1.6] The installer no longer runs a separate Tcl/Tk -installer; instead, it installs the needed Tcl/Tk files directly in the -Python directory. If you already have a Tcl/Tk installation, this -wastes some disk space (about 4 Megs) but avoids problems with -conflicting Tcl/Tk installations, and makes it much easier for Python -to ensure that Tcl/Tk can find all its files. - -[This was new in 1.6] The Windows installer now installs by default in -\Python20\ on the default volume, instead of \Program Files\Python-2.0\. - - -Updates to the changes between 1.5.2 and 1.6 --------------------------------------------- - -The 1.6 NEWS file can't be changed after the release is done, so here -is some late-breaking news: - -New APIs in locale.py: normalize(), getdefaultlocale(), resetlocale(), -and changes to getlocale() and setlocale(). - -The new module is now enabled per default. - -It is not true that the encodings codecs cannot be used for normal -strings: the string.encode() (which is also present on 8-bit strings -!) allows using them for 8-bit strings too, e.g. to convert files from -cp1252 (Windows) to latin-1 or vice-versa. - -Japanese codecs are available from Tamito KAJIYAMA: -http://pseudo.grad.sccs.chukyo-u.ac.jp/~kajiyama/python/ - - -====================================================================== - - -======================================= -==> Release 1.6 (September 5, 2000) <== -======================================= - -What's new in release 1.6? -========================== - -Below is a list of all relevant changes since release 1.5.2. - - -Source Incompatibilities ------------------------- - -Several small incompatible library changes may trip you up: - - - The append() method for lists can no longer be invoked with more - than one argument. This used to append a single tuple made out of - all arguments, but was undocumented. To append a tuple, use - e.g. l.append((a, b, c)). - - - The connect(), connect_ex() and bind() methods for sockets require - exactly one argument. Previously, you could call s.connect(host, - port), but this was undocumented. You must now write - s.connect((host, port)). - - - The str() and repr() functions are now different more often. For - long integers, str() no longer appends a 'L'. Thus, str(1L) == '1', - which used to be '1L'; repr(1L) is unchanged and still returns '1L'. - For floats, repr() now gives 17 digits of precision, to ensure no - precision is lost (on all current hardware). - - - The -X option is gone. Built-in exceptions are now always - classes. Many more library modules also have been converted to - class-based exceptions. - - -Binary Incompatibilities ------------------------- - -- Third party extensions built for Python 1.5.x cannot be used with -Python 1.6; these extensions will have to be rebuilt for Python 1.6. - -- On Windows, attempting to import a third party extension built for -Python 1.5.x results in an immediate crash; there's not much we can do -about this. Check your PYTHONPATH environment variable! - - -Overview of Changes since 1.5.2 -------------------------------- - -For this overview, I have borrowed from the document "What's New in -Python 2.0" by Andrew Kuchling and Moshe Zadka: -http://www.amk.ca/python/2.0/ . - -There are lots of new modules and lots of bugs have been fixed. A -list of all new modules is included below. - -Probably the most pervasive change is the addition of Unicode support. -We've added a new fundamental datatype, the Unicode string, a new -built-in function unicode(), and numerous C APIs to deal with Unicode -and encodings. See the file Misc/unicode.txt for details, or -http://starship.python.net/crew/lemburg/unicode-proposal.txt. - -Two other big changes, related to the Unicode support, are the -addition of string methods and (yet another) new regular expression -engine. - - - String methods mean that you can now say s.lower() etc. instead of - importing the string module and saying string.lower(s) etc. One - peculiarity is that the equivalent of string.join(sequence, - delimiter) is delimiter.join(sequence). Use " ".join(sequence) for - the effect of string.join(sequence); to make this more readable, try - space=" " first. Note that the maxsplit argument defaults in - split() and replace() have changed from 0 to -1. - - - The new regular expression engine, SRE by Fredrik Lundh, is fully - backwards compatible with the old engine, and is in fact invoked - using the same interface (the "re" module). You can explicitly - invoke the old engine by import pre, or the SRE engine by importing - sre. SRE is faster than pre, and supports Unicode (which was the - main reason to put effort in yet another new regular expression - engine -- this is at least the fourth!). - - -Other Changes -------------- - -Other changes that won't break code but are nice to know about: - -Deleting objects is now safe even for deeply nested data structures. - -Long/int unifications: long integers can be used in seek() calls, as -slice indexes. - -String formatting (s % args) has a new formatting option, '%r', which -acts like '%s' but inserts repr(arg) instead of str(arg). (Not yet in -alpha 1.) - -Greg Ward's "distutils" package is included: this will make -installing, building and distributing third party packages much -simpler. - -There's now special syntax that you can use instead of the apply() -function. f(*args, **kwds) is equivalent to apply(f, args, kwds). -You can also use variations f(a1, a2, *args, **kwds) and you can leave -one or the other out: f(*args), f(**kwds). - -The built-ins int() and long() take an optional second argument to -indicate the conversion base -- of course only if the first argument -is a string. This makes string.atoi() and string.atol() obsolete. -(string.atof() was already obsolete). - -When a local variable is known to the compiler but undefined when -used, a new exception UnboundLocalError is raised. This is a class -derived from NameError so code catching NameError should still work. -The purpose is to provide better diagnostics in the following example: - x = 1 - def f(): - print x - x = x+1 -This used to raise a NameError on the print statement, which confused -even experienced Python programmers (especially if there are several -hundreds of lines of code between the reference and the assignment to -x :-). - -You can now override the 'in' operator by defining a __contains__ -method. Note that it has its arguments backwards: x in a causes -a.__contains__(x) to be called. That's why the name isn't __in__. - -The exception AttributeError will have a more friendly error message, -e.g.: 'Spam' instance has no attribute 'eggs'. This may -break code that expects the message to be exactly the attribute -name. - - -New Modules in 1.6 ------------------- - -UserString - base class for deriving from the string type. - -distutils - tools for distributing Python modules. - -robotparser - parse a robots.txt file, for writing web spiders. -(Moved from Tools/webchecker/.) - -linuxaudiodev - audio for Linux. - -mmap - treat a file as a memory buffer. (Windows and Unix.) - -sre - regular expressions (fast, supports unicode). Currently, this -code is very rough. Eventually, the re module will be reimplemented -using sre (without changes to the re API). - -filecmp - supersedes the old cmp.py and dircmp.py modules. - -tabnanny - check Python sources for tab-width dependance. (Moved from -Tools/scripts/.) - -urllib2 - new and improved but incompatible version of urllib (still -experimental). - -zipfile - read and write zip archives. - -codecs - support for Unicode encoders/decoders. - -unicodedata - provides access to the Unicode 3.0 database. - -_winreg - Windows registry access. - -encodings - package which provides a large set of standard codecs -- -currently only for the new Unicode support. It has a drop-in extension -mechanism which allows you to add new codecs by simply copying them -into the encodings package directory. Asian codec support will -probably be made available as separate distribution package built upon -this technique and the new distutils package. - - -Changed Modules ---------------- - -readline, ConfigParser, cgi, calendar, posix, readline, xmllib, aifc, -chunk, wave, random, shelve, nntplib - minor enhancements. - -socket, httplib, urllib - optional OpenSSL support (Unix only). - -_tkinter - support for 8.0 up to 8.3. Support for versions older than -8.0 has been dropped. - -string - most of this module is deprecated now that strings have -methods. This no longer uses the built-in strop module, but takes -advantage of the new string methods to provide transparent support for -both Unicode and ordinary strings. - - -Changes on Windows ------------------- - -The installer no longer runs a separate Tcl/Tk installer; instead, it -installs the needed Tcl/Tk files directly in the Python directory. If -you already have a Tcl/Tk installation, this wastes some disk space -(about 4 Megs) but avoids problems with conflincting Tcl/Tk -installations, and makes it much easier for Python to ensure that -Tcl/Tk can find all its files. Note: the alpha installers don't -include the documentation. - -The Windows installer now installs by default in \Python16\ on the -default volume, instead of \Program Files\Python-1.6\. - - -Changed Tools -------------- - -IDLE - complete overhaul. See the IDLE home -page for more information. (Python 1.6 alpha 1 will come with -IDLE 0.6.) - -Tools/i18n/pygettext.py - Python equivalent of xgettext(1). A message -text extraction tool used for internationalizing applications written -in Python. - - -Obsolete Modules ----------------- - -stdwin and everything that uses it. (Get Python 1.5.2 if you need -it. :-) - -soundex. (Skip Montanaro has a version in Python but it won't be -included in the Python release.) - -cmp, cmpcache, dircmp. (Replaced by filecmp.) - -dump. (Use pickle.) - -find. (Easily coded using os.walk().) - -grep. (Not very useful as a library module.) - -packmail. (No longer has any use.) - -poly, zmod. (These were poor examples at best.) - -strop. (No longer needed by the string module.) - -util. (This functionality was long ago built in elsewhere). - -whatsound. (Use sndhdr.) - - -Detailed Changes from 1.6b1 to 1.6 ----------------------------------- - -- Slight changes to the CNRI license. A copyright notice has been -added; the requirement to indicate the nature of modifications now -applies when making a derivative work available "to others" instead of -just "to the public"; the version and date are updated. The new -license has a new handle. - -- Added the Tools/compiler package. This is a project led by Jeremy -Hylton to write the Python bytecode generator in Python. - -- The function math.rint() is removed. - -- In Python.h, "#define _GNU_SOURCE 1" was added. - -- Version 0.9.1 of Greg Ward's distutils is included (instead of -version 0.9). - -- A new version of SRE is included. It is more stable, and more -compatible with the old RE module. Non-matching ranges are indicated -by -1, not None. (The documentation said None, but the PRE -implementation used -1; changing to None would break existing code.) - -- The winreg module has been renamed to _winreg. (There are plans for -a higher-level API called winreg, but this has not yet materialized in -a form that is acceptable to the experts.) - -- The _locale module is enabled by default. - -- Fixed the configuration line for the _curses module. - -- A few crashes have been fixed, notably .writelines() with a -list containing non-string objects would crash, and there were -situations where a lost SyntaxError could dump core. - -- The .extend() method now accepts an arbitrary sequence -argument. - -- If __str__() or __repr__() returns a Unicode object, this is -converted to an 8-bit string. - -- Unicode string comparisons is no longer aware of UTF-16 -encoding peculiarities; it's a straight 16-bit compare. - -- The Windows installer now installs the LICENSE file and no longer -registers the Python DLL version in the registry (this is no longer -needed). It now uses Tcl/Tk 8.3.2. - -- A few portability problems have been fixed, in particular a -compilation error involving socklen_t. - -- The PC configuration is slightly friendlier to non-Microsoft -compilers. - - -====================================================================== - - -====================================== -==> Release 1.5.2 (April 13, 1999) <== -====================================== - -From 1.5.2c1 to 1.5.2 (final) -============================= - -Tue Apr 13 15:44:49 1999 Guido van Rossum - - * PCbuild/python15.wse: Bump version to 1.5.2 (final) - - * PCbuild/python15.dsp: Added shamodule.c - - * PC/config.c: Added sha module! - - * README, Include/patchlevel.h: Prepare for final release. - - * Misc/ACKS: - More (Cameron Laird is honorary; the others are 1.5.2c1 testers). - - * Python/thread_solaris.h: - While I can't really test this thoroughly, Pat Knight and the Solaris - man pages suggest that the proper thing to do is to add THR_NEW_LWP to - the flags on thr_create(), and that there really isn't a downside, so - I'll do that. - - * Misc/ACKS: - Bunch of new names who helped iron out the last wrinkles of 1.5.2. - - * PC/python_nt.rc: - Bump the myusterious M$ version number from 1,5,2,1 to 1,5,2,3. - (I can't even display this on NT, maybe Win/98 can?) - - * Lib/pstats.py: - Fix mysterious references to jprofile that were in the source since - its creation. I'm assuming these were once valid references to "Jim - Roskind's profile"... - - * Lib/Attic/threading_api.py: - Removed; since long subsumed in Doc/lib/libthreading.tex - - * Modules/socketmodule.c: - Put back __osf__ support for gethostbyname_r(); the real bug was that - it was being used even without threads. This of course might be an - all-platform problem so now we only use the _r variant when we are - using threads. - -Mon Apr 12 22:51:20 1999 Guido van Rossum - - * Modules/cPickle.c: - Fix accidentally reversed NULL test in load_mark(). Suggested by - Tamito Kajiyama. (This caused a bug only on platforms where malloc(0) - returns NULL.) - - * README: - Add note about popen2 problem on Linux noticed by Pablo Bleyer. - - * README: Add note about -D_REENTRANT for HP-UX 10.20. - - * Modules/Makefile.pre.in: 'clean' target should remove hassignal. - - * PC/Attic/vc40.mak, PC/readme.txt: - Remove all VC++ info (except VC 1.5) from readme.txt; - remove the VC++ 4.0 project file; remove the unused _tkinter extern defs. - - * README: Clarify PC build instructions (point to PCbuild). - - * Modules/zlibmodule.c: Cast added by Jack Jansen (for Mac port). - - * Lib/plat-sunos5/CDIO.py, Lib/plat-linux2/CDROM.py: - Forgot to add this file. CDROM device parameters. - - * Lib/gzip.py: Two different changes. - - 1. Jack Jansen reports that on the Mac, the time may be negative, and - solves this by adding a write32u() function that writes an unsigned - long. - - 2. On 64-bit platforms the CRC comparison fails; I've fixed this by - casting both values to be compared to "unsigned long" i.e. modulo - 0x100000000L. - -Sat Apr 10 18:42:02 1999 Guido van Rossum - - * PC/Attic/_tkinter.def: No longer needed. - - * Misc/ACKS: Correct missed character in Andrew Dalke's name. - - * README: Add DEC Ultrix notes (from Donn Cave's email). - - * configure: The usual - - * configure.in: - Quote a bunch of shell variables used in test, related to long-long. - - * Objects/fileobject.c, Modules/shamodule.c, Modules/regexpr.c: - casts for picky compilers. - - * Modules/socketmodule.c: - 3-arg gethostbyname_r doesn't really work on OSF/1. - - * PC/vc15_w31/_.c, PC/vc15_lib/_.c, Tools/pynche/__init__.py: - Avoid totally empty files. - -Fri Apr 9 14:56:35 1999 Guido van Rossum - - * Tools/scripts/fixps.py: Use re instead of regex. - Don't rewrite the file in place. - (Reported by Andy Dustman.) - - * Lib/netrc.py, Lib/shlex.py: Get rid of #! line - -Thu Apr 8 23:13:37 1999 Guido van Rossum - - * PCbuild/python15.wse: Use the Tcl 8.0.5 installer. - Add a variable %_TCL_% that makes it easier to switch to a different version. - - -====================================================================== - - -From 1.5.2b2 to 1.5.2c1 -======================= - -Thu Apr 8 23:13:37 1999 Guido van Rossum - - * PCbuild/python15.wse: - Release 1.5.2c1. Add IDLE and Uninstall to program group. - Don't distribute zlib.dll. Tweak some comments. - - * PCbuild/zlib.dsp: Now using static zlib 1.1.3 - - * Lib/dos-8x3/userdict.py, Lib/dos-8x3/userlist.py, Lib/dos-8x3/test_zli.py, Lib/dos-8x3/test_use.py, Lib/dos-8x3/test_pop.py, Lib/dos-8x3/test_pic.py, Lib/dos-8x3/test_ntp.py, Lib/dos-8x3/test_gzi.py, Lib/dos-8x3/test_fcn.py, Lib/dos-8x3/test_cpi.py, Lib/dos-8x3/test_bsd.py, Lib/dos-8x3/posixfil.py, Lib/dos-8x3/mimetype.py, Lib/dos-8x3/nturl2pa.py, Lib/dos-8x3/compilea.py, Lib/dos-8x3/exceptio.py, Lib/dos-8x3/basehttp.py: - The usual - - * Include/patchlevel.h: Release 1.5.2c1 - - * README: Release 1.5.2c1. - - * Misc/NEWS: News for the 1.5.2c1 release. - - * Lib/test/test_strftime.py: - On Windows, we suddenly find, strftime() may return "" for an - unsupported format string. (I guess this is because the logic for - deciding whether to reallocate the buffer or not has been improved.) - This caused the test code to crash on result[0]. Fix this by assuming - an empty result also means the format is not supported. - - * Demo/tkinter/matt/window-creation-w-location.py: - This demo imported some private code from Matt. Make it cripple along. - - * Lib/lib-tk/Tkinter.py: - Delete an accidentally checked-in feature that actually broke more - than was worth it: when deleting a canvas item, it would try to - automatically delete the bindings for that item. Since there's - nothing that says you can't reuse the tag and still have the bindings, - this is not correct. Also, it broke at least one demo - (Demo/tkinter/matt/rubber-band-box-demo-1.py). - - * Python/thread_wince.h: Win/CE thread support by Mark Hammond. - -Wed Apr 7 20:23:17 1999 Guido van Rossum - - * Modules/zlibmodule.c: - Patch by Andrew Kuchling to unflush() (flush() for deflating). - Without this, if inflate() returned Z_BUF_ERROR asking for more output - space, we would report the error; now, we increase the buffer size and - try again, just as for Z_OK. - - * Lib/test/test_gzip.py: Use binary mode for all gzip files we open. - - * Tools/idle/ChangeLog: New change log. - - * Tools/idle/README.txt, Tools/idle/NEWS.txt: New version. - - * Python/pythonrun.c: - Alas, get rid of the Win specific hack to ask the user to press Return - before exiting when an error happened. This didn't work right when - Python is invoked from a daemon. - - * Tools/idle/idlever.py: Version bump awaiting impending new release. - (Not much has changed :-( ) - - * Lib/lib-tk/Tkinter.py: - lower, tkraise/lift hide Misc.lower, Misc.tkraise/lift, - so the preferred name for them is tag_lower, tag_raise - (similar to tag_bind, and similar to the Text widget); - unfortunately can't delete the old ones yet (maybe in 1.6) - - * Python/thread.c, Python/strtod.c, Python/mystrtoul.c, Python/import.c, Python/ceval.c: - Changes by Mark Hammond for Windows CE. Mostly of the form - #ifdef DONT_HAVE_header_H ... #endif around #include . - - * Python/bltinmodule.c: - Remove unused variable from complex_from_string() code. - - * Include/patchlevel.h: - Add the possibility of a gamma release (release candidate). - Add '+' to string version number to indicate we're beyond b2 now. - - * Modules/posixmodule.c: Add extern decl for fsync() for SunOS 4.x. - - * Lib/smtplib.py: Changes by Per Cederquist and The Dragon. - - Per writes: - - """ - The application where Signum Support uses smtplib needs to be able to - report good error messages to the user when sending email fails. To - help in diagnosing problems it is useful to be able to report the - entire message sent by the server, not only the SMTP error code of the - offending command. - - A lot of the functions in sendmail.py unfortunately discards the - message, leaving only the code. The enclosed patch fixes that - problem. - - The enclosed patch also introduces a base class for exceptions that - include an SMTP error code and error message, and make the code and - message available on separate attributes, so that surrounding code can - deal with them in whatever way it sees fit. I've also added some - documentation to the exception classes. - - The constructor will now raise an exception if it cannot connect to - the SMTP server. - - The data() method will raise an SMTPDataError if it doesn't receive - the expected 354 code in the middle of the exchange. - - According to section 5.2.10 of RFC 1123 a smtp client must accept "any - text, including no text at all" after the error code. If the response - of a HELO command contains no text self.helo_resp will be set to the - empty string (""). The patch fixes the test in the sendmail() method - so that helo_resp is tested against None; if it has the empty string - as value the sendmail() method would invoke the helo() method again. - - The code no longer accepts a -1 reply from the ehlo() method in - sendmail(). - - [Text about removing SMTPRecipientsRefused deleted --GvR] - """ - - and also: - - """ - smtplib.py appends an extra blank line to the outgoing mail if the - `msg' argument to the sendmail method already contains a trailing - newline. This patch should fix the problem. - """ - - The Dragon writes: - - """ - Mostly I just re-added the SMTPRecipientsRefused exception - (the exeption object now has the appropriate info in it ) [Per had - removed this in his patch --GvR] and tweaked the behavior of the - sendmail method whence it throws the newly added SMTPHeloException (it - was closing the connection, which it shouldn't. whatever catches the - exception should do that. ) - - I pondered the change of the return values to tuples all around, - and after some thinking I decided that regularizing the return values was - too much of the Right Thing (tm) to not do. - - My one concern is that code expecting an integer & getting a tuple - may fail silently. - - (i.e. if it's doing : - - x.somemethod() >= 400: - expecting an integer, the expression will always be true if it gets a - tuple instead. ) - - However, most smtplib code I've seen only really uses the - sendmail() method, so this wouldn't bother it. Usually code I've seen - that calls the other methods usually only calls helo() and ehlo() for - doing ESMTP, a feature which was not in the smtplib included with 1.5.1, - and thus I would think not much code uses it yet. - """ - -Tue Apr 6 19:38:18 1999 Guido van Rossum - - * Lib/test/test_ntpath.py: - Fix the tests now that splitdrive() no longer treats UNC paths special. - (Some tests converted to splitunc() tests.) - - * Lib/ntpath.py: - Withdraw the UNC support from splitdrive(). Instead, a new function - splitunc() parses UNC paths. The contributor of the UNC parsing in - splitdrive() doesn't like it, but I haven't heard a good reason to - keep it, and it causes some problems. (I think there's a - philosophical problem -- to me, the split*() functions are purely - syntactical, and the fact that \\foo is not a valid path doesn't mean - that it shouldn't be considered an absolute path.) - - Also (quite separately, but strangely related to the philosophical - issue above) fix abspath() so that if win32api exists, it doesn't fail - when the path doesn't actually exist -- if GetFullPathName() fails, - fall back on the old strategy (join with getcwd() if necessary, and - then use normpath()). - - * configure.in, configure, config.h.in, acconfig.h: - For BeOS PowerPC. Chris Herborth. - -Mon Apr 5 21:54:14 1999 Guido van Rossum - - * Modules/timemodule.c: - Jonathan Giddy notes, and Chris Lawrence agrees, that some comments on - #else/#endif are wrong, and that #if HAVE_TM_ZONE should be #ifdef. - - * Misc/ACKS: - Bunch of new contributors, including 9 who contributed to the Docs, - reported by Fred. - -Mon Apr 5 18:37:59 1999 Fred Drake - - * Lib/gzip.py: - Oops, missed mode parameter to open(). - - * Lib/gzip.py: - Made the default mode 'rb' instead of 'r', for better cross-platform - support. (Based on comment on the documentation by Bernhard Reiter - ). - -Fri Apr 2 22:18:25 1999 Guido van Rossum - - * Tools/scripts/dutree.py: - For reasons I dare not explain, this script should always execute - main() when imported (in other words, it is not usable as a module). - -Thu Apr 1 15:32:30 1999 Guido van Rossum - - * Lib/test/test_cpickle.py: Jonathan Giddy write: - - In test_cpickle.py, the module os got imported, but the line to remove - the temp file has gone missing. - -Tue Mar 30 20:17:31 1999 Guido van Rossum - - * Lib/BaseHTTPServer.py: Per Cederqvist writes: - - If you send something like "PUT / HTTP/1.0" to something derived from - BaseHTTPServer that doesn't define do_PUT, you will get a response - that begins like this: - - HTTP/1.0 501 Unsupported method ('do_PUT') - Server: SimpleHTTP/0.3 Python/1.5 - Date: Tue, 30 Mar 1999 18:53:53 GMT - - The server should complain about 'PUT' instead of 'do_PUT'. This - patch should fix the problem. - -Mon Mar 29 20:33:21 1999 Guido van Rossum - - * Lib/smtplib.py: Patch by Per Cederqvist, who writes: - - """ - - It needlessly used the makefile() method for each response that is - read from the SMTP server. - - - If the remote SMTP server closes the connection unexpectedly the - code raised an IndexError. It now raises an SMTPServerDisconnected - exception instead. - - - The code now checks that all lines in a multiline response actually - contains an error code. - """ - - The Dragon approves. - -Mon Mar 29 20:25:40 1999 Fred Drake - - * Lib/compileall.py: - When run as a script, report failures in the exit code as well. - Patch largely based on changes by Andrew Dalke, as discussed in the - distutils-sig. - -Mon Mar 29 20:23:41 1999 Guido van Rossum - - * Lib/urllib.py: - Hack so that if a 302 or 301 redirect contains a relative URL, the - right thing "just happens" (basejoin() with old URL). - - * Modules/cPickle.c: - Protection against picling to/from closed (real) file. - The problem was reported by Moshe Zadka. - - * Lib/test/test_cpickle.py: - Test protection against picling to/from closed (real) file. - - * Modules/timemodule.c: Chris Lawrence writes: - - """ - The GNU folks, in their infinite wisdom, have decided not to implement - altzone in libc6; this would not be horrible, except that timezone - (which is implemented) includes the current DST setting (i.e. timezone - for Central is 18000 in summer and 21600 in winter). So Python's - timezone and altzone variables aren't set correctly during DST. - - Here's a patch relative to 1.5.2b2 that (a) makes timezone and altzone - show the "right" thing on Linux (by using the tm_gmtoff stuff - available in BSD, which is how the GLIBC manual claims things should - be done) and (b) should cope with the southern hemisphere. In pursuit - of (b), I also took the liberty of renaming the "summer" and "winter" - variables to "july" and "jan". This patch should also make certain - time calculations on Linux actually work right (like the tz-aware - functions in the rfc822 module). - - (It's hard to find DST that's currently being used in the southern - hemisphere; I tested using Africa/Windhoek.) - """ - - * Lib/test/output/test_gzip: - Jonathan Giddy discovered this file was missing. - - * Modules/shamodule.c: - Avoid warnings from AIX compiler. Reported by Vladimir (AIX is my - middlename) Marangozov, patch coded by Greg Stein. - - * Tools/idle/ScriptBinding.py, Tools/idle/PyShell.py: - At Tim Peters' recommendation, add a dummy flush() method to PseudoFile. - -Sun Mar 28 17:55:32 1999 Guido van Rossum - - * Tools/scripts/ndiff.py: Tim Peters writes: - - I should have waited overnight . Nothing wrong with the one I - sent, but I couldn't resist going on to add new -r1 / -r2 cmdline options - for recreating the original files from ndiff's output. That's attached, if - you're game! Us Windows guys don't usually have a sed sitting around - . - -Sat Mar 27 13:34:01 1999 Guido van Rossum - - * Tools/scripts/ndiff.py: Tim Peters writes: - - Attached is a cleaned-up version of ndiff (added useful module - docstring, now echo'ed in case of cmd line mistake); added -q option - to suppress initial file identification lines; + other minor cleanups, - & a slightly faster match engine. - -Fri Mar 26 22:36:00 1999 Fred Drake - - * Tools/scripts/dutree.py: - During display, if EPIPE is raised, it's probably because a pager was - killed. Discard the error in that case, but propagate it otherwise. - -Fri Mar 26 16:20:45 1999 Guido van Rossum - - * Lib/test/output/test_userlist, Lib/test/test_userlist.py: - Test suite for UserList. - - * Lib/UserList.py: Use isinstance() where appropriate. - Reformatted with 4-space indent. - -Fri Mar 26 16:11:40 1999 Barry Warsaw - - * Tools/pynche/PyncheWidget.py: - Helpwin.__init__(): The text widget should get focus. - - * Tools/pynche/pyColorChooser.py: - Removed unnecessary import `from PyncheWidget import PyncheWidget' - -Fri Mar 26 15:32:05 1999 Guido van Rossum - - * Lib/test/output/test_userdict, Lib/test/test_userdict.py: - Test suite for UserDict - - * Lib/UserDict.py: Improved a bunch of things. - The constructor now takes an optional dictionary. - Use isinstance() where appropriate. - -Thu Mar 25 22:38:49 1999 Guido van Rossum - - * Lib/test/output/test_pickle, Lib/test/output/test_cpickle, Lib/test/test_pickle.py, Lib/test/test_cpickle.py: - Basic regr tests for pickle/cPickle - - * Lib/pickle.py: - Don't use "exec" in find_class(). It's slow, unnecessary, and (as AMK - points out) it doesn't work in JPython Applets. - -Thu Mar 25 21:50:27 1999 Andrew Kuchling - - * Lib/test/test_gzip.py: - Added a simple test suite for gzip. It simply opens a temp file, - writes a chunk of compressed data, closes it, writes another chunk, and - reads the contents back to verify that they are the same. - - * Lib/gzip.py: - Based on a suggestion from bruce@hams.com, make a trivial change to - allow using the 'a' flag as a mode for opening a GzipFile. gzip - files, surprisingly enough, can be concatenated and then decompressed; - the effect is to concatenate the two chunks of data. - - If we support it on writing, it should also be supported on reading. - This *wasn't* trivial, and required rearranging the code in the - reading path, particularly the _read() method. - - Raise IOError instead of RuntimeError in two cases, 'Not a gzipped file' - and 'Unknown compression method' - -Thu Mar 25 21:25:01 1999 Guido van Rossum - - * Lib/test/test_b1.py: - Add tests for float() and complex() with string args (Nick/Stephanie - Lockwood). - -Thu Mar 25 21:21:08 1999 Andrew Kuchling - - * Modules/zlibmodule.c: - Add an .unused_data attribute to decompressor objects. If .unused_data - is not an empty string, this means that you have arrived at the - end of the stream of compressed data, and the contents of .unused_data are - whatever follows the compressed stream. - -Thu Mar 25 21:16:07 1999 Guido van Rossum - - * Python/bltinmodule.c: - Patch by Nick and Stephanie Lockwood to implement complex() with a string - argument. This closes TODO item 2.19. - -Wed Mar 24 19:09:00 1999 Guido van Rossum - - * Tools/webchecker/wcnew.py: Added Samuel Bayer's new webchecker. - Unfortunately his code breaks wcgui.py in a way that's not easy - to fix. I expect that this is a temporary situation -- - eventually Sam's changes will be merged back in. - (The changes add a -t option to specify exceptions to the -x - option, and explicit checking for #foo style fragment ids.) - - * Objects/dictobject.c: - Vladimir Marangozov contributed updated comments. - - * Objects/bufferobject.c: Folded long lines. - - * Lib/test/output/test_sha, Lib/test/test_sha.py: - Added Jeremy's test code for the sha module. - - * Modules/shamodule.c, Modules/Setup.in: - Added Greg Stein and Andrew Kuchling's sha module. - Fix comments about zlib version and URL. - - * Lib/test/test_bsddb.py: Remove the temp file when we're done. - - * Include/pythread.h: Conform to standard boilerplate. - - * configure.in, configure, BeOS/linkmodule, BeOS/ar-fake: - Chris Herborth: the new compiler in R4.1 needs some new options to work... - - * Modules/socketmodule.c: - Implement two suggestions by Jonathan Giddy: (1) in AIX, clear the - data struct before calling gethostby{name,addr}_r(); (2) ignore the - 3/5/6 args determinations made by the configure script and switch on - platform identifiers instead: - - AIX, OSF have 3 args - Sun, SGI have 5 args - Linux has 6 args - - On all other platforms, undef HAVE_GETHOSTBYNAME_R altogether. - - * Modules/socketmodule.c: - Vladimir Marangozov implements the AIX 3-arg gethostbyname_r code. - - * Lib/mailbox.py: - Add readlines() to _Subfile class. Not clear who would need it, but - Chris Lawrence sent me a broken version; this one is a tad simpler and - more conforming to the standard. - -Tue Mar 23 23:05:34 1999 Jeremy Hylton - - * Lib/gzip.py: use struct instead of bit-manipulate in Python - -Tue Mar 23 19:00:55 1999 Guido van Rossum - - * Modules/Makefile.pre.in: - Add $(EXE) to various occurrences of python so it will work on Cygwin - with egcs (after setting EXE=.exe). Patch by Norman Vine. - - * configure, configure.in: - Ack! It never defined HAVE_GETHOSTBYNAME_R so that code was never tested! - -Mon Mar 22 22:25:39 1999 Guido van Rossum - - * Include/thread.h: - Adding thread.h -- unused but for b/w compatibility. - As requested by Bill Janssen. - - * configure.in, configure: - Add code to test for all sorts of gethostbyname_r variants, - donated by David Arnold. - - * config.h.in, acconfig.h: - Add symbols for gethostbyname_r variants (sigh). - - * Modules/socketmodule.c: Clean up pass for the previous patches. - - - Use HAVE_GETHOSTBYNAME_R_6_ARG instead of testing for Linux and - glibc2. - - - If gethostbyname takes 3 args, undefine HAVE_GETHOSTBYNAME_R -- - don't know what code should be used. - - - New symbol USE_GETHOSTBYNAME_LOCK defined iff the lock should be used. - - - Modify the gethostbyaddr() code to also hold on to the lock until - after it is safe to release, overlapping with the Python lock. - - (Note: I think that it could in theory be possible that Python code - executed while gethostbyname_lock is held could attempt to reacquire - the lock -- e.g. in a signal handler or destructor. I will simply say - "don't do that then.") - - * Modules/socketmodule.c: Jonathan Giddy writes: - - Here's a patch to fix the race condition, which wasn't fixed by Rob's - patch. It holds the gethostbyname lock until the results are copied out, - which means that this lock and the Python global lock are held at the same - time. This shouldn't be a problem as long as the gethostbyname lock is - always acquired when the global lock is not held. - -Mon Mar 22 19:25:30 1999 Andrew Kuchling - - * Modules/zlibmodule.c: - Fixed the flush() method of compression objects; the test for - the end of loop was incorrect, and failed when the flushmode != Z_FINISH. - Logic cleaned up and commented. - - * Lib/test/test_zlib.py: - Added simple test for the flush() method of compression objects, trying the - different flush values Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FULL_FLUSH. - -Mon Mar 22 15:28:08 1999 Guido van Rossum - - * Lib/shlex.py: - Bug reported by Tobias Thelen: missing "self." in assignment target. - -Fri Mar 19 21:50:11 1999 Guido van Rossum - - * Modules/arraymodule.c: - Use an unsigned cast to avoid a warning in VC++. - - * Lib/dospath.py, Lib/ntpath.py: - New code for split() by Tim Peters, behaves more like posixpath.split(). - - * Objects/floatobject.c: - Fix a problem with Vladimir's PyFloat_Fini code: clear the free list; if - a block cannot be freed, add its free items back to the free list. - This is necessary to avoid leaking when Python is reinitialized later. - - * Objects/intobject.c: - Fix a problem with Vladimir's PyInt_Fini code: clear the free list; if - a block cannot be freed, add its free items back to the free list, and - add its valid ints back to the small_ints array if they are in range. - This is necessary to avoid leaking when Python is reinitialized later. - - * Lib/types.py: - Added BufferType, the type returned by the new builtin buffer(). Greg Stein. - - * Python/bltinmodule.c: - New builtin buffer() creates a derived read-only buffer from any - object that supports the buffer interface (e.g. strings, arrays). - - * Objects/bufferobject.c: - Added check for negative offset for PyBuffer_FromObject and check for - negative size for PyBuffer_FromMemory. Greg Stein. - -Thu Mar 18 15:10:44 1999 Guido van Rossum - - * Lib/urlparse.py: Sjoerd Mullender writes: - - If a filename on Windows starts with \\, it is converted to a URL - which starts with ////. If this URL is passed to urlparse.urlparse - you get a path that starts with // (and an empty netloc). If you pass - the result back to urlparse.urlunparse, you get a URL that starts with - //, which is parsed differently by urlparse.urlparse. The fix is to - add the (empty) netloc with accompanying slashes if the path in - urlunparse starts with //. Do this for all schemes that use a netloc. - - * Lib/nturl2path.py: Sjoerd Mullender writes: - - Pathnames of files on other hosts in the same domain - (\\host\path\to\file) are not translated correctly to URLs and back. - The URL should be something like file:////host/path/to/file. - Note that a combination of drive letter and remote host is not - possible. - -Wed Mar 17 22:30:10 1999 Guido van Rossum - - * Lib/urlparse.py: - Delete non-standard-conforming code in urljoin() that would use the - netloc from the base url as the default netloc for the resulting url - even if the schemes differ. - - Once upon a time, when the web was wild, this was a valuable hack - because some people had a URL referencing an ftp server colocated with - an http server without having the host in the ftp URL (so they could - replicate it or change the hostname easily). - - More recently, after the file: scheme got added back to the list of - schemes that accept a netloc, it turns out that this caused weirdness - when joining an http: URL with a file: URL -- the resulting file: URL - would always inherit the host from the http: URL because the file: - scheme supports a netloc but in practice never has one. - - There are two reasons to get rid of the old, once-valuable hack, - instead of removing the file: scheme from the uses_netloc list. One, - the RFC says that file: uses the netloc syntax, and does not endorse - the old hack. Two, neither netscape 4.5 nor IE 4.0 support the old - hack. - - * Include/ceval.h, Include/abstract.h: - Add DLL level b/w compat for PySequence_In and PyEval_CallObject - -Tue Mar 16 21:54:50 1999 Guido van Rossum - - * Lib/lib-tk/Tkinter.py: Bug reported by Jim Robinson: - - An attempt to execute grid_slaves with arguments (0,0) results in - *all* of the slaves being returned, not just the slave associated with - row 0, column 0. This is because the test for arguments in the method - does not test to see if row (and column) does not equal None, but - rather just whether is evaluates to non-false. A value of 0 fails - this test. - -Tue Mar 16 14:17:48 1999 Fred Drake - - * Modules/cmathmodule.c: - Docstring fix: acosh() returns the hyperbolic arccosine, not the - hyperbolic cosine. Problem report via David Ascher by one of his - students. - -Mon Mar 15 21:40:59 1999 Guido van Rossum - - * configure.in: - Should test for gethost*by*name_r, not for gethostname_r (which - doesn't exist and doesn't make sense). - - * Modules/socketmodule.c: - Patch by Rob Riggs for Linux -- glibc2 has a different argument - converntion for gethostbyname_r() etc. than Solaris! - - * Python/thread_pthread.h: Rob Riggs wrote: - - """ - Spec says that on success pthread_create returns 0. It does not say - that an error code will be < 0. Linux glibc2 pthread_create() returns - ENOMEM (12) when one exceed process limits. (It looks like it should - return EAGAIN, but that's another story.) - - For reference, see: - http://www.opengroup.org/onlinepubs/7908799/xsh/pthread_create.html - """ - - [I have a feeling that similar bugs were fixed before; perhaps someone - could check that all error checks no check for != 0?] - - * Tools/bgen/bgen/bgenObjectDefinition.py: - New mixin class that defines cmp and hash that use - the ob_itself pointer. This allows (when using the mixin) - different Python objects pointing to the same C object and - behaving well as dictionary keys. - - Or so sez Jack Jansen... - - * Lib/urllib.py: Yet another patch by Sjoerd Mullender: - - Don't convert URLs to URLs using pathname2url. - -Fri Mar 12 22:15:43 1999 Guido van Rossum - - * Lib/cmd.py: Patch by Michael Scharf. He writes: - - The module cmd requires for each do_xxx command a help_xxx - function. I think this is a little old fashioned. - - Here is a patch: use the docstring as help if no help_xxx - function can be found. - - [I'm tempted to rip out all the help_* functions from pdb, but I'll - resist it. Any takers? --Guido] - - * Tools/freeze/freeze.py: Bug submitted by Wayne Knowles, who writes: - - Under Windows, python freeze.py -o hello hello.py - creates all the correct files in the hello subdirectory, but the - Makefile has the directory prefix in it for frozen_extensions.c - nmake fails because it tries to locate hello/frozen_extensions.c - - (His fix adds a call to os.path.basename() in the appropriate place.) - - * Objects/floatobject.c, Objects/intobject.c: - Vladimir has restructured his code somewhat so that the blocks are now - represented by an explicit structure. (There are still too many casts - in the code, but that may be unavoidable.) - - Also added code so that with -vv it is very chatty about what it does. - - * Demo/zlib/zlibdemo.py, Demo/zlib/minigzip.py: - Change #! line to modern usage; also chmod +x - - * Demo/pdist/rrcs, Demo/pdist/rcvs, Demo/pdist/rcsbump: - Change #! line to modern usage - - * Lib/nturl2path.py, Lib/urllib.py: From: Sjoerd Mullender - - The filename to URL conversion didn't properly quote special - characters. - The URL to filename didn't properly unquote special chatacters. - - * Objects/floatobject.c: - OK, try again. Vladimir gave me a fix for the alignment bus error, - so here's his patch again. This time it works (at least on Solaris, - Linux and Irix). - -Thu Mar 11 23:21:23 1999 Guido van Rossum - - * Tools/idle/PathBrowser.py: - Don't crash when sys.path contains an empty string. - - * Tools/idle/PathBrowser.py: - - Don't crash in the case where a superclass is a string instead of a - pyclbr.Class object; this can happen when the superclass is - unrecognizable (to pyclbr), e.g. when module renaming is used. - - - Show a watch cursor when calling pyclbr (since it may take a while - recursively parsing imported modules!). - -Thu Mar 11 16:04:04 1999 Fred Drake - - * Lib/mimetypes.py: - Added .rdf and .xsl as application/xml types. (.rdf is for the - Resource Description Framework, a metadata encoding, and .xsl is for - the Extensible Stylesheet Language.) - -Thu Mar 11 13:26:23 1999 Guido van Rossum - - * Lib/test/output/test_popen2, Lib/test/test_popen2.py: - Test for popen2 module, by Chris Tismer. - - * Objects/floatobject.c: - Alas, Vladimir's patch caused a bus error (probably double - alignment?), and I didn't test it. Withdrawing it for now. - -Wed Mar 10 22:55:47 1999 Guido van Rossum - - * Objects/floatobject.c: - Patch by Vladimir Marangoz to allow freeing of the allocated blocks of - floats on finalization. - - * Objects/intobject.c: - Patch by Vladimir Marangoz to allow freeing of the allocated blocks of - integers on finalization. - - * Tools/idle/EditorWindow.py, Tools/idle/Bindings.py: - Add PathBrowser to File module - - * Tools/idle/PathBrowser.py: - "Path browser" - 4 scrolled lists displaying: - directories on sys.path - modules in selected directory - classes in selected module - methods of selected class - - Sinlge clicking in a directory, module or class item updates the next - column with info about the selected item. Double clicking in a - module, class or method item opens the file (and selects the clicked - item if it is a class or method). - - I guess eventually I should be using a tree widget for this, but the - ones I've seen don't work well enough, so for now I use the old - Smalltalk or NeXT style multi-column hierarchical browser. - - * Tools/idle/MultiScrolledLists.py: - New utility: multiple scrolled lists in parallel - - * Tools/idle/ScrolledList.py: - White background. - - Display "(None)" (or text of your choosing) when empty. - - Don't set the focus. - -Tue Mar 9 19:31:21 1999 Guido van Rossum - - * Lib/urllib.py: - open_http also had the 'data is None' test backwards. don't call with the - extra argument if data is None. - - * Demo/embed/demo.c: - Call Py_SetProgramName() instead of redefining getprogramname(), - reflecting changes in the runtime around 1.5 or earlier. - - * Python/ceval.c: - Always test for an error return (usually NULL or -1) without setting - an exception. - - * Modules/timemodule.c: Patch by Chris Herborth for BeOS code. - He writes: - - I had an off-by-1000 error in floatsleep(), - and the problem with time.clock() is that it's not implemented properly - on QNX... ANSI says it's supposed to return _CPU_ time used by the - process, but on QNX it returns the amount of real time used... so I was - confused. - - * Tools/bgen/bgen/macsupport.py: Small change by Jack Jansen. - Test for self.returntype behaving like OSErr rather than being it. - -Thu Feb 25 16:14:58 1999 Jeremy Hylton - - * Lib/urllib.py: - http_error had the 'data is None' test backwards. don't call with the - extra argument if data is None. - - * Lib/urllib.py: change indentation from 8 spaces to 4 spaces - - * Lib/urllib.py: pleasing the tabnanny - -Thu Feb 25 14:26:02 1999 Fred Drake - - * Lib/colorsys.py: - Oops, one more "x, y, z" to convert... - - * Lib/colorsys.py: - Adjusted comment at the top to be less confusing, following Fredrik - Lundh's example. - - Converted comment to docstring. - -Wed Feb 24 18:49:15 1999 Fred Drake - - * Lib/toaiff.py: - Use sndhdr instead of the obsolete whatsound module. - -Wed Feb 24 18:42:38 1999 Jeremy Hylton - - * Lib/urllib.py: - When performing a POST request, i.e. when the second argument to - urlopen is used to specify form data, make sure the second argument is - threaded through all of the http_error_NNN calls. This allows error - handlers like the redirect and authorization handlers to properly - re-start the connection. - -Wed Feb 24 16:25:17 1999 Guido van Rossum - - * Lib/mhlib.py: Patch by Lars Wirzenius: - - o the initial comment is wrong: creating messages is already - implemented - - o Message.getbodytext: if the mail or it's part contains an - empty content-transfer-encoding header, the code used to - break; the change below treats an empty encoding value the same - as the other types that do not need decoding - - o SubMessage.getbodytext was missing the decode argument; the - change below adds it; I also made it unconditionally return - the raw text if decoding was not desired, because my own - routines needed that (and it was easier than rewriting my - own routines ;-) - -Wed Feb 24 00:35:43 1999 Barry Warsaw - - * Python/bltinmodule.c (initerrors): - Make sure that the exception tuples ("base-classes" when - string-based exceptions are used) reflect the real class hierarchy, - i.e. that SystemExit derives from Exception not StandardError. - - * Lib/exceptions.py: - Document the correct class hierarchy for SystemExit. It is not an - error and so it derives from Exception and not SystemError. The - docstring was incorrect but the implementation was fine. - -Tue Feb 23 23:07:51 1999 Guido van Rossum - - * Lib/shutil.py: - Add import sys, needed by reference to sys.exc_info() in rmtree(). - Discovered by Mitch Chapman. - - * config.h.in: - Now that we don't have AC_CHECK_LIB(m, pow), the HAVE_LIBM symbol - disappears. It wasn't used anywhere anyway... - - * Modules/arraymodule.c: - Carefully check for overflow when allocating the memory for fromfile - -- someone tried to pass in sys.maxint and got bitten by the bogus - calculations. - - * configure.in: - Get rid of AC_CHECK_LIB(m, pow) since this is taken care of later with - LIBM (from --with-libm=...); this actually broke the customizability - offered by the latter option. Thanks go to Clay Spence for reporting - this. - - * Lib/test/test_dl.py: - 1. Print the error message (carefully) when a dl.open() fails in verbose mode. - 2. When no test case worked, raise ImportError instead of failing. - - * Python/bltinmodule.c: - Patch by Tim Peters to improve the range checks for range() and - xrange(), especially for platforms where int and long are different - sizes (so sys.maxint isn't actually the theoretical limit for the - length of a list, but the largest C int is -- sys.maxint is the - largest Python int, which is actually a C long). - - * Makefile.in: - 1. Augment the DG/UX rule so it doesn't break the BeOS build. - 2. Add $(EXE) to various occurrences of python so it will work on - Cygwin with egcs (after setting EXE=.exe). These patches by - Norman Vine. - - * Lib/posixfile.py: - According to Jeffrey Honig, bsd/os 2.0 - 4.0 should be added to the - list (of bsd variants that have a different lock structure). - - * Lib/test/test_fcntl.py: - According to Jeffrey Honig, bsd/os 4.0 should be added to the list. - - * Modules/timemodule.c: - Patch by Tadayoshi Funaba (with some changes) to be smarter about - guessing what happened when strftime() returns 0. Is it buffer - overflow or was the result simply 0 bytes long? (This happens for an - empty format string, or when the format string is a single %Z and the - timezone is unknown.) if the buffer is at least 256 times as long as - the format, assume the latter. - -Mon Feb 22 19:01:42 1999 Guido van Rossum - - * Lib/urllib.py: - As Des Barry points out, we need to call pathname2url(file) in two - calls to addinfourl() in open_file(). - - * Modules/Setup.in: Document *static* -- in two places! - - * Modules/timemodule.c: - We don't support leap seconds, so the seconds field of a time 9-tuple - should be in the range [0-59]. Noted by Tadayoshi Funaba. - - * Modules/stropmodule.c: - In atoi(), don't use isxdigit() to test whether the last character - converted was a "digit" -- use isalnum(). This test is there only to - guard against "+" or "-" being interpreted as a valid int literal. - Reported by Takahiro Nakayama. - - * Lib/os.py: - As Finn Bock points out, _P_WAIT etc. don't have a leading underscore - so they don't need to be treated specially here. - -Mon Feb 22 15:38:58 1999 Fred Drake - - * Misc/NEWS: - Typo: "apparentlt" --> "apparently" - -Mon Feb 22 15:38:46 1999 Guido van Rossum - - * Lib/urlparse.py: Steve Clift pointed out that 'file' allows a netloc. - - * Modules/posixmodule.c: - The docstring for ttyname(..) claims a second "mode" argument. The - actual code does not allow such an argument. (Finn Bock.) - - * Lib/lib-old/poly.py: - Dang. Even though this is obsolete code, somebody found a bug, and I - fix it. Oh well. - -Thu Feb 18 20:51:50 1999 Fred Drake - - * Lib/pyclbr.py: - Bow to font-lock at the end of the docstring, since it throws stuff - off. - - Make sure the path parameter to readmodule() is a list before adding it - with sys.path, or the addition could fail. - - -====================================================================== - - -From 1.5.2b1 to 1.5.2b2 -======================= - -General -------- - -- Many memory leaks fixed. - -- Many small bugs fixed. - -- Command line option -OO (or -O -O) suppresses inclusion of doc -strings in resulting bytecode. - -Windows-specific changes ------------------------- - -- New built-in module winsound provides an interface to the Win32 -PlaySound() call. - -- Re-enable the audioop module in the config.c file. - -- On Windows, support spawnv() and associated P_* symbols. - -- Fixed the conversion of times() return values on Windows. - -- Removed freeze from the installer -- it doesn't work without the -source tree. (See FAQ 8.11.) - -- On Windows 95/98, the Tkinter module now is smart enough to find -Tcl/Tk even when the PATH environment variable hasn't been set -- when -the import of _tkinter fails, it searches in a standard locations, -patches os.environ["PATH"], and tries again. When it still fails, a -clearer error message is produced. This should avoid most -installation problems with Tkinter use (e.g. in IDLE). - -- The -i option doesn't make any calls to set[v]buf() for stdin -- -this apparently screwed up _kbhit() and the _tkinter main loop. - -- The ntpath module (and hence, os.path on Windows) now parses out UNC -paths (e.g. \\host\mountpoint\dir\file) as "drive letters", so that -splitdrive() will \\host\mountpoint as the drive and \dir\file as the -path. ** EXPERIMENTAL ** - -- Added a hack to the exit code so that if (1) the exit status is -nonzero and (2) we think we have our own DOS box (i.e. we're not -started from a command line shell), we print a message and wait for -the user to hit a key before the DOS box is closed. - -- Updated the installer to WISE 5.0g. Added a dialog warning about -the imminent Tcl installation. Added a dialog to specify the program -group name in the start menu. Upgraded the Tcl installer to Tcl -8.0.4. - -Changes to intrinsics ---------------------- - -- The repr() or str() of a module object now shows the __file__ -attribute (i.e., the file which it was loaded), or the string -"(built-in)" if there is no __file__ attribute. - -- The range() function now avoids overflow during its calculations (if -at all possible). - -- New info string sys.hexversion, which is an integer encoding the -version in hexadecimal. In other words, hex(sys.hexversion) == -0x010502b2 for Python 1.5.2b2. - -New or improved ports ---------------------- - -- Support for Nextstep descendants (future Mac systems). - -- Improved BeOS support. - -- Support dynamic loading of shared libraries on NetBSD platforms that -use ELF (i.e., MIPS and Alpha systems). - -Configuration/build changes ---------------------------- - -- The Lib/test directory is no longer included in the default module -search path (sys.path) -- "test" has been a package ever since 1.5. - -- Now using autoconf 2.13. - -New library modules -------------------- - -- New library modules asyncore and asynchat: these form Sam Rushing's -famous asynchronous socket library. Sam has gracefully allowed me to -incorporate these in the standard Python library. - -- New module statvfs contains indexing constants for [f]statvfs() -return tuple. - -Changes to the library ----------------------- - -- The wave module (platform-independent support for Windows sound -files) has been fixed to actually make it work. - -- The sunau module (platform-independent support for Sun/NeXT sound -files) has been fixed to work across platforms. Also, a weird -encoding bug in the header of the audio test data file has been -corrected. - -- Fix a bug in the urllib module that occasionally tripped up -webchecker and other ftp retrieves. - -- ConfigParser's get() method now accepts an optional keyword argument -(vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your -configuration file. - -- Some improvements to the Queue module, including a put_nowait() -module and an optional "block" second argument, to get() and put(), -defaulting to 1. - -- The updated xmllib module is once again compatible with the version -present in Python 1.5.1 (this was accidentally broken in 1.5.2b1). - -- The bdb module (base class for the debugger) now supports -canonicalizing pathnames used in breakpoints. The derived class must -override the new canonical() method for this to work. Also changed -clear_break() to the backwards compatible old signature, and added -clear_bpbynumber() for the new functionality. - -- In sgmllib (and hence htmllib), recognize attributes even if they -don't have space in front of them. I.e. '' will now have two attributes recognized. - -- In the debugger (pdb), change clear syntax to support three -alternatives: clear; clear file:line; clear bpno bpno ... - -- The os.path module now pretends to be a submodule within the os -"package", so you can do things like "from os.path import exists". - -- The standard exceptions now have doc strings. - -- In the smtplib module, exceptions are now classes. Also avoid -inserting a non-standard space after "TO" in rcpt() command. - -- The rfc822 module's getaddrlist() method now uses all occurrences of -the specified header instead of just the first. Some other bugfixes -too (to handle more weird addresses found in a very large test set, -and to avoid crashes on certain invalid dates), and a small test -module has been added. - -- Fixed bug in urlparse in the common-case code for HTTP URLs; it -would lose the query, fragment, and/or parameter information. - -- The sndhdr module no longer supports whatraw() -- it depended on a -rare extenral program. - -- The UserList module/class now supports the extend() method, like -real list objects. - -- The uu module now deals better with trailing garbage generated by -some broke uuencoders. - -- The telnet module now has a my_interact() method which uses threads -instead of select. The interact() method uses this by default on -Windows (where the single-threaded version doesn't work). - -- Add a class to mailbox.py for dealing with qmail directory -mailboxes. The test code was extended to notice these being used as -well. - -Changes to extension modules ----------------------------- - -- Support for the [f]statvfs() system call, where it exists. - -- Fixed some bugs in cPickle where bad input could cause it to dump -core. - -- Fixed cStringIO to make the writelines() function actually work. - -- Added strop.expandtabs() so string.expandtabs() is now much faster. - -- Added fsync() and fdatasync(), if they appear to exist. - -- Support for "long files" (64-bit seek pointers). - -- Fixed a bug in the zlib module's flush() function. - -- Added access() system call. It returns 1 if access granted, 0 if -not. - -- The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) - -Changes to tools ----------------- - -- Some changes to IDLE; see Tools/idle/NEWS.txt. - -- Latest version of Misc/python-mode.el included. - -Changes to Tkinter ------------------- - -- Avoid tracebacks when an image is deleted after its root has been -destroyed. - -Changes to the Python/C API ---------------------------- - -- When parentheses are used in a PyArg_Parse[Tuple]() call, any -sequence is now accepted, instead of requiring a tuple. This is in -line with the general trend towards accepting arbitrary sequences. - -- Added PyModule_GetFilename(). - -- In PyNumber_Power(), remove unneeded and even harmful test for float -to the negative power (which is already and better done in -floatobject.c). - -- New version identification symbols; read patchlevel.h for info. The -version numbers are now exported by Python.h. - -- Rolled back the API version change -- it's back to 1007! - -- The frozenmain.c function calls PyInitFrozenExtensions(). - -- Added 'N' format character to Py_BuildValue -- like 'O' but doesn't -INCREF. - - -====================================================================== - - -From 1.5.2a2 to 1.5.2b1 -======================= - -Changes to intrinsics ---------------------- - -- New extension NotImplementedError, derived from RuntimeError. Not -used, but recommended use is for "abstract" methods to raise this. - -- The parser will now spit out a warning or error when -t or -tt is -used for parser input coming from a string, too. - -- The code generator now inserts extra SET_LINENO opcodes when -compiling multi-line argument lists. - -- When comparing bound methods, use identity test on the objects, not -equality test. - -New or improved ports ---------------------- - -- Chris Herborth has redone his BeOS port; it now works on PowerPC -(R3/R4) and x86 (R4 only). Threads work too in this port. - -Renaming --------- - -- Thanks to Chris Herborth, the thread primitives now have proper Py* -names in the source code (they already had those for the linker, -through some smart macros; but the source still had the old, un-Py -names). - -Configuration/build changes ---------------------------- - -- Improved support for FreeBSD/3. - -- Check for pthread_detach instead of pthread_create in libc. - -- The makesetup script now searches EXECINCLUDEPY before INCLUDEPY. - -- Misc/Makefile.pre.in now also looks at Setup.thread and Setup.local. -Otherwise modules such as thread didn't get incorporated in extensions. - -New library modules -------------------- - -- shlex.py by Eric Raymond provides a lexical analyzer class for -simple shell-like syntaxes. - -- netrc.py by Eric Raymond provides a parser for .netrc files. (The -undocumented Netrc class in ftplib.py is now obsolete.) - -- codeop.py is a new module that contains the compile_command() -function that was previously in code.py. This is so that JPython can -provide its own version of this function, while still sharing the -higher-level classes in code.py. - -- turtle.py is a new module for simple turtle graphics. I'm still -working on it; let me know if you use this to teach Python to children -or other novices without prior programming experience. - -Obsoleted library modules -------------------------- - -- poly.py and zmod.py have been moved to Lib/lib-old to emphasize -their status of obsoleteness. They don't do a particularly good job -and don't seem particularly relevant to the Python core. - -New tools ---------- - -- I've added IDLE: my Integrated DeveLopment Environment for Python. -Requires Tcl/Tk (and Tkinter). Works on Windows and Unix (and should -work on Macintosh, but I haven't been able to test it there; it does -depend on new features in 1.5.2 and perhaps even new features in -1.5.2b1, especially the new code module). This is very much a work in -progress. I'd like to hear how people like it compared to PTUI (or -any other IDE they are familiar with). - -- New tools by Barry Warsaw: - - = audiopy: controls the Solaris Audio device - = pynche: The PYthonically Natural Color and Hue Editor - = world: Print mappings between country names and DNS country codes - -New demos ---------- - -- Demo/scripts/beer.py prints the lyrics to an arithmetic drinking -song. - -- Demo/tkinter/guido/optionmenu.py shows how to do an option menu in -Tkinter. (By Fredrik Lundh -- not by me!) - -Changes to the library ----------------------- - -- compileall.py now avoids recompiling .py files that haven't changed; -it adds a -f option to force recompilation. - -- New version of xmllib.py by Sjoerd Mullender (0.2 with latest -patches). - -- nntplib.py: statparse() no longer lowercases the message-id. - -- types.py: use type(__stdin__) for FileType. - -- urllib.py: fix translations for filenames with "funny" characters. -Patch by Sjoerd Mullender. Note that if you subclass one of the -URLopener classes, and you have copied code from the old urllib.py, -your subclass may stop working. A long-term solution is to provide -more methods so that you don't have to copy code. - -- cgi.py: In read_multi, allow a subclass to override the class we -instantiate when we create a recursive instance, by setting the class -variable 'FieldStorageClass' to the desired class. By default, this -is set to None, in which case we use self.__class__ (as before). -Also, a patch by Jim Fulton to pass additional arguments to recursive -calls to the FieldStorage constructor from its read_multi method. - -- UserList.py: In __getslice__, use self.__class__ instead of -UserList. - -- In SimpleHTTPServer.py, the server specified in test() should be -BaseHTTPServer.HTTPServer, in case the request handler should want to -reference the two attributes added by BaseHTTPServer.server_bind. (By -Jeff Rush, for Bobo). Also open the file in binary mode, so serving -images from a Windows box might actually work. - -- In CGIHTTPServer.py, the list of acceptable formats is -split- -on spaces but -joined- on commas, resulting in double commas -in the joined text. (By Jeff Rush.) - -- SocketServer.py, patch by Jeff Bauer: a minor change to declare two -new threaded versions of Unix Server classes, using the ThreadingMixIn -class: ThreadingUnixStreamServer, ThreadingUnixDatagramServer. - -- bdb.py: fix bomb on deleting a temporary breakpoint: there's no -method do_delete(); do_clear() was meant. By Greg Ward. - -- getopt.py: accept a non-list sequence for the long options (request -by Jack Jansen). Because it might be a common mistake to pass a -single string, this situation is treated separately. Also added -docstrings (copied from the library manual) and removed the (now -redundant) module comments. - -- tempfile.py: improvements to avoid security leaks. - -- code.py: moved compile_command() to new module codeop.py. - -- pickle.py: support pickle format 1.3 (binary float added). By Jim -Fulton. Also get rid of the undocumented obsolete Pickler dump_special -method. - -- uu.py: Move 'import sys' to top of module, as noted by Tim Peters. - -- imaplib.py: fix problem with some versions of IMAP4 servers that -choose to mix the case in their CAPABILITIES response. - -- cmp.py: use (f1, f2) as cache key instead of f1 + ' ' + f2. Noted -by Fredrik Lundh. - -Changes to extension modules ----------------------------- - -- More doc strings for several modules were contributed by Chris -Petrilli: math, cmath, fcntl. - -- Fixed a bug in zlibmodule.c that could cause core dumps on -decompression of rarely occurring input. - -- cPickle.c: new version from Jim Fulton, with Open Source copyright -notice. Also, initialize self->safe_constructors early on to prevent -crash in early dealloc. - -- cStringIO.c: new version from Jim Fulton, with Open Source copyright -notice. Also fixed a core dump in cStringIO.c when doing seeks. - -- mpzmodule.c: fix signed character usage in mpz.mpz(stringobjecty). - -- readline.c: Bernard Herzog pointed out that rl_parse_and_bind -modifies its argument string (bad function!), so we make a temporary -copy. - -- sunaudiodev.c: Barry Warsaw added more smarts to get the device and -control pseudo-device, per audio(7I). - -Changes to tools ----------------- - -- New, improved version of Barry Warsaw's Misc/python-mode.el (editing -support for Emacs). - -- tabnanny.py: added a -q ('quiet') option to tabnanny, which causes -only the names of offending files to be printed. - -- freeze: when printing missing modules, also print the module they -were imported from. - -- untabify.py: patch by Detlef Lannert to implement -t option -(set tab size). - -Changes to Tkinter ------------------- - -- grid_bbox(): support new Tk API: grid bbox ?column row? ?column2 -row2? - -- _tkinter.c: RajGopal Srinivasan noted that the latest code (1.5.2a2) -doesn't work when running in a non-threaded environment. He added -some #ifdefs that fix this. - -Changes to the Python/C API ---------------------------- - -- Bumped API version number to 1008 -- enough things have changed! - -- There's a new macro, PyThreadState_GET(), which does the same work -as PyThreadState_Get() without the overhead of a function call (it -also avoids the error check). The two top calling locations of -PyThreadState_Get() have been changed to use this macro. - -- All symbols intended for export from a DLL or shared library are now -marked as such (with the DL_IMPORT() macro) in the header file that -declares them. This was needed for the BeOS port, and should also -make some other ports easier. The PC port no longer needs the file -with exported symbols (PC/python_nt.def). There's also a DL_EXPORT -macro which is only used for init methods in extension modules, and -for Py_Main(). - -Invisible changes to internals ------------------------------- - -- Fixed a bug in new_buffersize() in fileobject.c which could -return a buffer size that was way too large. - -- Use PySys_WriteStderr instead of fprintf in most places. - -- dictobject.c: remove dead code discovered by Vladimir Marangozov. - -- tupleobject.c: make tuples less hungry -- an extra item was -allocated but never used. Tip by Vladimir Marangozov. - -- mymath.h: Metrowerks PRO4 finally fixes the hypot snafu. (Jack -Jansen) - -- import.c: Jim Fulton fixes a reference count bug in -PyEval_GetGlobals. - -- glmodule.c: check in the changed version after running the stubber -again -- this solves the conflict with curses over the 'clear' entry -point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) - -- frameobject.c: fix reference count bug in PyFrame_New. Vladimir -Marangozov. - -- stropmodule.c: add a missing DECREF in an error exit. Submitted by -Jonathan Giddy. - - -====================================================================== - - -From 1.5.2a1 to 1.5.2a2 -======================= - -General -------- - -- It is now a syntax error to have a function argument without a -default following one with a default. - -- __file__ is now set to the .py file if it was parsed (it used to -always be the .pyc/.pyo file). - -- Don't exit with a fatal error during initialization when there's a -problem with the exceptions.py module. - -- New environment variable PYTHONOPTIMIZE can be used to set -O. - -- New version of python-mode.el for Emacs. - -Miscellaneous fixed bugs ------------------------- - -- No longer print the (confusing) error message about stack underflow -while compiling. - -- Some threading and locking bugs fixed. - -- When errno is zero, report "Error", not "Success". - -Documentation -------------- - -- Documentation will be released separately. - -- Doc strings added to array and md5 modules by Chris Petrilli. - -Ports and build procedure -------------------------- - -- Stop installing when a move or copy fails. - -- New version of the OS/2 port code by Jeff Rush. - -- The makesetup script handles absolute filenames better. - -- The 'new' module is now enabled by default in the Setup file. - -- I *think* I've solved the problem with the Linux build blowing up -sometimes due to a conflict between sigcheck/intrcheck and -signalmodule. - -Built-in functions ------------------- - -- The second argument to apply() can now be any sequence, not just a -tuple. - -Built-in types --------------- - -- Lists have a new method: L1.extend(L2) is equivalent to the common -idiom L1[len(L1):] = L2. - -- Better error messages when a sequence is indexed with a non-integer. - -- Bettter error message when calling a non-callable object (include -the type in the message). - -Python services ---------------- - -- New version of cPickle.c fixes some bugs. - -- pickle.py: improved instantiation error handling. - -- code.py: reworked quite a bit. New base class -InteractiveInterpreter and derived class InteractiveConsole. Fixed -several problems in compile_command(). - -- py_compile.py: print error message and continue on syntax errors. -Also fixed an old bug with the fstat code (it was never used). - -- pyclbr.py: support submodules of packages. - -String Services ---------------- - -- StringIO.py: raise the right exception (ValueError) for attempted -I/O on closed StringIO objects. - -- re.py: fixed a bug in subn(), which caused .groups() to fail inside -the replacement function called by sub(). - -- The struct module has a new format 'P': void * in native mode. - -Generic OS Services -------------------- - -- Module time: Y2K robustness. 2-digit year acceptance depends on -value of time.accept2dyear, initialized from env var PYTHONY2K, -default 0. Years 00-68 mean 2000-2068, while 69-99 mean 1969-1999 -(POSIX or X/Open recommendation). - -- os.path: normpath(".//x") should return "x", not "/x". - -- getpass.py: fall back on default_getpass() when sys.stdin.fileno() -doesn't work. - -- tempfile.py: regenerate the template after a fork() call. - -Optional OS Services --------------------- - -- In the signal module, disable restarting interrupted system calls -when we have siginterrupt(). - -Debugger --------- - -- No longer set __args__; this feature is no longer supported and can -affect the debugged code. - -- cmd.py, pdb.py and bdb.py have been overhauled by Richard Wolff, who -added aliases and some other useful new features, e.g. much better -breakpoint support: temporary breakpoint, disabled breakpoints, -breakpoints with ignore counts, and conditions; breakpoints can be set -on a file before it is loaded. - -Profiler --------- - -- Changes so that JPython can use it. Also fix the calibration code -so it actually works again -. -Internet Protocols and Support ------------------------------- - -- imaplib.py: new version from Piers Lauder. - -- smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). - -- poplib.py: LIST with a msg argument fixed. - -- urlparse.py: some optimizations for common case (http). - -- urllib.py: support content-length in info() for ftp protocol; -support for a progress meter through a third argument to -urlretrieve(); commented out gopher test (the test site is dead). - -Internet Data handling ----------------------- - -- sgmllib.py: support tags with - or . in their name. - -- mimetypes.py: guess_type() understands 'data' URLs. - -Restricted Execution --------------------- - -- The classes rexec.RModuleLoader and rexec.RModuleImporter no -longer exist. - -Tkinter -------- - -- When reporting an exception, store its info in sys.last_*. Also, -write all of it to stderr. - -- Added NS, EW, and NSEW constants, for grid's sticky option. - -- Fixed last-minute bug in 1.5.2a1 release: need to include "mytime.h". - -- Make bind variants without a sequence return a tuple of sequences -(formerly it returned a string, which wasn't very convenient). - -- Add image commands to the Text widget (these are new in Tk 8.0). - -- Added new listbox and canvas methods: {xview,yview}_{scroll,moveto}.) - -- Improved the thread code (but you still can't call update() from -another thread on Windows). - -- Fixed unnecessary references to _default_root in the new dialog -modules. - -- Miscellaneous problems fixed. - - -Windows General ---------------- - -- Call LoadLibraryEx(..., ..., LOAD_WITH_ALTERED_SEARCH_PATH) to -search for dependent dlls in the directory containing the .pyd. - -- In debugging mode, call DebugBreak() in Py_FatalError(). - -Windows Installer ------------------ - -- Install zlib.dll in the DLLs directory instead of in the win32 -system directory, to avoid conflicts with other applications that have -their own zlib.dll. - -Test Suite ----------- - -- test_long.py: new test for long integers, by Tim Peters. - -- regrtest.py: improved so it can be used for other test suites as -well. - -- test_strftime.py: use re to compare test results, to support legal -variants (e.g. on Linux). - -Tools and Demos ---------------- - -- Four new scripts in Tools/scripts: crlf.py and lfcr.py (to -remove/add Windows style '\r\n' line endings), untabify.py (to remove -tabs), and rgrep.yp (reverse grep). - -- Improvements to Tools/freeze/. Each Python module is now written to -its own C file. This prevents some compilers or assemblers from -blowing up on large frozen programs, and saves recompilation time if -only a few modules are changed. Other changes too, e.g. new command -line options -x and -i. - -- Much improved (and smaller!) version of Tools/scripts/mailerdaemon.py. - -Python/C API ------------- - -- New mechanism to support extensions of the type object while -remaining backward compatible with extensions compiled for previous -versions of Python 1.5. A flags field indicates presence of certain -fields. - -- Addition to the buffer API to differentiate access to bytes and -8-bit characters (in anticipation of Unicode characters). - -- New argument parsing format t# ("text") to indicate 8-bit -characters; s# simply means 8-bit bytes, for backwards compatibility. - -- New object type, bufferobject.c is an example and can be used to -create buffers from memory. - -- Some support for 64-bit longs, including some MS platforms. - -- Many calls to fprintf(stderr, ...) have been replaced with calls to -PySys_WriteStderr(...). - -- The calling context for PyOS_Readline() has changed: it must now be -called with the interpreter lock held! It releases the lock around -the call to the function pointed to by PyOS_ReadlineFunctionPointer -(default PyOS_StdioReadline()). - -- New APIs PyLong_FromVoidPtr() and PyLong_AsVoidPtr(). - -- Renamed header file "thread.h" to "pythread.h". - -- The code string of code objects may now be anything that supports the -buffer API. - - -====================================================================== - - -From 1.5.1 to 1.5.2a1 -===================== - -General -------- - -- When searching for the library, a landmark that is a compiled module -(string.pyc or string.pyo) is also accepted. - -- When following symbolic links to the python executable, use a loop -so that a symlink to a symlink can work. - -- Added a hack so that when you type 'quit' or 'exit' at the -interpreter, you get a friendly explanation of how to press Ctrl-D (or -Ctrl-Z) to exit. - -- New and improved Misc/python-mode.el (Python mode for Emacs). - -- Revert a new feature in Unix dynamic loading: for one or two -revisions, modules were loaded using the RTLD_GLOBAL flag. It turned -out to be a bad idea. - -Miscellaneous fixed bugs ------------------------- - -- All patches on the patch page have been integrated. (But much more -has been done!) - -- Several memory leaks plugged (e.g. the one for classes with a -__getattr__ method). - -- Removed the only use of calloc(). This triggered an obscure bug on -multiprocessor Sparc Solaris 2.6. - -- Fix a peculiar bug that would allow "import sys.time" to succeed -(believing the built-in time module to be a part of the sys package). - -- Fix a bug in the overflow checking when converting a Python long to -a C long (failed to convert -2147483648L, and some other cases). - -Documentation -------------- - -- Doc strings have been added to many extension modules: __builtin__, -errno, select, signal, socket, sys, thread, time. Also to methods of -list objects (try [].append.__doc__). A doc string on a type will now -automatically be propagated to an instance if the instance has methods -that are accessed in the usual way. - -- The documentation has been expanded and the formatting improved. -(Remember that the documentation is now unbundled and has its own -release cycle though; see http://www.python.org/doc/.) - -- Added Misc/Porting -- a mini-FAQ on porting to a new platform. - -Ports and build procedure -------------------------- - -- The BeOS port is now integrated. Courtesy Chris Herborth. - -- Symbol files for FreeBSD 2.x and 3.x have been contributed -(Lib/plat-freebsd[23]/*). - -- Support HPUX 10.20 DCE threads. - -- Finally fixed the configure script so that (on SGI) if -OPT:Olimit=0 -works, it won't also use -Olimit 1500 (which gives a warning for every -file). Also support the SGI_ABI environment variable better. - -- The makesetup script now understands absolute pathnames ending in .o -in the module -- it assumes it's a file for which we have no source. - -- Other miscellaneous improvements to the configure script and -Makefiles. - -- The test suite now uses a different sound sample. - -Built-in functions ------------------- - -- Better checks for invalid input to int(), long(), string.atoi(), -string.atol(). (Formerly, a sign without digits would be accepted as -a legal ways to spell zero.) - -- Changes to map() and filter() to use the length of a sequence only -as a hint -- if an IndexError happens earlier, take that. (Formerly, -this was considered an error.) - -- Experimental feature in getattr(): a third argument can specify a -default (instead of raising AttributeError). - -- Implement round() slightly different, so that for negative ndigits -no additional errors happen in the last step. - -- The open() function now adds the filename to the exception when it -fails. - -Built-in exceptions -------------------- - -- New standard exceptions EnvironmentError and PosixError. -EnvironmentError is the base class for IOError and PosixError; -PosixError is the same as os.error. All this so that either exception -class can be instantiated with a third argument indicating a filename. -The built-in function open() and most os/posix functions that take a -filename argument now use this. - -Built-in types --------------- - -- List objects now have an experimental pop() method; l.pop() returns -and removes the last item; l.pop(i) returns and removes the item at -i. Also, the sort() method is faster again. Sorting is now also -safer: it is impossible for the sorting function to modify the list -while the sort is going on (which could cause core dumps). - -- Changes to comparisons: numbers are now smaller than any other type. -This is done to prevent the circularity where [] < 0L < 1 < [] is -true. As a side effect, cmp(None, 0) is now positive instead of -negative. This *shouldn't* affect any working code, but I've found -that the change caused several "sleeping" bugs to become active, so -beware! - -- Instance methods may now have other callable objects than just -Python functions as their im_func. Use new.instancemethod() or write -your own C code to create them; new.instancemethod() may be called -with None for the instance to create an unbound method. - -- Assignment to __name__, __dict__ or __bases__ of a class object is -now allowed (with stringent type checks); also allow assignment to -__getattr__ etc. The cached values for __getattr__ etc. are -recomputed after such assignments (but not for derived classes :-( ). - -- Allow assignment to some attributes of function objects: func_code, -func_defaults and func_doc / __doc__. (With type checks except for -__doc__ / func_doc .) - -Python services ---------------- - -- New tests (in Lib/test): reperf.py (regular expression benchmark), -sortperf.py (list sorting benchmark), test_MimeWriter.py (test case -for the MimeWriter module). - -- Generalized test/regrtest.py so that it is useful for testing other -packages. - -- The ihooks.py module now understands package imports. - -- In code.py, add a class that subsumes Fredrik Lundh's -PythonInterpreter class. The interact() function now uses this. - -- In rlcompleter.py, in completer(), return None instead of raising an -IndexError when there are no more completions left. - -- Fixed the marshal module to test for certain common kinds of invalid -input. (It's still not foolproof!) - -- In the operator module, add an alias (now the preferred name) -"contains" for "sequenceincludes". - -String Services ---------------- - -- In the string and strop modules, in the replace() function, treat an -empty pattern as an error (since it's not clear what was meant!). - -- Some speedups to re.py, especially the string substitution and split -functions. Also added new function/method findall(), to find all -occurrences of a given substring. - -- In cStringIO, add better argument type checking and support the -readonly 'closed' attribute (like regular files). - -- In the struct module, unsigned 1-2 byte sized formats no longer -result in long integer values. - -Miscellaneous services ----------------------- - -- In whrandom.py, added new method and function randrange(), same as -choice(range(start, stop, step)) but faster. This addresses the -problem that randint() was accidentally defined as taking an inclusive -range. Also, randint(a, b) is now redefined as randrange(a, b+1), -adding extra range and type checking to its arguments! - -- Add some semi-thread-safety to random.gauss() (it used to be able to -crash when invoked from separate threads; now the worst it can do is -give a duplicate result occasionally). - -- Some restructuring and generalization done to cmd.py. - -- Major upgrade to ConfigParser.py; converted to using 're', added new -exceptions, support underscore in section header and option name. No -longer add 'name' option to every section; instead, add '__name__'. - -- In getpass.py, don't use raw_input() to ask for the password -- we -don't want it to show up in the readline history! Also don't catch -interrupts (the try-finally already does all necessary cleanup). - -Generic OS Services -------------------- - -- New functions in os.py: makedirs(), removedirs(), renames(). New -variable: linesep (the line separator as found in binary files, -i.e. '\n' on Unix, '\r\n' on DOS/Windows, '\r' on Mac. Do *not* use -this with files opened in (default) text mode; the line separator used -will always be '\n'! - -- Changes to the 'os.path' submodule of os.py: added getsize(), -getmtime(), getatime() -- these fetch the most popular items from the -stat return tuple. - -- In the time module, add strptime(), if it exists. (This parses a -time according to a format -- the inverse of strftime().) Also, -remove the call to mktime() from strftime() -- it messed up the -formatting of some non-local times. - -- In the socket module, added a new function gethostbyname_ex(). -Also, don't use #ifdef to test for some symbols that are enums on some -platforms (and should exist everywhere). - -Optional OS Services --------------------- - -- Some fixes to gzip.py. In particular, the readlines() method now -returns the lines *with* trailing newline characters, like readlines() -of regular file objects. Also, it didn't work together with cPickle; -fixed that. - -- In whichdb.py, support byte-swapped dbhash (bsddb) files. - -- In anydbm.py, look at the type of an existing database to determine -which module to use to open it. (The anydbm.error exception is now a -tuple.) - -Unix Services -------------- - -- In the termios module, in tcsetattr(), initialize the structure vy -calling tcgetattr(). - -- Added some of the "wait status inspection" macros as functions to -the posix module (and thus to the os module): WEXITSTATUS(), -WIFEXITED(), WIFSIGNALED(), WIFSTOPPED(), WSTOPSIG(), WTERMSIG(). - -- In the syslog module, make the default facility more intuitive -(matching the docs). - -Debugger --------- - -- In pdb.py, support for setting breaks on files/modules that haven't -been loaded yet. - -Internet Protocols and Support ------------------------------- - -- Changes in urllib.py; sped up unquote() and quote(). Fixed an -obscure bug in quote_plus(). Added urlencode(dict) -- convenience -function for sending a POST request with urlopen(). Use the getpass -module to ask for a password. Rewrote the (test) main program so that -when used as a script, it can retrieve one or more URLs to stdout. -Use -t to run the self-test. Made the proxy code work again. - -- In cgi.py, treat "HEAD" the same as "GET", so that CGI scripts don't -fail when someone asks for their HEAD. Also, for POST, set the -default content-type to application/x-www-form-urlencoded. Also, in -FieldStorage.__init__(), when method='GET', always get the query -string from environ['QUERY_STRING'] or sys.argv[1] -- ignore an -explicitly passed in fp. - -- The smtplib.py module now supports ESMTP and has improved standard -compliance, for picky servers. - -- Improved imaplib.py. - -- Fixed UDP support in SocketServer.py (it never worked). - -- Fixed a small bug in CGIHTTPServer.py. - -Internet Data handling ----------------------- - -- In rfc822.py, add a new class AddressList. Also support a new -overridable method, isheader(). Also add a get() method similar to -dictionaries (and make getheader() an alias for it). Also, be smarter -about seekable (test whether fp.tell() works) and test for presence of -unread() method before trying seeks. - -- In sgmllib.py, restore the call to report_unbalanced() that was lost -long ago. Also some other improvements: handle , allow . and - in entity names, and allow \r\n as line -separator. - -- Some restructuring and generalization done to multifile.py; support -a 'seekable' flag. - -Restricted Execution --------------------- - -- Improvements to rexec.py: package support; support a (minimal) -sys.exc_info(). Also made the (test) main program a bit fancier (you -can now use it to run arbitrary Python scripts in restricted mode). - -Tkinter -------- - -- On Unix, Tkinter can now safely be used from a multi-threaded -application. (Formerly, no threads would make progress while -Tkinter's mainloop() was active, because it didn't release the Python -interpreter lock.) Unfortunately, on Windows, threads other than the -main thread should not call update() or update_idletasks() because -this will deadlock the application. - -- An interactive interpreter that uses readline and Tkinter no longer -uses up all available CPU time. - -- Even if readline is not used, Tk windows created in an interactive -interpreter now get continuously updated. (This even works in Windows -as long as you don't hit a key.) - -- New demos in Demo/tkinter/guido/: brownian.py, redemo.py, switch.py. - -- No longer register Tcl_finalize() as a low-level exit handler. It -may call back into Python, and that's a bad idea. - -- Allow binding of Tcl commands (given as a string). - -- Some minor speedups; replace explicitly coded getint() with int() in -most places. - -- In FileDialog.py, remember the directory of the selected file, if -given. - -- Change the names of all methods in the Wm class: they are now -wm_title(), etc. The old names (title() etc.) are still defined as -aliases. - -- Add a new method of interpreter objects, interpaddr(). This returns -the address of the Tcl interpreter object, as an integer. Not very -useful for the Python programmer, but this can be called by another C -extension that needs to make calls into the Tcl/Tk C API and needs to -get the address of the Tcl interpreter object. A simple cast of the -return value to (Tcl_Interp *) will do the trick. - -Windows General ---------------- - -- Don't insist on proper case for module source files if the filename -is all uppercase (e.g. FOO.PY now matches foo; but FOO.py still -doesn't). This should address problems with this feature on -oldfashioned filesystems (Novell servers?). - -Windows Library ---------------- - -- os.environ is now all uppercase, but accesses are case insensitive, -and the putenv() calls made as a side effect of changing os.environ -are case preserving. - -- Removed samefile(), sameopenfile(), samestat() from os.path (aka -ntpath.py) -- these cannot be made to work reliably (at least I -wouldn't know how). - -- Fixed os.pipe() so that it returns file descriptors acceptable to -os.read() and os.write() (like it does on Unix), rather than Windows -file handles. - -- Added a table of WSA error codes to socket.py. - -- In the select module, put the (huge) file descriptor arrays on the -heap. - -- The getpass module now raises KeyboardInterrupt when it sees ^C. - -- In mailbox.py, fix tell/seek when using files opened in text mode. - -- In rfc822.py, fix tell/seek when using files opened in text mode. - -- In the msvcrt extension module, release the interpreter lock for -calls that may block: _locking(), _getch(), _getche(). Also fix a -bogus error return when open_osfhandle() doesn't have the right -argument list. - -Windows Installer ------------------ - -- The registry key used is now "1.5" instead of "1.5.x" -- so future -versions of 1.5 and Mark Hammond's win32all installer don't need to be -resynchronized. - -Windows Tools -------------- - -- Several improvements to freeze specifically for Windows. - -Windows Build Procedure ------------------------ - -- The VC++ project files and the WISE installer have been moved to the -PCbuild subdirectory, so they are distributed in the same subdirectory -where they must be used. This avoids confusion. - -- New project files for Windows 3.1 port by Jim Ahlstrom. - -- Got rid of the obsolete subdirectory PC/setup_nt/. - -- The projects now use distinct filenames for the .exe, .dll, .lib and -.pyd files built in debug mode (by appending "_d" to the base name, -before the extension). This makes it easier to switch between the two -and get the right versions. There's a pragma in config.h that directs -the linker to include the appropriate .lib file (so python15.lib no -longer needs to be explicit in your project). - -- The installer now installs more files (e.g. config.h). The idea is -that you shouldn't need the source distribution if you want build your -own extensions in C or C++. - -Tools and Demos ---------------- - -- New script nm2def.py by Marc-Andre Lemburg, to construct -PC/python_nt.def automatically (some hand editing still required). - -- New tool ndiff.py: Tim Peters' text diffing tool. - -- Various and sundry improvements to the freeze script. - -- The script texi2html.py (which was part of the Doc tree but is no -longer used there) has been moved to the Tools/scripts subdirectory. - -- Some generalizations in the webchecker code. There's now a -primnitive gui for websucker.py: wsgui.py. (In Tools/webchecker/.) - -- The ftpmirror.py script now handles symbolic links properly, and -also files with multiple spaces in their names. - -- The 1.5.1 tabnanny.py suffers an assert error if fed a script whose -last line is both indented and lacks a newline. This is now fixed. - -Python/C API ------------- - -- Added missing prototypes for PyEval_CallFunction() and -PyEval_CallMethod(). - -- New macro PyList_SET_ITEM(). - -- New macros to access object members for PyFunction, PyCFunction -objects. - -- New APIs PyImport_AppendInittab() and PyImport_ExtendInittab() to -dynamically add one or many entries to the table of built-in modules. - -- New macro Py_InitModule3(name, methods, doc) which calls -Py_InitModule4() with appropriate arguments. (The -4 variant requires -you to pass an obscure version number constant which is always the same.) - -- New APIs PySys_WriteStdout() and PySys_WriteStderr() to write to -sys.stdout or sys.stderr using a printf-like interface. (Used in -_tkinter.c, for example.) - -- New APIs for conversion between Python longs and C 'long long' if -your compiler supports it. - -- PySequence_In() is now called PySequence_Contains(). -(PySequence_In() is still supported for b/w compatibility; it is -declared obsolete because its argument order is confusing.) - -- PyDict_GetItem() and PyDict_GetItemString() are changed so that they -*never* raise an exception -- (even if the hash() fails, simply clear -the error). This was necessary because there is lots of code out -there that already assumes this. - -- Changes to PySequence_Tuple() and PySequence_List() to use the -length of a sequence only as a hint -- if an IndexError happens -earlier, take that. (Formerly, this was considered an error.) - -- Reformatted abstract.c to give it a more familiar "look" and fixed -many error checking bugs. - -- Add NULL pointer checks to all calls of a C function through a type -object and extensions (e.g. nb_add). - -- The code that initializes sys.path now calls Py_GetPythonHome() -instead of getenv("PYTHONHOME"). This, together with the new API -Py_SetPythonHome(), makes it easier for embedding applications to -change the notion of Python's "home" directory (where the libraries -etc. are sought). - -- Fixed a very old bug in the parsing of "O?" format specifiers. - - -====================================================================== - - -======================================== -==> Release 1.5.1 (October 31, 1998) <== -======================================== - -From 1.5 to 1.5.1 -================= - -General -------- - -- The documentation is now unbundled. It has also been extensively -modified (mostly to implement a new and more uniform formatting -style). We figure that most people will prefer to download one of the -preformatted documentation sets (HTML, PostScript or PDF) and that -only a minority have a need for the LaTeX or FrameMaker sources. Of -course, the unbundled documentation sources still released -- just not -in the same archive file, and perhaps not on the same date. - -- All bugs noted on the errors page (and many unnoted) are fixed. All -new bugs take their places. - -- No longer a core dump when attempting to print (or repr(), or str()) -a list or dictionary that contains an instance of itself; instead, the -recursive entry is printed as [...] or {...}. See Py_ReprEnter() and -Py_ReprLeave() below. Comparisons of such objects still go beserk, -since this requires a different kind of fix; fortunately, this is a -less common scenario in practice. - -Syntax change -------------- - -- The raise statement can now be used without arguments, to re-raise -a previously set exception. This should be used after catching an -exception with an except clause only, either in the except clause or -later in the same function. - -Import and module handling --------------------------- - -- The implementation of import has changed to use a mutex (when -threading is supported). This means that when two threads -simultaneously import the same module, the import statements are -serialized. Recursive imports are not affected. - -- Rewrote the finalization code almost completely, to be much more -careful with the order in which modules are destroyed. Destructors -will now generally be able to reference built-in names such as None -without trouble. - -- Case-insensitive platforms such as Mac and Windows require the case -of a module's filename to match the case of the module name as -specified in the import statement (see below). - -- The code for figuring out the default path now distinguishes between -files, modules, executable files, and directories. When expecting a -module, we also look for the .pyc or .pyo file. - -Parser/tokenizer changes ------------------------- - -- The tokenizer can now warn you when your source code mixes tabs and -spaces for indentation in a manner that depends on how much a tab is -worth in spaces. Use "python -t" or "python -v" to enable this -option. Use "python -tt" to turn the warnings into errors. (See also -tabnanny.py and tabpolice.py below.) - -- Return unsigned characters from tok_nextc(), so '\377' isn't -mistaken for an EOF character. - -- Fixed two pernicious bugs in the tokenizer that only affected AIX. -One was actually a general bug that was triggered by AIX's smaller I/O -buffer size. The other was a bug in the AIX optimizer's loop -unrolling code; swapping two statements made the problem go away. - -Tools, demos and miscellaneous files ------------------------------------- - -- There's a new version of Misc/python-mode.el (the Emacs mode for -Python) which is much smarter about guessing the indentation style -used in a particular file. Lots of other cool features too! - -- There are two new tools in Tools/scripts: tabnanny.py and -tabpolice.py, implementing two different ways of checking whether a -file uses indentation in a way that is sensitive to the interpretation -of a tab. The preferred module is tabnanny.py (by Tim Peters). - -- Some new demo programs: - - Demo/tkinter/guido/paint.py -- Dave Mitchell - Demo/sockets/unixserver.py -- Piet van Oostrum - - -- Much better freeze support. The freeze script can now freeze -hierarchical module names (with a corresponding change to import.c), -and has a few extra options (e.g. to suppress freezing specific -modules). It also does much more on Windows NT. - -- Version 1.0 of the faq wizard is included (only very small changes -since version 0.9.0). - -- New feature for the ftpmirror script: when removing local files -(i.e., only when -r is used), do a recursive delete. - -Configuring and building Python -------------------------------- - -- Get rid of the check for -linet -- recent Sequent Dynix systems don't -need this any more and apparently it screws up their configuration. - -- Some changes because gcc on SGI doesn't support '-all'. - -- Changed the build rules to use $(LIBRARY) instead of - -L.. -lpython$(VERSION) -since the latter trips up the SunOS 4.1.x linker (sigh). - -- Fix the bug where the '# dgux is broken' comment in the Makefile -tripped over Make on some platforms. - -- Changes for AIX: install the python.exp file; properly use -$(srcdir); the makexp_aix script now removes C++ entries of the form -Class::method. - -- Deleted some Makefile targets only used by the (long obsolete) -gMakefile hacks. - -Extension modules ------------------ - -- Performance and threading improvements to the socket and bsddb -modules, by Christopher Lindblad of Infoseek. - -- Added operator.__not__ and operator.not_. - -- In the thread module, when a thread exits due to an unhandled -exception, don't store the exception information in sys.last_*; it -prevents proper calling of destructors of local variables. - -- Fixed a number of small bugs in the cPickle module. - -- Changed find() and rfind() in the strop module so that -find("x","",2) returns -1, matching the implementation in string.py. - -- In the time module, be more careful with the result of ctime(), and -test for HAVE_MKTIME before usinmg mktime(). - -- Doc strings contributed by Mitch Chapman to the termios, pwd, gdbm -modules. - -- Added the LOG_SYSLOG constant to the syslog module, if defined. - -Standard library modules ------------------------- - -- All standard library modules have been converted to an indentation -style using either only tabs or only spaces -- never a mixture -- if -they weren't already consistent according to tabnanny. This means -that the new -t option (see above) won't complain about standard -library modules. - -- New standard library modules: - - threading -- GvR and the thread-sig - Java style thread objects -- USE THIS!!! - - getpass -- Piers Lauder - simple utilities to prompt for a password and to - retrieve the current username - - imaplib -- Piers Lauder - interface for the IMAP4 protocol - - poplib -- David Ascher, Piers Lauder - interface for the POP3 protocol - - smtplib -- Dragon De Monsyne - interface for the SMTP protocol - -- Some obsolete modules moved to a separate directory (Lib/lib-old) -which is *not* in the default module search path: - - Para - addpack - codehack - fmt - lockfile - newdir - ni - rand - tb - -- New version of the PCRE code (Perl Compatible Regular Expressions -- -the re module and the supporting pcre extension) by Andrew Kuchling. -Incompatible new feature in re.sub(): the handling of escapes in the -replacement string has changed. - -- Interface change in the copy module: a __deepcopy__ method is now -called with the memo dictionary as an argument. - -- Feature change in the tokenize module: differentiate between NEWLINE -token (an official newline) and NL token (a newline that the grammar -ignores). - -- Several bugfixes to the urllib module. It is now truly thread-safe, -and several bugs and a portability problem have been fixed. New -features, all due to Sjoerd Mullender: When creating a temporary file, -it gives it an appropriate suffix. Support the "data:" URL scheme. -The open() method uses the tempcache. - -- New version of the xmllib module (this time with a test suite!) by -Sjoerd Mullender. - -- Added debugging code to the telnetlib module, to be able to trace -the actual traffic. - -- In the rfc822 module, added support for deleting a header (still no -support for adding headers, though). Also fixed a bug where an -illegal address would cause a crash in getrouteaddr(), fixed a -sign reversal in mktime_tz(), and use the local timezone by default -(the latter two due to Bill van Melle). - -- The normpath() function in the dospath and ntpath modules no longer -does case normalization -- for that, use the separate function -normcase() (which always existed); normcase() has been sped up and -fixed (it was the cause of a crash in Mark Hammond's installer in -certain locales). - -- New command supported by the ftplib module: rmd(); also fixed some -minor bugs. - -- The profile module now uses a different timer function by default -- -time.clock() is generally better than os.times(). This makes it work -better on Windows NT, too. - -- The tempfile module now recovers when os.getcwd() raises an -exception. - -- Fixed some bugs in the random module; gauss() was subtly wrong, and -vonmisesvariate() should return a full circle. Courtesy Mike Miller, -Lambert Meertens (gauss()), and Magnus Kessler (vonmisesvariate()). - -- Better default seed in the whrandom module, courtesy Andrew Kuchling. - -- Fix slow close() in shelve module. - -- The Unix mailbox class in the mailbox module is now more robust when -a line begins with the string "From " but is definitely not the start -of a new message. The pattern used can be changed by overriding a -method or class variable. - -- Added a rmtree() function to the copy module. - -- Fixed several typos in the pickle module. Also fixed problems when -unpickling in restricted execution environments. - -- Added docstrings and fixed a typo in the py_compile and compileall -modules. At Mark Hammond's repeated request, py_compile now append a -newline to the source if it needs one. Both modules support an extra -parameter to specify the purported source filename (to be used in -error messages). - -- Some performance tweaks by Jeremy Hylton to the gzip module. - -- Fixed a bug in the merge order of dictionaries in the ConfigParser -module. Courtesy Barry Warsaw. - -- In the multifile module, support the optional second parameter to -seek() when possible. - -- Several fixes to the gopherlib module by Lars Marius Garshol. Also, -urlparse now correctly handles Gopher URLs with query strings. - -- Fixed a tiny bug in format_exception() in the traceback module. -Also rewrite tb_lineno() to be compatible with JPython (and not -disturb the current exception!); by Jim Hugunin. - -- The httplib module is more robust when servers send a short response --- courtesy Tim O'Malley. - -Tkinter and friends -------------------- - -- Various typos and bugs fixed. - -- New module Tkdnd implements a drag-and-drop protocol (within one -application only). - -- The event_*() widget methods have been restructured slightly -- they -no longer use the default root. - -- The interfaces for the bind*() and unbind() widget methods have been -redesigned; the bind*() methods now return the name of the Tcl command -created for the callback, and this can be passed as an optional -argument to unbind() in order to delete the command (normally, such -commands are automatically unbound when the widget is destroyed, but -for some applications this isn't enough). - -- Variable objects now have trace methods to interface to Tcl's -variable tracing facilities. - -- Image objects now have an optional keyword argument, 'master', to -specify a widget (tree) to which they belong. The image_names() and -image_types() calls are now also widget methods. - -- There's a new global call, Tkinter.NoDefaultRoot(), which disables -all use of the default root by the Tkinter library. This is useful to -debug applications that are in the process of being converted from -relying on the default root to explicit specification of the root -widget. - -- The 'exit' command is deleted from the Tcl interpreter, since it -provided a loophole by which one could (accidentally) exit the Python -interpreter without invoking any cleanup code. - -- Tcl_Finalize() is now registered as a Python low-level exit handle, -so Tcl will be finalized when Python exits. - -The Python/C API ----------------- - -- New function PyThreadState_GetDict() returns a per-thread dictionary -intended for storing thread-local global variables. - -- New functions Py_ReprEnter() and Py_ReprLeave() use the per-thread -dictionary to allow recursive container types to detect recursion in -their repr(), str() and print implementations. - -- New function PyObject_Not(x) calculates (not x) according to Python's -standard rules (basically, it negates the outcome PyObject_IsTrue(x). - -- New function _PyModule_Clear(), which clears a module's dictionary -carefully without removing the __builtins__ entry. This is implied -when a module object is deallocated (this used to clear the dictionary -completely). - -- New function PyImport_ExecCodeModuleEx(), which extends -PyImport_ExecCodeModule() by adding an extra parameter to pass it the -true file. - -- New functions Py_GetPythonHome() and Py_SetPythonHome(), intended to -allow embedded applications to force a different value for PYTHONHOME. - -- New global flag Py_FrozenFlag is set when this is a "frozen" Python -binary; it suppresses warnings about not being able to find the -standard library directories. - -- New global flag Py_TabcheckFlag is incremented by the -t option and -causes the tokenizer to issue warnings or errors about inconsistent -mixing of tabs and spaces for indentation. - -Miscellaneous minor changes and bug fixes ------------------------------------------ - -- Improved the error message when an attribute of an attribute-less -object is requested -- include the name of the attribute and the type -of the object in the message. - -- Sped up int(), long(), float() a bit. - -- Fixed a bug in list.sort() that would occasionally dump core. - -- Fixed a bug in PyNumber_Power() that caused numeric arrays to fail -when taken tothe real power. - -- Fixed a number of bugs in the file reading code, at least one of -which could cause a core dump on NT, and one of which would -occasionally cause file.read() to return less than the full contents -of the file. - -- Performance hack by Vladimir Marangozov for stack frame creation. - -- Make sure setvbuf() isn't used unless HAVE_SETVBUF is defined. - -Windows 95/NT -------------- - -- The .lib files are now part of the distribution; they are collected -in the subdirectory "libs" of the installation directory. - -- The extension modules (.pyd files) are now collected in a separate -subdirectory of the installation directory named "DLLs". - -- The case of a module's filename must now match the case of the -module name as specified in the import statement. This is an -experimental feature -- if it turns out to break in too many -situations, it will be removed (or disabled by default) in the future. -It can be disabled on a per-case basis by setting the environment -variable PYTHONCASEOK (to any value). - - -====================================================================== - - -===================================== -==> Release 1.5 (January 3, 1998) <== -===================================== - - -From 1.5b2 to 1.5 -================= - -- Newly documentated module: BaseHTTPServer.py, thanks to Greg Stein. - -- Added doc strings to string.py, stropmodule.c, structmodule.c, -thanks to Charles Waldman. - -- Many nits fixed in the manuals, thanks to Fred Drake and many others -(especially Rob Hooft and Andrew Kuchling). The HTML version now uses -HTML markup instead of inline GIF images for tables; only two images -are left (for obsure bits of math). The index of the HTML version has -also been much improved. Finally, it is once again possible to -generate an Emacs info file from the library manual (but I don't -commit to supporting this in future versions). - -- New module: telnetlib.py (a simple telnet client library). - -- New tool: Tools/versioncheck/, by Jack Jansen. - -- Ported zlibmodule.c and bsddbmodule.c to NT; The project file for MS -DevStudio 5.0 now includes new subprojects to build the zlib and bsddb -extension modules. - -- Many small changes again to Tkinter.py -- mostly bugfixes and adding -missing routines. Thanks to Greg McFarlane for reporting a bunch of -problems and proofreading my fixes. - -- The re module and its documentation are up to date with the latest -version released to the string-sig (Dec. 22). - -- Stop test_grp.py from failing when the /etc/group file is empty -(yes, this happens!). - -- Fix bug in integer conversion (mystrtoul.c) that caused -4294967296==0 to be true! - -- The VC++ 4.2 project file should be complete again. - -- In tempfile.py, use a better template on NT, and add a new optional -argument "suffix" with default "" to specify a specific extension for -the temporary filename (needed sometimes on NT but perhaps also handy -elsewhere). - -- Fixed some bugs in the FAQ wizard, and converted it to use re -instead of regex. - -- Fixed a mysteriously undetected error in dlmodule.c (it was using a -totally bogus routine name to raise an exception). - -- Fixed bug in import.c which wasn't using the new "dos-8x3" name yet. - -- Hopefully harmless changes to the build process to support shared -libraries on DG/UX. This adds a target to create -libpython$(VERSION).so; however this target is *only* for DG/UX. - -- Fixed a bug in the new format string error checking in getargs.c. - -- A simple fix for infinite recursion when printing __builtins__: -reset '_' to None before printing and set it to the printed variable -*after* printing (and only when printing is successful). - -- Fixed lib-tk/SimpleDialog.py to keep the dialog visible even if the -parent window is not (Skip Montanaro). - -- Fixed the two most annoying problems with ftp URLs in -urllib.urlopen(); an empty file now correctly raises an error, and it -is no longer required to explicitly close the returned "file" object -before opening another ftp URL to the same host and directory. - - -====================================================================== - - -From 1.5b1 to 1.5b2 -=================== - -- Fixed a bug in cPickle.c that caused it to crash right away because -the version string had a different format. - -- Changes in pickle.py and cPickle.c: when unpickling an instance of a -class that doesn't define the __getinitargs__() method, the __init__() -constructor is no longer called. This makes a much larger group of -classes picklable by default, but may occasionally change semantics. -To force calling __init__() on unpickling, define a __getinitargs__() -method. Other changes too, in particular cPickle now handles classes -defined in packages correctly. The same change applies to copying -instances with copy.py. The cPickle.c changes and some pickle.py -changes are courtesy Jim Fulton. - -- Locale support in he "re" (Perl regular expressions) module. Use -the flag re.L (or re.LOCALE) to enable locale-specific matching -rules for \w and \b. The in-line syntax for this flag is (?L). - -- The built-in function isinstance(x, y) now also succeeds when y is -a type object and type(x) is y. - -- repr() and str() of class and instance objects now reflect the -package/module in which the class is defined. - -- Module "ni" has been removed. (If you really need it, it's been -renamed to "ni1". Let me know if this causes any problems for you. -Package authors are encouraged to write __init__.py files that -support both ni and 1.5 package support, so the same version can be -used with Python 1.4 as well as 1.5.) - -- The thread module is now automatically included when threads are -configured. (You must remove it from your existing Setup file, -since it is now in its own Setup.thread file.) - -- New command line option "-x" to skip the first line of the script; -handy to make executable scripts on non-Unix platforms. - -- In importdl.c, add the RTLD_GLOBAL to the dlopen() flags. I -haven't checked how this affects things, but it should make symbols -in one shared library available to the next one. - -- The Windows installer now installs in the "Program Files" folder on -the proper volume by default. - -- The Windows configuration adds a new main program, "pythonw", and -registers a new extension, ".pyw" that invokes this. This is a -pstandard Python interpreter that does not pop up a console window; -handy for pure Tkinter applications. All output to the original -stdout and stderr is lost; reading from the original stdin yields -EOF. Also, both python.exe and pythonw.exe now have a pretty icon -(a green snake in a box, courtesy Mark Hammond). - -- Lots of improvements to emacs-mode.el again. See Barry's web page: -http://www.python.org/ftp/emacs/pmdetails.html. - -- Lots of improvements and additions to the library reference manual; -many by Fred Drake. - -- Doc strings for the following modules: rfc822.py, posixpath.py, -ntpath.py, httplib.py. Thanks to Mitch Chapman and Charles Waldman. - -- Some more regression testing. - -- An optional 4th (maxsplit) argument to strop.replace(). - -- Fixed handling of maxsplit in string.splitfields(). - -- Tweaked os.environ so it can be pickled and copied. - -- The portability problems caused by indented preprocessor commands -and C++ style comments should be gone now. - -- In random.py, added Pareto and Weibull distributions. - -- The crypt module is now disabled in Modules/Setup.in by default; it -is rarely needed and causes errors on some systems where users often -don't know how to deal with those. - -- Some improvements to the _tkinter build line suggested by Case Roole. - -- A full suite of platform specific files for NetBSD 1.x, submitted by -Anders Andersen. - -- New Solaris specific header STROPTS.py. - -- Moved a confusing occurrence of *shared* from the comments in -Modules/Setup.in (people would enable this one instead of the real -one, and get disappointing results). - -- Changed the default mode for directories to be group-writable when -the installation process creates them. - -- Check for pthread support in "-l_r" for FreeBSD/NetBSD, and support -shared libraries for both. - -- Support FreeBSD and NetBSD in posixfile.py. - -- Support for the "event" command, new in Tk 4.2. By Case Roole. - -- Add Tix_SafeInit() support to tkappinit.c. - -- Various bugs fixed in "re.py" and "pcre.c". - -- Fixed a bug (broken use of the syntax table) in the old "regexpr.c". - -- In frozenmain.c, stdin is made unbuffered too when PYTHONUNBUFFERED -is set. - -- Provide default blocksize for retrbinary in ftplib.py (Skip -Montanaro). - -- In NT, pick the username up from different places in user.py (Jeff -Bauer). - -- Patch to urlparse.urljoin() for ".." and "..#1", Marc Lemburg. - -- Many small improvements to Jeff Rush' OS/2 support. - -- ospath.py is gone; it's been obsolete for so many years now... - -- The reference manual is now set up to prepare better HTML (still -using webmaker, alas). - -- Add special handling to /Tools/freeze for Python modules that are -imported implicitly by the Python runtime: 'site' and 'exceptions'. - -- Tools/faqwiz 0.8.3 -- add an option to suppress URL processing -inside 
    , by "Scott".
-
-- Added ConfigParser.py, a generic parser for sectioned configuration
-files.
-
-- In _localemodule.c, LC_MESSAGES is not always defined; put it
-between #ifdefs.
-
-- Typo in resource.c: RUSAGE_CHILDERN -> RUSAGE_CHILDREN.
-
-- Demo/scripts/newslist.py: Fix the way the version number is gotten
-out of the RCS revision.
-
-- PyArg_Parse[Tuple] now explicitly check for bad characters at the
-end of the format string.
-
-- Revamped PC/example_nt to support VC++ 5.x.
-
-- .sort() now uses a modified quicksort by Raymund Galvin,
-after studying the GNU libg++ quicksort.  This should be much faster
-if there are lots of duplicates, and otherwise at least as good.
-
-- Added "uue" as an alias for "uuencode" to mimetools.py.  (Hm, the
-uudecode bug where it complaints about trailing garbage is still there
-:-( ).
-
-- pickle.py requires integers in text mode to be in decimal notation
-(it used to accept octal and hex, even though it would only generate
-decimal numbers).
-
-- In string.atof(), don't fail when the "re" module is unavailable.
-Plug the ensuing security leak by supplying an empty __builtins__
-directory to eval().
-
-- A bunch of small fixes and improvements to Tkinter.py.
-
-- Fixed a buffer overrun in PC/getpathp.c.
-
-
-======================================================================
-
-
-From 1.5a4 to 1.5b1
-===================
-
-- The Windows NT/95 installer now includes full HTML of all manuals.
-It also has a checkbox that lets you decide whether to install the
-interpreter and library.  The WISE installer script for the installer
-is included in the source tree as PC/python15.wse, and so are the
-icons used for Python files.  The config.c file for the Windows build
-is now complete with the pcre module.
-
-- sys.ps1 and sys.ps2 can now arbitrary objects; their str() is
-evaluated for the prompt.
-
-- The reference manual is brought up to date (more or less -- it still
-needs work, e.g. in the area of package import).
-
-- The icons used by latex2html are now included in the Doc
-subdirectory (mostly so that tarring up the HTML files can be fully
-automated).  A simple index.html is also added to Doc (it only works
-after you have successfully run latex2html).
-
-- For all you would-be proselytizers out there: a new version of
-Misc/BLURB describes Python more concisely, and Misc/comparisons
-compares Python to several other languages.  Misc/BLURB.WINDOWS
-contains a blurb specifically aimed at Windows programmers (by Mark
-Hammond).
-
-- A new version of the Python mode for Emacs is included as
-Misc/python-mode.el.  There are too many new features to list here.
-See http://www.python.org/ftp/emacs/pmdetails.html for more info.
-
-- New module fileinput makes iterating over the lines of a list of
-files easier.  (This still needs some more thinking to make it more
-extensible.)
-
-- There's full OS/2 support, courtesy Jeff Rush.  To build the OS/2
-version, see PC/readme.txt and PC/os2vacpp.  This is for IBM's Visual
-Age C++ compiler.  I expect that Jeff will also provide a binary
-release for this platform.
-
-- On Linux, the configure script now uses '-Xlinker -export-dynamic'
-instead of '-rdynamic' to link the main program so that it exports its
-symbols to shared libraries it loads dynamically.  I hope this doesn't
-break on older Linux versions; it is needed for mklinux and appears to
-work on Linux 2.0.30.
-
-- Some Tkinter resstructuring: the geometry methods that apply to a
-master are now properly usable on toplevel master widgets.  There's a
-new (internal) widget class, BaseWidget.  New, longer "official" names
-for the geometry manager methods have been added,
-e.g. "grid_columnconfigure()" instead of "columnconfigure()".  The old
-shorter names still work, and where there's ambiguity, pack wins over
-place wins over grid.  Also, the bind_class method now returns its
-value.
-
-- New, RFC-822 conformant parsing of email addresses and address lists
-in the rfc822 module, courtesy Ben Escoto.
-
-- New, revamped tkappinit.c with support for popular packages (PIL,
-TIX, BLT, TOGL).  For the last three, you need to execute the Tcl
-command "load {} Tix" (or Blt, or Togl) to gain access to them.
-The Modules/Setup line for the _tkinter module has been rewritten
-using the cool line-breaking feature of most Bourne shells.
-
-- New socket method connect_ex() returns the error code from connect()
-instead of raising an exception on errors; this makes the logic
-required for asynchronous connects simpler and more efficient.
-
-- New "locale" module with (still experimental) interface to the
-standard C library locale interface, courtesy Martin von Löwis.  This
-does not repeat my mistake in 1.5a4 of always calling
-setlocale(LC_ALL, "").  In fact, we've pretty much decided that
-Python's standard numerical formatting operations should always use
-the conventions for the C locale; the locale module contains utility
-functions to format numbers according to the user specified locale.
-(All this is accomplished by an explicit call to setlocale(LC_NUMERIC,
-"C") after locale-changing calls.)  See the library manual. (Alas, the
-promised changes to the "re" module for locale support have not been
-materialized yet.  If you care, volunteer!)
-
-- Memory leak plugged in Py_BuildValue when building a dictionary.
-
-- Shared modules can now live inside packages (hierarchical module
-namespaces).  No changes to the shared module itself are needed.
-
-- Improved policy for __builtins__: this is a module in __main__ and a
-dictionary everywhere else.
-
-- Python no longer catches SIGHUP and SIGTERM by default.  This was
-impossible to get right in the light of thread contexts.  If you want
-your program to clean up when a signal happens, use the signal module
-to set up your own signal handler.
-
-- New Python/C API PyNumber_CoerceEx() does not return an exception
-when no coercion is possible.  This is used to fix a problem where
-comparing incompatible numbers for equality would raise an exception
-rather than return false as in Python 1.4 -- it once again will return
-false.
-
-- The errno module is changed again -- the table of error messages
-(errorstr) is removed.  Instead, you can use os.strerror().  This
-removes redundance and a potential locale dependency.
-
-- New module xmllib, to parse XML files.  By Sjoerd Mullender.
-
-- New C API PyOS_AfterFork() is called after fork() in posixmodule.c.
-It resets the signal module's notion of what the current process ID
-and thread are, so that signal handlers will work after (and across)
-calls to os.fork().
-
-- Fixed most occurrences of fatal errors due to missing thread state.
-
-- For vgrind (a flexible source pretty printer) fans, there's a simple
-Python definition in Misc/vgrindefs, courtesy Neale Pickett.
-
-- Fixed memory leak in exec statement.
-
-- The test.pystone module has a new function, pystones(loops=LOOPS),
-which returns a (benchtime, stones) tuple.  The main() function now
-calls this and prints the report.
-
-- Package directories now *require* the presence of an __init__.py (or
-__init__.pyc) file before they are considered as packages.  This is
-done to prevent accidental subdirectories with common names from
-overriding modules with the same name.
-
-- Fixed some strange exceptions in __del__ methods in library modules
-(e.g. urllib).  This happens because the built-in names are already
-deleted by the time __del__ is called.  The solution (a hack, but it
-works) is to set some instance variables to 0 instead of None.
-
-- The table of built-in module initializers is replaced by a pointer
-variable.  This makes it possible to switch to a different table at
-run time, e.g. when a collection of modules is loaded from a shared
-library.  (No example code of how to do this is given, but it is
-possible.)  The table is still there of course, its name prefixed with
-an underscore and used to initialize the pointer.
-
-- The warning about a thread still having a frame now only happens in
-verbose mode.
-
-- Change the signal finalization so that it also resets the signal
-handlers.  After this has been called, our signal handlers are no
-longer active!
-
-- New version of tokenize.py (by Ka-Ping Yee) recognizes raw string
-literals.  There's now also a test fort this module.
-
-- The copy module now also uses __dict__.update(state) instead of
-going through individual attribute assignments, for class instances
-without a __setstate__ method.
-
-- New module reconvert translates old-style (regex module) regular
-expressions to new-style (re module, Perl-style) regular expressions.
-
-- Most modules that used to use the regex module now use the re
-module.  The grep module has a new pgrep() function which uses
-Perl-style regular expressions.
-
-- The (very old, backwards compatibility) regexp.py module has been
-deleted.
-
-- Restricted execution (rexec): added the pcre module (support for the
-re module) to the list of trusted extension modules.
-
-- New version of Jim Fulton's CObject object type, adds
-PyCObject_FromVoidPtrAndDesc() and PyCObject_GetDesc() APIs.
-
-- Some patches to Lee Busby's fpectl mods that accidentally didn't
-make it into 1.5a4.
-
-- In the string module, add an optional 4th argument to count(),
-matching find() etc.
-
-- Patch for the nntplib module by Charles Waldman to add optional user
-and password arguments to NNTP.__init__(), for nntp servers that need
-them.
-
-- The str() function for class objects now returns
-"modulename.classname" instead of returning the same as repr().
-
-- The parsing of \xXX escapes no longer relies on sscanf().
-
-- The "sharedmodules" subdirectory of the installation is renamed to
-"lib-dynload".  (You may have to edit your Modules/Setup file to fix
-this in an existing installation!)
-
-- Fixed Don Beaudry's mess-up with the OPT test in the configure
-script.  Certain SGI platforms will still issue a warning for each
-compile; there's not much I can do about this since the compiler's
-exit status doesn't indicate that I was using an obsolete option.
-
-- Fixed Barry's mess-up with {}.get(), and added test cases for it.
-
-- Shared libraries didn't quite work under AIX because of the change
-in status of the GNU readline interface.  Fix due to by Vladimir
-Marangozov.
-
-
-======================================================================
-
-
-From 1.5a3 to 1.5a4
-===================
-
-- faqwiz.py: version 0.8; Recognize https:// as URL; ...
-feature; better install instructions; removed faqmain.py (which was an
-older version).
-
-- nntplib.py: Fixed some bugs reported by Lars Wirzenius (to Debian)
-about the treatment of lines starting with '.'.  Added a minimal test
-function.
-
-- struct module: ignore most whitespace in format strings.
-
-- urllib.py: close the socket and temp file in URLopener.retrieve() so
-that multiple retrievals using the same connection work.
-
-- All standard exceptions are now classes by default; use -X to make
-them strings (for backward compatibility only).
-
-- There's a new standard exception hierarchy, defined in the standard
-library module exceptions.py (which you never need to import
-explicitly).  See
-http://grail.cnri.reston.va.us/python/essays/stdexceptions.html for
-more info.
-
-- Three new C API functions:
-
-  - int PyErr_GivenExceptionMatches(obj1, obj2)
-
-    Returns 1 if obj1 and obj2 are the same object, or if obj1 is an
-    instance of type obj2, or of a class derived from obj2
-
-  - int PyErr_ExceptionMatches(obj)
-
-    Higher level wrapper around PyErr_GivenExceptionMatches() which uses
-    PyErr_Occurred() as obj1.  This will be the more commonly called
-    function.
-
-  - void PyErr_NormalizeException(typeptr, valptr, tbptr)
-
-    Normalizes exceptions, and places the normalized values in the
-    arguments.  If type is not a class, this does nothing.  If type is a
-    class, then it makes sure that value is an instance of the class by:
-
-    1. if instance is of the type, or a class derived from type, it does
-       nothing.
-
-    2. otherwise it instantiates the class, using the value as an
-       argument.  If value is None, it uses an empty arg tuple, and if
-       the value is a tuple, it uses just that.
-
-- Another new C API function: PyErr_NewException() creates a new
-exception class derived from Exception; when -X is given, it creates a
-new string exception.
-
-- core interpreter: remove the distinction between tuple and list
-unpacking; allow an arbitrary sequence on the right hand side of any
-unpack instruction.  (UNPACK_LIST and UNPACK_TUPLE now do the same
-thing, which should really be called UNPACK_SEQUENCE.)
-
-- classes: Allow assignments to an instance's __dict__ or __class__,
-so you can change ivars (including shared ivars -- shock horror) and
-change classes dynamically.  Also make the check on read-only
-attributes of classes less draconic -- only the specials names
-__dict__, __bases__, __name__ and __{get,set,del}attr__ can't be
-assigned.
-
-- Two new built-in functions: issubclass() and isinstance().  Both
-take classes as their second arguments.  The former takes a class as
-the first argument and returns true iff first is second, or is a
-subclass of second.  The latter takes any object as the first argument
-and returns true iff first is an instance of the second, or any
-subclass of second.
-
-- configure: Added configuration tests for presence of alarm(),
-pause(), and getpwent().
-
-- Doc/Makefile: changed latex2html targets.
-
-- classes: Reverse the search order for the Don Beaudry hook so that
-the first class with an applicable hook wins.  Makes more sense.
-
-- Changed the checks made in Py_Initialize() and Py_Finalize().  It is
-now legal to call these more than once.  The first call to
-Py_Initialize() initializes, the first call to Py_Finalize()
-finalizes.  There's also a new API, Py_IsInitialized() which checks
-whether we are already initialized (in case you want to leave things
-as they were).
-
-- Completely disable the declarations for malloc(), realloc() and
-free().  Any 90's C compiler has these in header files, and the tests
-to decide whether to suppress the declarations kept failing on some
-platforms.
-
-- *Before* (instead of after) signalmodule.o is added, remove both
-intrcheck.o and sigcheck.o.  This should get rid of warnings in ar or
-ld on various systems.
-
-- Added reop to PC/config.c
-
-- configure: Decided to use -Aa -D_HPUX_SOURCE on HP-UX platforms.
-Removed outdated HP-UX comments from README.  Added Cray T3E comments.
-
-- Various renames of statically defined functions that had name
-conflicts on some systems, e.g. strndup (GNU libc), join (Cray),
-roundup (sys/types.h).
-
-- urllib.py: Interpret three slashes in file: URL as local file (for
-Netscape on Windows/Mac).
-
-- copy.py: Make sure the objects returned by __getinitargs__() are
-kept alive (in the memo) to avoid a certain kind of nasty crash.  (Not
-easily reproducible because it requires a later call to
-__getinitargs__() to return a tuple that happens to be allocated at
-the same address.)
-
-- Added definition of AR to toplevel Makefile.  Renamed @buildno temp
-file to buildno1.
-
-- Moved Include/assert.h to Parser/assert.h, which seems to be the
-only place where it's needed.
-
-- Tweaked the dictionary lookup code again for some more speed
-(Vladimir Marangozov).
-
-- NT build: Changed the way python15.lib is included in the other
-projects.  Per Mark Hammond's suggestion, add it to the extra libs in
-Settings instead of to the project's source files.
-
-- regrtest.py: Change default verbosity so that there are only three
-levels left: -q, default and -v.  In default mode, the name of each
-test is now printed.  -v is the same as the old -vv.  -q is more quiet
-than the old default mode.
-
-- Removed the old FAQ from the distribution.  You now have to get it
-from the web!
-
-- Removed the PC/make_nt.in file from the distribution; it is no
-longer needed.
-
-- Changed the build sequence so that shared modules are built last.
-This fixes things for AIX and doesn't hurt elsewhere.
-
-- Improved test for GNU MP v1 in mpzmodule.c
-
-- fileobject.c: ftell() on Linux discards all buffered data; changed
-read() code to use lseek() instead to get the same effect
-
-- configure.in, configure, importdl.c: NeXT sharedlib fixes
-
-- tupleobject.c: PyTuple_SetItem asserts refcnt==1
-
-- resource.c: Different strategy regarding whether to declare
-getrusage() and getpagesize() -- #ifdef doesn't work, Linux has
-conflicting decls in its headers.  Choice: only declare the return
-type, not the argument prototype, and not on Linux.
-
-- importdl.c, configure*: set sharedlib extensions properly for NeXT
-
-- configure*, Makefile.in, Modules/Makefile.pre.in: AIX shared libraries
-fixed; moved addition of PURIFY to LINKCC to configure
-
-- reopmodule.c, regexmodule.c, regexpr.c, zlibmodule.c: needed casts
-added to shup up various compilers.
-
-- _tkinter.c: removed buggy mac #ifndef
-
-- Doc: various Mac documentation changes, added docs for 'ic' module
-
-- PC/make_nt.in: deleted
-
-- test_time.py, test_strftime.py: tweaks to catch %Z (which may return
-"")
-
-- test_rotor.py: print b -> print `b`
-
-- Tkinter.py: (tagOrId) -> (tagOrId,)
-
-- Tkinter.py: the Tk class now also has a configure() method and
-friends (they have been moved to the Misc class to accomplish this).
-
-- dict.get(key[, default]) returns dict[key] if it exists, or default
-if it doesn't.  The default defaults to None.  This is quicker for
-some applications than using either has_key() or try:...except
-KeyError:....
-
-- Tools/webchecker/: some small changes to webchecker.py; added
-websucker.py (a simple web site mirroring script).
-
-- Dictionary objects now have a get() method (also in UserDict.py).
-dict.get(key, default) returns dict[key] if it exists and default
-otherwise; default defaults to None.
-
-- Tools/scripts/logmerge.py: print the author, too.
-
-- Changes to import: support for "import a.b.c" is now built in.  See
-http://grail.cnri.reston.va.us/python/essays/packages.html
-for more info.  Most important deviations from "ni.py": __init__.py is
-executed in the package's namespace instead of as a submodule; and
-there's no support for "__" or "__domain__".  Note that "ni.py" is not
-changed to match this -- it is simply declared obsolete (while at the
-same time, it is documented...:-( ).
-Unfortunately, "ihooks.py" has not been upgraded (but see "knee.py"
-for an example implementation of hierarchical module import written in
-Python).
-
-- More changes to import: the site.py module is now imported by
-default when Python is initialized; use -S to disable it.  The site.py
-module extends the path with several more directories: site-packages
-inside the lib/python1.5/ directory, site-python in the lib/
-directory, and pathnames mentioned in *.pth files found in either of
-those directories.  See
-http://grail.cnri.reston.va.us/python/essays/packages.html
-for more info.
-
-- Changes to standard library subdirectory names: those subdirectories
-that are not packages have been renamed with a hypen in their name,
-e.g. lib-tk, lib-stdwin, plat-win, plat-linux2, plat-sunos5, dos-8x3.
-The test suite is now a package -- to run a test, you must now use
-"import test.test_foo".
-
-- A completely new re.py module is provided (thanks to Andrew
-Kuchling, Tim Peters and Jeffrey Ollie) which uses Philip Hazel's
-"pcre" re compiler and engine.  For a while, the "old" re.py (which
-was new in 1.5a3!) will be kept around as re1.py.  The "old" regex
-module and underlying parser and engine are still present -- while
-regex is now officially obsolete, it will probably take several major
-release cycles before it can be removed.
-
-- The posix module now has a strerror() function which translates an
-error code to a string.
-
-- The emacs.py module (which was long obsolete) has been removed.
-
-- The universal makefile Misc/Makefile.pre.in now features an
-"install" target.  By default, installed shared libraries go into
-$exec_prefix/lib/python$VERSION/site-packages/.
-
-- The install-sh script is installed with the other configuration
-specific files (in the config/ subdirectory).
-
-- It turns out whatsound.py and sndhdr.py were identical modules.
-Since there's also an imghdr.py file, I propose to make sndhdr.py the
-official one.  For compatibility, whatsound.py imports * from
-sndhdr.py.
-
-- Class objects have a new attribute, __module__, giving the name of
-the module in which they were declared.  This is useful for pickle and
-for printing the full name of a class exception.
-
-- Many extension modules no longer issue a fatal error when their
-initialization fails; the importing code now checks whether an error
-occurred during module initialization, and correctly propagates the
-exception to the import statement.
-
-- Most extension modules now raise class-based exceptions (except when
--X is used).
-
-- Subtle changes to PyEval_{Save,Restore}Thread(): always swap the
-thread state -- just don't manipulate the lock if it isn't there.
-
-- Fixed a bug in Python/getopt.c that made it do the wrong thing when
-an option was a single '-'.  Thanks to Andrew Kuchling.
-
-- New module mimetypes.py will guess a MIME type from a filename's
-extension.
-
-- Windows: the DLL version is now settable via a resource rather than
-being hardcoded.  This can be used for "branding" a binary Python
-distribution.
-
-- urllib.py is now threadsafe -- it now uses re instead of regex, and
-sys.exc_info() instead of sys.exc_{type,value}.
-
-- Many other library modules that used to use
-sys.exc_{type,value,traceback} are now more thread-safe by virtue of
-using sys.exc_info().
-
-- The functions in popen2 have an optional buffer size parameter.
-Also, the command argument can now be either a string (passed to the
-shell) or a list of arguments (passed directly to execv).
-
-- Alas, the thread support for _tkinter released with 1.5a3 didn't
-work.  It's been rewritten.  The bad news is that it now requires a
-modified version of a file in the standard Tcl distribution, which you
-must compile with a -I option pointing to the standard Tcl source
-tree.  For this reason, the thread support is disabled by default.
-
-- The errno extension module adds two tables: errorcode maps errno
-numbers to errno names (e.g. EINTR), and errorstr maps them to
-message strings.  (The latter is redundant because the new call
-posix.strerror() now does the same, but alla...)  (Marc-Andre Lemburg)
-
-- The readline extension module now provides some interfaces to
-internal readline routines that make it possible to write a completer
-in Python.  An example completer, rlcompleter.py, is provided.
-
-	When completing a simple identifier, it completes keywords,
-	built-ins and globals in __main__; when completing
-	NAME.NAME..., it evaluates (!) the expression up to the last
-	dot and completes its attributes.
-
-	It's very cool to do "import string" type "string.", hit the
-	completion key (twice), and see the list of names defined by
-	the string module!
-
-	Tip: to use the tab key as the completion key, call
-
-	    readline.parse_and_bind("tab: complete")
-
-- The traceback.py module has a new function tb_lineno() by Marc-Andre
-Lemburg which extracts the line number from the linenumber table in
-the code object.  Apparently the traceback object doesn't contain the
-right linenumber when -O is used.  Rather than guessing whether -O is
-on or off, the module itself uses tb_lineno() unconditionally.
-
-- Fixed Demo/tkinter/matt/canvas-moving-or-creating.py: change bind()
-to tag_bind() so it works again.
-
-- The pystone script is now a standard library module.  Example use:
-"import test.pystone; test.pystone.main()".
-
-- The import of the readline module in interactive mode is now also
-attempted when -i is specified.  (Yes, I know, giving in to Marc-Andre
-Lemburg, who asked for this. :-)
-
-- rfc822.py: Entirely rewritten parseaddr() function by Sjoerd
-Mullender, to be closer to the standard.  This fixes the getaddr()
-method.  Unfortunately, getaddrlist() is as broken as ever, since it
-splits on commas without regard for RFC 822 quoting conventions.
-
-- pprint.py: correctly emit trailing "," in singleton tuples.
-
-- _tkinter.c: export names for its type objects, TkappType and
-TkttType.
-
-- pickle.py: use __module__ when defined; fix a particularly hard to
-reproduce bug that confuses the memo when temporary objects are
-returned by custom pickling interfaces; and a semantic change: when
-unpickling the instance variables of an instance, use
-inst.__dict__.update(value) instead of a for loop with setattr() over
-the value.keys().  This is more consistent (the pickling doesn't use
-getattr() either but pickles inst.__dict__) and avoids problems with
-instances that have a __setattr__ hook.  But it *is* a semantic change
-(because the setattr hook is no longer used).  So beware!
-
-- config.h is now installed (at last) in
-$exec_prefix/include/python1.5/.  For most sites, this means that it
-is actually in $prefix/include/python1.5/, with all the other Python
-include files, since $prefix and $exec_prefix are the same by
-default.
-
-- The imp module now supports parts of the functionality to implement
-import of hierarchical module names.  It now supports find_module()
-and load_module() for all types of modules.  Docstrings have been
-added for those functions in the built-in imp module that are still
-relevant (some old interfaces are obsolete).  For a sample
-implementation of hierarchical module import in Python, see the new
-library module knee.py.
-
-- The % operator on string objects now allows arbitrary nested parens
-in a %(...)X style format.  (Brad Howes)
-
-- Reverse the order in which Setup and Setup.local are passed to the
-makesetup script.  This allows variable definitions in Setup.local to
-override definitions in Setup.  (But you'll still have to edit Setup
-if you want to disable modules that are enabled by default, or if such
-modules need non-standard options.)
-
-- Added PyImport_ImportModuleEx(name, globals, locals, fromlist); this
-is like PyImport_ImporModule(name) but receives the globals and locals
-dict and the fromlist arguments as well.  (The name is a char*; the
-others are PyObject*s).
-
-- The 'p' format in the struct extension module alloded to above is
-new in 1.5a4.
-
-- The types.py module now uses try-except in a few places to make it
-more likely that it can be imported in restricted mode.  Some type
-names are undefined in that case, e.g. CodeType (inaccessible),
-FileType (not always accessible), and TracebackType and FrameType
-(inaccessible).
-
-- In urllib.py: added separate administration of temporary files
-created y URLopener.retrieve() so cleanup() can properly remove them.
-The old code removed everything in tempcache which was a bad idea if
-the user had passed a non-temp file into it.  Also, in basejoin(),
-interpret relative paths starting in "../".  This is necessary if the
-server uses symbolic links.
-
-- The Windows build procedure and project files are now based on
-Microsoft Visual C++ 5.x.  The build now takes place in the PCbuild
-directory.  It is much more robust, and properly builds separate Debug
-and Release versions.  (The installer will be added shortly.)
-
-- Added casts and changed some return types in regexpr.c to avoid
-compiler warnings or errors on some platforms.
-
-- The AIX build tools for shared libraries now supports VPATH.  (Donn
-Cave)
-
-- By default, disable the "portable" multimedia modules audioop,
-imageop, and rgbimg, since they don't work on 64-bit platforms.
-
-- Fixed a nasty bug in cStringIO.c when code was actually using the
-close() method (the destructors would try to free certain fields a
-second time).
-
-- For those who think they need it, there's a "user.py" module.  This
-is *not* imported by default, but can be imported to run user-specific
-setup commands, ~/.pythonrc.py.
-
-- Various speedups suggested by Fredrik Lundh, Marc-Andre Lemburg,
-Vladimir Marangozov, and others.
-
-- Added os.altsep; this is '/' on DOS/Windows, and None on systems
-with a sane filename syntax.
-
-- os.py: Write out the dynamic OS choice, to avoid exec statements.
-Adding support for a new OS is now a bit more work, but I bet that
-'dos' or 'nt' will cover most situations...
-
-- The obsolete exception AccessError is now really gone.
-
-- Tools/faqwiz/: New installation instructions show how to maintain
-multiple FAQs.  Removed bootstrap script from end of faqwiz.py module.
-Added instructions to bootstrap script, too.  Version bumped to 0.8.1.
-Added ... feature suggested by Skip Montanaro.  Added
-leading text for Roulette, default to 'Hit Reload ...'.  Fix typo in
-default SRCDIR.
-
-- Documentation for the relatively new modules "keyword" and "symbol"
-has been added (to the end of the section on the parser extension
-module).
-
-- In module bisect.py, but functions have two optional argument 'lo'
-and 'hi' which allow you to specify a subsequence of the array to
-operate on.
-
-- In ftplib.py, changed most methods to return their status (even when
-it is always "200 OK") rather than swallowing it.
-
-- main() now calls setlocale(LC_ALL, ""), if setlocale() and
- are defined.
-
-- Changes to configure.in, the configure script, and both
-Makefile.pre.in files, to support SGI's SGI_ABI platform selection
-environment variable.
-
-
-======================================================================
-
-
-From 1.4 to 1.5a3
-=================
-
-Security
---------
-
-- If you are using the setuid script C wrapper (Misc/setuid-prog.c),
-please use the new version.  The old version has a huge security leak.
-
-Miscellaneous
--------------
-
-- Because of various (small) incompatible changes in the Python
-bytecode interpreter, the magic number for .pyc files has changed
-again.
-
-- The default module search path is now much saner.  Both on Unix and
-Windows, it is essentially derived from the path to the executable
-(which can be overridden by setting the environment variable
-$PYTHONHOME).  The value of $PYTHONPATH on Windows is now inserted in
-front of the default path, like in Unix (instead of overriding the
-default path).  On Windows, the directory containing the executable is
-added to the end of the path.
-
-- A new version of python-mode.el for Emacs has been included.  Also,
-a new file ccpy-style.el has been added to configure Emacs cc-mode for
-the preferred style in Python C sources.
-
-- On Unix, when using sys.argv[0] to insert the script directory in
-front of sys.path, expand a symbolic link.  You can now install a
-program in a private directory and have a symbolic link to it in a
-public bin directory, and it will put the private directory in the
-module search path.  Note that the symlink is expanded in sys.path[0]
-but not in sys.argv[0], so you can still tell the name by which you
-were invoked.
-
-- It is now recommended to use ``#!/usr/bin/env python'' instead of
-``#!/usr/local/bin/python'' at the start of executable scripts, except
-for CGI scripts.  It has been determined that the use of /usr/bin/env
-is more portable than that of /usr/local/bin/python -- scripts almost
-never have to be edited when the Python interpreter lives in a
-non-standard place.  Note that this doesn't work for CGI scripts since
-the python executable often doesn't live in the HTTP server's default
-search path.
-
-- The silly -s command line option and the corresponding
-PYTHONSUPPRESS environment variable (and the Py_SuppressPrint global
-flag in the Python/C API) are gone.
-
-- Most problems on 64-bit platforms should now be fixed.  Andrew
-Kuchling helped.  Some uncommon extension modules are still not
-clean (image and audio ops?).
-
-- Fixed a bug where multiple anonymous tuple arguments would be mixed up
-when using the debugger or profiler (reported by Just van Rossum).
-The simplest example is ``def f((a,b),(c,d)): print a,b,c,d''; this
-would print the wrong value when run under the debugger or profiler.
-
-- The hacks that the dictionary implementation used to speed up
-repeated lookups of the same C string were removed; these were a
-source of subtle problems and don't seem to serve much of a purpose
-any longer.
-
-- All traces of support for the long dead access statement have been
-removed from the sources.
-
-- Plugged the two-byte memory leak in the tokenizer when reading an
-interactive EOF.
-
-- There's a -O option to the interpreter that removes SET_LINENO
-instructions and assert statements (see below); it uses and produces
-.pyo files instead of .pyc files.  The speedup is only a few percent
-in most cases.  The line numbers are still available in the .pyo file,
-as a separate table (which is also available in .pyc files).  However,
-the removal of the SET_LINENO instructions means that the debugger
-(pdb) can't set breakpoints on lines in -O mode.  The traceback module
-contains a function to extract a line number from the code object
-referenced in a traceback object.  In the future it should be possible
-to write external bytecode optimizers that create better optimized
-.pyo files, and there should be more control over optimization;
-consider the -O option a "teaser".  Without -O, the assert statement
-actually generates code that first checks __debug__; if this variable
-is false, the assertion is not checked.  __debug__ is a built-in
-variable whose value is initialized to track the -O flag (it's true
-iff -O is not specified).  With -O, no code is generated for assert
-statements, nor for code of the form ``if __debug__: ''.
-Sorry, no further constant folding happens.
-
-
-Performance
------------
-
-- It's much faster (almost twice for pystone.py -- see
-Tools/scripts).  See the entry on string interning below.
-
-- Some speedup by using separate free lists for method objects (both
-the C and the Python variety) and for floating point numbers.
-
-- Big speedup by allocating frame objects with a single malloc() call.
-The Python/C API for frames is changed (you shouldn't be using this
-anyway).
-
-- Significant speedup by inlining some common opcodes for common operand
-types (e.g.  i+i, i-i, and list[i]).  Fredrik Lundh.
-
-- Small speedup by reordering the method tables of some common
-objects (e.g. list.append is now first).
-
-- Big optimization to the read() method of file objects.  A read()
-without arguments now attempts to use fstat to allocate a buffer of
-the right size; for pipes and sockets, it will fall back to doubling
-the buffer size.  While that the improvement is real on all systems,
-it is most dramatic on Windows.
-
-
-Documentation
--------------
-
-- Many new pieces of library documentation were contributed, mostly by
-Andrew Kuchling.  Even cmath is now documented!  There's also a
-chapter of the library manual, "libundoc.tex", which provides a
-listing of all undocumented modules, plus their status (e.g. internal,
-obsolete, or in need of documentation).  Also contributions by Sue
-Williams, Skip Montanaro, and some module authors who succumbed to
-pressure to document their own contributed modules :-).  Note that
-printing the documentation now kills fewer trees -- the margins have
-been reduced.
-
-- I have started documenting the Python/C API. Unfortunately this project
-hasn't been completed yet.  It will be complete before the final release of
-Python 1.5, though.  At the moment, it's better to read the LaTeX source
-than to attempt to run it through LaTeX and print the resulting dvi file.
-
-- The posix module (and hence os.py) now has doc strings!  Thanks to Neil
-Schemenauer.  I received a few other contributions of doc strings.  In most
-other places, doc strings are still wishful thinking...
-
-
-Language changes
-----------------
-
-- Private variables with leading double underscore are now a permanent
-feature of the language.  (These were experimental in release 1.4.  I have
-favorable experience using them; I can't label them "experimental"
-forever.)
-
-- There's new string literal syntax for "raw strings".  Prefixing a string
-literal with the letter r (or R) disables all escape processing in the
-string; for example, r'\n' is a two-character string consisting of a
-backslash followed by the letter n.  This combines with all forms of string
-quotes; it is actually useful for triple quoted doc strings which might
-contain references to \n or \t.  An embedded quote prefixed with a
-backslash does not terminate the string, but the backslash is still
-included in the string; for example, r'\'' is a two-character string
-consisting of a backslash and a quote.  (Raw strings are also
-affectionately known as Robin strings, after their inventor, Robin
-Friedrich.)
-
-- There's a simple assert statement, and a new exception
-AssertionError.  For example, ``assert foo > 0'' is equivalent to ``if
-not foo > 0: raise AssertionError''.  Sorry, the text of the asserted
-condition is not available; it would be too complicated to generate
-code for this (since the code is generated from a parse tree).
-However, the text is displayed as part of the traceback!
-
-- The raise statement has a new feature: when using "raise SomeClass,
-somevalue" where somevalue is not an instance of SomeClass, it
-instantiates SomeClass(somevalue).  In 1.5a4, if somevalue is an
-instance of a *derived* class of SomeClass, the exception class raised
-is set to somevalue.__class__, and SomeClass is ignored after that.
-
-- Duplicate keyword arguments are now detected at compile time;
-f(a=1,a=2) is now a syntax error.
-
-
-Changes to built-in features
-----------------------------
-
-- There's a new exception FloatingPointError (used only by Lee Busby's
-patches to catch floating point exceptions, at the moment).
-
-- The obsolete exception ConflictError (presumably used by the long
-obsolete access statement) has been deleted.
-
-- There's a new function sys.exc_info() which returns the tuple
-(sys.exc_type, sys.exc_value, sys.exc_traceback) in a thread-safe way.
-
-- There's a new variable sys.executable, pointing to the executable file
-for the Python interpreter.
-
-- The sort() methods for lists no longer uses the C library qsort(); I
-wrote my own quicksort implementation, with lots of help (in the form
-of a kind of competition) from Tim Peters.  This solves a bug in
-dictionary comparisons on some Solaris versions when Python is built
-with threads, and makes sorting lists even faster.
-
-- The semantics of comparing two dictionaries have changed, to make
-comparison of unequal dictionaries faster.  A shorter dictionary is
-always considered smaller than a larger dictionary.  For dictionaries
-of the same size, the smallest differing element determines the
-outcome (which yields the same results as before in this case, without
-explicit sorting).  Thanks to Aaron Watters for suggesting something
-like this.
-
-- The semantics of try-except have changed subtly so that calling a
-function in an exception handler that itself raises and catches an
-exception no longer overwrites the sys.exc_* variables.  This also
-alleviates the problem that objects referenced in a stack frame that
-caught an exception are kept alive until another exception is caught
--- the sys.exc_* variables are restored to their previous value when
-returning from a function that caught an exception.
-
-- There's a new "buffer" interface.  Certain objects (e.g. strings and
-arrays) now support the "buffer" protocol.  Buffer objects are acceptable
-whenever formerly a string was required for a write operation; mutable
-buffer objects can be the target of a read operation using the call
-f.readinto(buffer).  A cool feature is that regular expression matching now
-also work on array objects.  Contribution by Jack Jansen.  (Needs
-documentation.)
-
-- String interning: dictionary lookups are faster when the lookup
-string object is the same object as the key in the dictionary, not
-just a string with the same value.  This is done by having a pool of
-"interned" strings.  Most names generated by the interpreter are now
-automatically interned, and there's a new built-in function intern(s)
-that returns the interned version of a string.  Interned strings are
-not a different object type, and interning is totally optional, but by
-interning most keys a speedup of about 15% was obtained for the
-pystone benchmark.
-
-- Dictionary objects have several new methods; clear() and copy() have
-the obvious semantics, while update(d) merges the contents of another
-dictionary d into this one, overriding existing keys.  The dictionary
-implementation file is now called dictobject.c rather than the
-confusing mappingobject.c.
-
-- The intrinsic function dir() is much smarter; it looks in __dict__,
-__members__ and __methods__.
-
-- The intrinsic functions int(), long() and float() can now take a
-string argument and then do the same thing as string.atoi(),
-string.atol(), and string.atof().  No second 'base' argument is
-allowed, and complex() does not take a string (nobody cared enough).
-
-- When a module is deleted, its globals are now deleted in two phases.
-In the first phase, all variables whose name begins with exactly one
-underscore are replaced by None; in the second phase, all variables
-are deleted.  This makes it possible to have global objects whose
-destructors depend on other globals.  The deletion order within each
-phase is still random.
-
-- It is no longer an error for a function to be called without a
-global variable __builtins__ -- an empty directory will be provided
-by default.
-
-- Guido's corollary to the "Don Beaudry hook": it is now possible to
-do metaprogramming by using an instance as a base class.  Not for the
-faint of heart; and undocumented as yet, but basically if a base class
-is an instance, its class will be instantiated to create the new
-class.  Jim Fulton will love it -- it also works with instances of his
-"extension classes", since it is triggered by the presence of a
-__class__ attribute on the purported base class.  See
-Demo/metaclasses/index.html for an explanation and see that directory
-for examples.
-
-- Another change is that the Don Beaudry hook is now invoked when
-*any* base class is special.  (Up to 1.5a3, the *last* special base
-class is used; in 1.5a4, the more rational choice of the *first*
-special base class is used.)
-
-- New optional parameter to the readlines() method of file objects.
-This indicates the number of bytes to read (the actual number of bytes
-read will be somewhat larger due to buffering reading until the end of
-the line).  Some optimizations have also been made to speed it up (but
-not as much as read()).
-
-- Complex numbers no longer have the ".conj" pseudo attribute; use
-z.conjugate() instead, or complex(z.real, -z.imag).  Complex numbers
-now *do* support the __members__ and __methods__ special attributes.
-
-- The complex() function now looks for a __complex__() method on class
-instances before giving up.
-
-- Long integers now support arbitrary shift counts, so you can now
-write 1L<<1000000, memory permitting.  (Python 1.4 reports "outrageous
-shift count for this.)
-
-- The hex() and oct() functions have been changed so that for regular
-integers, they never emit a minus sign.  For example, on a 32-bit
-machine, oct(-1) now returns '037777777777' and hex(-1) returns
-'0xffffffff'.  While this may seem inconsistent, it is much more
-useful.  (For long integers, a minus sign is used as before, to fit
-the result in memory :-)
-
-- The hash() function computes better hashes for several data types,
-including strings, floating point numbers, and complex numbers.
-
-
-New extension modules
----------------------
-
-- New extension modules cStringIO.c and cPickle.c, written by Jim
-Fulton and other folks at Digital Creations.  These are much more
-efficient than their Python counterparts StringIO.py and pickle.py,
-but don't support subclassing.  cPickle.c clocks up to 1000 times
-faster than pickle.py; cStringIO.c's improvement is less dramatic but
-still significant.
-
-- New extension module zlibmodule.c, interfacing to the free zlib
-library (gzip compatible compression).  There's also a module gzip.py
-which provides a higher level interface.  Written by Andrew Kuchling
-and Jeremy Hylton.
-
-- New module readline; see the "miscellaneous" section above.
-
-- New Unix extension module resource.c, by Jeremy Hylton, provides
-access to getrlimit(), getrusage(), setrusage(), getpagesize(), and
-related symbolic constants.
-
-- New extension puremodule.c, by Barry Warsaw, which interfaces to the
-Purify(TM) C API.  See also the file Misc/PURIFY.README.  It is also
-possible to enable Purify by simply setting the PURIFY Makefile
-variable in the Modules/Setup file.
-
-
-Changes in extension modules
-----------------------------
-
-- The struct extension module has several new features to control byte
-order and word size.  It supports reading and writing IEEE floats even
-on platforms where this is not the native format.  It uses uppercase
-format codes for unsigned integers of various sizes (always using
-Python long ints for 'I' and 'L'), 's' with a size prefix for strings,
-and 'p' for "Pascal strings" (with a leading length byte, included in
-the size; blame Hannu Krosing; new in 1.5a4).  A prefix '>' forces
-big-endian data and '<' forces little-endian data; these also select
-standard data sizes and disable automatic alignment (use pad bytes as
-needed).
-
-- The array module supports uppercase format codes for unsigned data
-formats (like the struct module).
-
-- The fcntl extension module now exports the needed symbolic
-constants.  (Formerly these were in FCNTL.py which was not available
-or correct for all platforms.)
-
-- The extension modules dbm, gdbm and bsddb now check that the
-database is still open before making any new calls.
-
-- The dbhash module is no more.  Use bsddb instead.  (There's a third
-party interface for the BSD 2.x code somewhere on the web; support for
-bsddb will be deprecated.)
-
-- The gdbm module now supports a sync() method.
-
-- The socket module now has some new functions: getprotobyname(), and
-the set {ntoh,hton}{s,l}().
-
-- Various modules now export their type object: socket.SocketType,
-array.ArrayType.
-
-- The socket module's accept() method now returns unknown addresses as
-a tuple rather than raising an exception.  (This can happen in
-promiscuous mode.)  Theres' also a new function getprotobyname().
-
-- The pthread support for the thread module now works on most platforms.
-
-- STDWIN is now officially obsolete.  Support for it will eventually
-be removed from the distribution.
-
-- The binascii extension module is now hopefully fully debugged.
-(XXX Oops -- Fredrik Lundh promised me a uuencode fix that I never
-received.)
-
-- audioop.c: added a ratecv() function; better handling of overflow in
-add().
-
-- posixmodule.c: now exports the O_* flags (O_APPEND etc.).  On
-Windows, also O_TEXT and O_BINARY.  The 'error' variable (the
-exception is raises) is renamed -- its string value is now "os.error",
-so newbies don't believe they have to import posix (or nt) to catch
-it when they see os.error reported as posix.error.  The execve()
-function now accepts any mapping object for the environment.
-
-- A new version of the al (audio library) module for SGI was
-contributed by Sjoerd Mullender.
-
-- The regex module has a new function get_syntax() which retrieves the
-syntax setting set by set_syntax().  The code was also sanitized,
-removing worries about unclean error handling.  See also below for its
-successor, re.py.
-
-- The "new" module (which creates new objects of various types) once
-again has a fully functioning new.function() method.  Dangerous as
-ever!  Also, new.code() has several new arguments.
-
-- A problem has been fixed in the rotor module: on systems with signed
-characters, rotor-encoded data was not portable when the key contained
-8-bit characters.  Also, setkey() now requires its argument rather
-than having broken code to default it.
-
-- The sys.builtin_module_names variable is now a tuple.  Another new
-variables in sys is sys.executable (the full path to the Python
-binary, if known).
-
-- The specs for time.strftime() have undergone some revisions.  It
-appears that not all format characters are supported in the same way
-on all platforms.  Rather than reimplement it, we note these
-differences in the documentation, and emphasize the shared set of
-features.  There's also a thorough test set (that occasionally finds
-problems in the C library implementation, e.g. on some Linuxes),
-thanks to Skip Montanaro.
-
-- The nis module seems broken when used with NIS+; unfortunately
-nobody knows how to fix it.  It should still work with old NIS.
-
-
-New library modules
--------------------
-
-- New (still experimental) Perl-style regular expression module,
-re.py, which uses a new interface for matching as well as a new
-syntax; the new interface avoids the thread-unsafety of the regex
-interface.  This comes with a helper extension reopmodule.c and vastly
-rewritten regexpr.c.  Most work on this was done by Jeffrey Ollie, Tim
-Peters, and Andrew Kuchling.  See the documentation libre.tex.  In
-1.5, the old regex module is still fully supported; in the future, it
-will become obsolete.
-
-- New module gzip.py; see zlib above.
-
-- New module keyword.py exports knowledge about Python's built-in
-keywords.  (New version by Ka-Ping Yee.)
-
-- New module pprint.py (with documentation) which supports
-pretty-printing of lists, tuples, & dictionaries recursively.  By Fred
-Drake.
-
-- New module code.py.  The function code.compile_command() can
-determine whether an interactively entered command is complete or not,
-distinguishing incomplete from invalid input.  (XXX Unfortunately,
-this seems broken at this moment, and I don't have the time to fix
-it.  It's probably better to add an explicit interface to the parser
-for this.)
-
-- There is now a library module xdrlib.py which can read and write the
-XDR data format as used by Sun RPC, for example.  It uses the struct
-module.
-
-
-Changes in library modules
---------------------------
-
-- Module codehack.py is now completely obsolete.
-
-- The pickle.py module has been updated to make it compatible with the
-new binary format that cPickle.c produces.  By default it produces the
-old all-ASCII format compatible with the old pickle.py, still much
-faster than pickle.py; it will read both formats automatically.  A few
-other updates have been made.
-
-- A new helper module, copy_reg.py, is provided to register extensions
-to the pickling code.
-
-- Revamped module tokenize.py is much more accurate and has an
-interface that makes it a breeze to write code to colorize Python
-source code.  Contributed by Ka-Ping Yee.
-
-- In ihooks.py, ModuleLoader.load_module() now closes the file under
-all circumstances.
-
-- The tempfile.py module has a new class, TemporaryFile, which creates
-an open temporary file that will be deleted automatically when
-closed.  This works on Windows and MacOS as well as on Unix.  (Jim
-Fulton.)
-
-- Changes to the cgi.py module: Most imports are now done at the
-top of the module, which provides a speedup when using ni (Jim
-Fulton).  The problem with file upload to a Windows platform is solved
-by using the new tempfile.TemporaryFile class; temporary files are now
-always opened in binary mode (Jim Fulton).  The cgi.escape() function
-now takes an optional flag argument that quotes '"' to '"'.  It
-is now possible to invoke cgi.py from a command line script, to test
-cgi scripts more easily outside an http server.  There's an optional
-limit to the size of uploads to POST (Skip Montanaro).  Added a
-'strict_parsing' option to all parsing functions (Jim Fulton).  The
-function parse_qs() now uses urllib.unquote() on the name as well as
-the value of fields (Clarence Gardner).  The FieldStorage class now
-has a __len__() method.
-
-- httplib.py: the socket object is no longer closed; all HTTP/1.*
-responses are now accepted; and it is now thread-safe (by not using
-the regex module).
-
-- BaseHTTPModule.py: treat all HTTP/1.* versions the same.
-
-- The popen2.py module is now rewritten using a class, which makes
-access to the standard error stream and the process id of the
-subprocess possible.
-
-- Added timezone support to the rfc822.py module, in the form of a
-getdate_tz() method and a parsedate_tz() function; also a mktime_tz().
-Also added recognition of some non-standard date formats, by Lars
-Wirzenius, and RFC 850 dates (Chris Lawrence).
-
-- mhlib.py: various enhancements, including almost compatible parsing
-of message sequence specifiers without invoking a subprocess.  Also
-added a createmessage() method by Lars Wirzenius.
-
-- The StringIO.StringIO class now supports readline(nbytes).  (Lars
-Wirzenius.)  (Of course, you should be using cStringIO for performance.)
-
-- UserDict.py supports the new dictionary methods as well.
-
-- Improvements for whrandom.py by Tim Peters: use 32-bit arithmetic to
-speed it up, and replace 0 seed values by 1 to avoid degeneration.
-A bug was fixed in the test for invalid arguments.
-
-- Module ftplib.py: added support for parsing a .netrc file (Fred
-Drake).  Also added an ntransfercmd() method to the FTP class, which
-allows access to the expected size of a transfer when available, and a
-parse150() function to the module which parses the corresponding 150
-response.
-
-- urllib.py: the ftp cache is now limited to 10 entries.  Added
-quote_plus() and unquote_plus() functions which are like quote() and
-unquote() but also replace spaces with '+' or vice versa, for
-encoding/decoding CGI form arguments.  Catch all errors from the ftp
-module.  HTTP requests now add the Host: header line.  The proxy
-variable names are now mapped to lower case, for Windows.  The
-spliturl() function no longer erroneously throws away all data past
-the first newline.  The basejoin() function now intereprets "../"
-correctly.  I *believe* that the problems with "exception raised in
-__del__" under certain circumstances have been fixed (mostly by
-changes elsewher in the interpreter).
-
-- In urlparse.py, there is a cache for results in urlparse.urlparse();
-its size limit is set to 20.  Also, new URL schemes shttp, https, and
-snews are "supported".
-
-- shelve.py: use cPickle and cStringIO when available.  Also added
-a sync() method, which calls the database's sync() method if there is
-one.
-
-- The mimetools.py module now uses the available Python modules for
-decoding quoted-printable, uuencode and base64 formats, rather than
-creating a subprocess.
-
-- The python debugger (pdb.py, and its base class bdb.py) now support
-conditional breakpoints.  See the docs.
-
-- The modules base64.py, uu.py and quopri.py can now be used as simple
-command line utilities.
-
-- Various small fixes to the nntplib.py module that I can't bother to
-document in detail.
-
-- Sjoerd Mullender's mimify.py module now supports base64 encoding and
-includes functions to handle the funny encoding you sometimes see in mail
-headers.  It is now documented.
-
-- mailbox.py: Added BabylMailbox.  Improved the way the mailbox is
-gotten from the environment.
-
-- Many more modules now correctly open files in binary mode when this
-is necessary on non-Unix platforms.
-
-- The copying functions in the undocumented module shutil.py are
-smarter.
-
-- The Writer classes in the formatter.py module now have a flush()
-method.
-
-- The sgmllib.py module accepts hyphens and periods in the middle of
-attribute names.  While this is against the SGML standard, there is
-some HTML out there that uses this...
-
-- The interface for the Python bytecode disassembler module, dis.py,
-has been enhanced quite a bit.  There's now one main function,
-dis.dis(), which takes almost any kind of object (function, module,
-class, instance, method, code object) and disassembles it; without
-arguments it disassembles the last frame of the last traceback.  The
-other functions have changed slightly, too.
-
-- The imghdr.py module recognizes new image types: BMP, PNG.
-
-- The string.py module has a new function replace(str, old, new,
-[maxsplit]) which does substring replacements.  It is actually
-implemented in C in the strop module.  The functions [r]find() an
-[r]index() have an optional 4th argument indicating the end of the
-substring to search, alsoo implemented by their strop counterparts.
-(Remember, never import strop -- import string uses strop when
-available with zero overhead.)
-
-- The string.join() function now accepts any sequence argument, not
-just lists and tuples.
-
-- The string.maketrans() requires its first two arguments to be
-present.  The old version didn't require them, but there's not much
-point without them, and the documentation suggests that they are
-required, so we fixed the code to match the documentation.
-
-- The regsub.py module has a function clear_cache(), which clears its
-internal cache of compiled regular expressions.  Also, the cache now
-takes the current syntax setting into account.  (However, this module
-is now obsolete -- use the sub() or subn() functions or methods in the
-re module.)
-
-- The undocumented module Complex.py has been removed, now that Python
-has built-in complex numbers.  A similar module remains as
-Demo/classes/Complex.py, as an example.
-
-
-Changes to the build process
-----------------------------
-
-- The way GNU readline is configured is totally different.  The
---with-readline configure option is gone.  It is now an extension
-module, which may be loaded dynamically.  You must enable it (and
-specify the correct libraries to link with) in the Modules/Setup file.
-Importing the module installs some hooks which enable command line
-editing.  When the interpreter shell is invoked interactively, it
-attempts to import the readline module; when this fails, the default
-input mechanism is used.  The hook variables are PyOS_InputHook and
-PyOS_ReadlineFunctionPointer.  (Code contributed by Lee Busby, with
-ideas from William Magro.)
-
-- New build procedure: a single library, libpython1.5.a, is now built,
-which contains absolutely everything except for a one-line main()
-program (which calls Py_Main(argc, argv) to start the interpreter
-shell).  This makes life much simpler for applications that need to
-embed Python.  The serial number of the build is now included in the
-version string (sys.version).
-
-- As far as I can tell, neither gcc -Wall nor the Microsoft compiler
-emits a single warning any more when compiling Python.
-
-- A number of new Makefile variables have been added for special
-situations, e.g. LDLAST is appended to the link command.  These are
-used by editing the Makefile or passing them on the make command
-line.
-
-- A set of patches from Lee Busby has been integrated that make it
-possible to catch floating point exceptions.  Use the configure option
---with-fpectl to enable the patches; the extension modules fpectl and
-fpetest provide control to enable/disable and test the feature,
-respectively.
-
-- The support for shared libraries under AIX is now simpler and more
-robust.  Thanks to Vladimir Marangozov for revamping his own patches!
-
-- The Modules/makesetup script now reads a file Setup.local as well as
-a file Setup.  Most changes to the Setup script can be done by editing
-Setup.local instead, which makes it easier to carry a particular setup
-over from one release to the next.
-
-- The Modules/makesetup script now copies any "include" lines it
-encounters verbatim into the output Makefile.  It also recognizes .cxx
-and .cpp as C++ source files.
-
-- The configure script is smarter about C compiler options; e.g. with
-gcc it uses -O2 and -g when possible, and on some other platforms it
-uses -Olimit 1500 to avoid a warning from the optimizer about the main
-loop in ceval.c (which has more than 1000 basic blocks).
-
-- The configure script now detects whether malloc(0) returns a NULL
-pointer or a valid block (of length zero).  This avoids the nonsense
-of always adding one byte to all malloc() arguments on most platforms.
-
-- The configure script has a new option, --with-dec-threads, to enable
-DEC threads on DEC Alpha platforms.  Also, --with-threads is now an
-alias for --with-thread (this was the Most Common Typo in configure
-arguments).
-
-- Many changes in Doc/Makefile; amongst others, latex2html is now used
-to generate HTML from all latex documents.
-
-
-Change to the Python/C API
---------------------------
-
-- Because some interfaces have changed, the PYTHON_API macro has been
-bumped.  Most extensions built for the old API version will still run,
-but I can't guarantee this.  Python prints a warning message on
-version mismatches; it dumps core when the version mismatch causes a
-serious problem :-)
-
-- I've completed the Grand Renaming, with the help of Roger Masse and
-Barry Warsaw.  This makes reading or debugging the code much easier.
-Many other unrelated code reorganizations have also been carried out.
-The allobjects.h header file is gone; instead, you would have to
-include Python.h followed by rename2.h.  But you're better off running
-Tools/scripts/fixcid.py -s Misc/RENAME on your source, so you can omit
-the rename2.h; it will disappear in the next release.
-
-- Various and sundry small bugs in the "abstract" interfaces have been
-fixed.  Thanks to all the (involuntary) testers of the Python 1.4
-version!  Some new functions have been added, e.g. PySequence_List(o),
-equivalent to list(o) in Python.
-
-- New API functions PyLong_FromUnsignedLong() and
-PyLong_AsUnsignedLong().
-
-- The API functions in the file cgensupport.c are no longer
-supported.  This file has been moved to Modules and is only ever
-compiled when the SGI specific 'gl' module is built.
-
-- PyObject_Compare() can now raise an exception.  Check with
-PyErr_Occurred().  The comparison function in an object type may also
-raise an exception.
-
-- The slice interface uses an upper bound of INT_MAX when no explicit
-upper bound is given (e.x. for a[1:]).  It used to ask the object for
-its length and do the calculations.
-
-- Support for multiple independent interpreters.  See Doc/api.tex,
-functions Py_NewInterpreter() and Py_EndInterpreter().  Since the
-documentation is incomplete, also see the new Demo/pysvr example
-(which shows how to use these in a threaded application) and the
-source code.
-
-- There is now a Py_Finalize() function which "de-initializes"
-Python.  It is possible to completely restart the interpreter
-repeatedly by calling Py_Finalize() followed by Py_Initialize().  A
-change of functionality in Py_Initialize() means that it is now a
-fatal error to call it while the interpreter is already initialized.
-The old, half-hearted Py_Cleanup() routine is gone.  Use of Py_Exit()
-is deprecated (it is nothing more than Py_Finalize() followed by
-exit()).
-
-- There are no known memory leaks left.  While Py_Finalize() doesn't
-free *all* allocated memory (some of it is hard to track down),
-repeated calls to Py_Finalize() and Py_Initialize() do not create
-unaccessible heap blocks.
-
-- There is now explicit per-thread state.  (Inspired by, but not the
-same as, Greg Stein's free threading patches.)
-
-- There is now better support for threading C applications.  There are
-now explicit APIs to manipulate the interpreter lock.  Read the source
-or the Demo/pysvr example; the new functions are
-PyEval_{Acquire,Release}{Lock,Thread}().
-
-- The test macro DEBUG has changed to Py_DEBUG, to avoid interference
-with other libraries' DEBUG macros.  Likewise for any other test
-macros that didn't yet start with Py_.
-
-- New wrappers around malloc() and friends: Py_Malloc() etc. call
-malloc() and call PyErr_NoMemory() when it fails; PyMem_Malloc() call
-just malloc().  Use of these wrappers could be essential if multiple
-memory allocators exist (e.g. when using certain DLL setups under
-Windows).  (Idea by Jim Fulton.)
-
-- New C API PyImport_Import() which uses whatever __import__() hook
-that is installed for the current execution environment.  By Jim
-Fulton.
-
-- It is now possible for an extension module's init function to fail
-non-fatally, by calling one of the PyErr_* functions and returning.
-
-- The PyInt_AS_LONG() and PyFloat_AS_DOUBLE() macros now cast their
-argument to the proper type, like the similar PyString macros already
-did.  (Suggestion by Marc-Andre Lemburg.)  Similar for PyList_GET_SIZE
-and PyList_GET_ITEM.
-
-- Some of the Py_Get* function, like Py_GetVersion() (but not yet
-Py_GetPath()) are now declared as returning a const char *.  (More
-should follow.)
-
-- Changed the run-time library to check for exceptions after object
-comparisons.  PyObject_Compare() can now return an exception; use
-PyErr_Occurred() to check (there is *no* special return value).
-
-- PyFile_WriteString() and Py_Flushline() now return error indicators
-instead of clearing exceptions.  This fixes an obscure bug where using
-these would clear a pending exception, discovered by Just van Rossum.
-
-- There's a new function, PyArg_ParseTupleAndKeywords(), which parses
-an argument list including keyword arguments.  Contributed by Geoff
-Philbrick.
-
-- PyArg_GetInt() is gone.
-
-- It's no longer necessary to include graminit.h when calling one of
-the extended parser API functions.  The three public grammar start
-symbols are now in Python.h as Py_single_input, Py_file_input, and
-Py_eval_input.
-
-- The CObject interface has a new function,
-PyCObject_Import(module, name).  It calls PyCObject_AsVoidPtr()
-on the object referenced by "module.name".
-
-
-Tkinter
--------
-
-- On popular demand, _tkinter once again installs a hook for readline
-that processes certain Tk events while waiting for the user to type
-(using PyOS_InputHook).
-
-- A patch by Craig McPheeters plugs the most obnoxious memory leaks,
-caused by command definitions referencing widget objects beyond their
-lifetime.
-
-- New standard dialog modules: tkColorChooser.py, tkCommonDialog.py,
-tkMessageBox.py, tkFileDialog.py, tkSimpleDialog.py These interface
-with the new Tk dialog scripts, and provide more "native platform"
-style file selection dialog boxes on some platforms.  Contributed by
-Fredrik Lundh.
-
-- Tkinter.py: when the first Tk object is destroyed, it sets the
-hiddel global _default_root to None, so that when another Tk object is
-created it becomes the new default root.  Other miscellaneous
-changes and fixes.
-
-- The Image class now has a configure method.
-
-- Added a bunch of new winfo options to Tkinter.py; we should now be
-up to date with Tk 4.2.  The new winfo options supported are:
-mananger, pointerx, pointerxy, pointery, server, viewable, visualid,
-visualsavailable.
-
-- The broken bind() method on Canvas objects defined in the Canvas.py
-module has been fixed.  The CanvasItem and Group classes now also have
-an unbind() method.
-
-- The problem with Tkinter.py falling back to trying to import
-"tkinter" when "_tkinter" is not found has been fixed -- it no longer
-tries "tkinter", ever.  This makes diagnosing the problem "_tkinter
-not configured" much easier and will hopefully reduce the newsgroup
-traffic on this topic.
-
-- The ScrolledText module once again supports the 'cnf' parameter, to
-be compatible with the examples in Mark Lutz' book (I know, I know,
-too late...)
-
-- The _tkinter.c extension module has been revamped.  It now support
-Tk versions 4.1 through 8.0; support for 4.0 has been dropped.  It
-works well under Windows and Mac (with the latest Tk ports to those
-platforms).  It also supports threading -- it is safe for one
-(Python-created) thread to be blocked in _tkinter.mainloop() while
-other threads modify widgets.  To make the changes visible, those
-threads must use update_idletasks()method.  (The patch for threading
-in 1.5a3 was broken; in 1.5a4, it is back in a different version,
-which requires access to the Tcl sources to get it to work -- hence it
-is disabled by default.)
-
-- A bug in _tkinter.c has been fixed, where Split() with a string
-containing an unmatched '"' could cause an exception or core dump.
-
-- Unfortunately, on Windows and Mac, Tk 8.0 no longer supports
-CreateFileHandler, so _tkinter.createfilehandler is not available on
-those platforms when using Tk 8.0 or later.  I will have to rethink
-how to interface with Tcl's lower-level event mechanism, or with its
-channels (which are like Python's file-like objects).  Jack Jansen has
-provided a fix for the Mac, so createfilehandler *is* actually
-supported there; maybe I can adapt his fix for Windows.
-
-
-Tools and Demos
----------------
-
-- A new regression test suite is provided, which tests most of the
-standard and built-in modules.  The regression test is run by invoking
-the script Lib/test/regrtest.py.  Barry Warsaw wrote the test harnass;
-he and Roger Masse contributed most of the new tests.
-
-- New tool: faqwiz -- the CGI script that is used to maintain the
-Python FAQ (http://grail.cnri.reston.va.us/cgi-bin/faqw.py).  In
-Tools/faqwiz.
-
-- New tool: webchecker -- a simple extensible web robot that, when
-aimed at a web server, checks that server for dead links.  Available
-are a command line utility as well as a Tkinter based GUI version.  In
-Tools/webchecker.  A simplified version of this program is dissected
-in my article in O'Reilly's WWW Journal, the issue on Scripting
-Languages (Vol 2, No 2); Scripting the Web with Python (pp 97-120).
-Includes a parser for robots.txt files by Skip Montanaro.
-
-- New small tools: cvsfiles.py (prints a list of all files under CVS
-n a particular directory tree), treesync.py (a rather Guido-specific
-script to synchronize two source trees, one on Windows NT, the other
-one on Unix under CVS but accessible from the NT box), and logmerge.py
-(sort a collection of RCS or CVS logs by date).  In Tools/scripts.
-
-- The freeze script now also works under Windows (NT).  Another
-feature allows the -p option to be pointed at the Python source tree
-instead of the installation prefix.  This was loosely based on part of
-xfreeze by Sam Rushing and Bill Tutt.
-
-- New examples (Demo/extend) that show how to use the generic
-extension makefile (Misc/Makefile.pre.in).
-
-- Tools/scripts/h2py.py now supports C++ comments.
-
-- Tools/scripts/pystone.py script is upgraded to version 1.1; there
-was a bug in version 1.0 (distributed with Python 1.4) that leaked
-memory.  Also, in 1.1, the LOOPS variable is incremented to 10000.
-
-- Demo/classes/Rat.py completely rewritten by Sjoerd Mullender.
-
-
-Windows (NT and 95)
--------------------
-
-- New project files for Developer Studio (Visual C++) 5.0 for Windows
-NT (the old VC++ 4.2 Makefile is also still supported, but will
-eventually be withdrawn due to its bulkiness).
-
-- See the note on the new module search path in the "Miscellaneous" section
-above.
-
-- Support for Win32s (the 32-bit Windows API under Windows 3.1) is
-basically withdrawn.  If it still works for you, you're lucky.
-
-- There's a new extension module, msvcrt.c, which provides various
-low-level operations defined in the Microsoft Visual C++ Runtime Library.
-These include locking(), setmode(), get_osfhandle(), set_osfhandle(), and
-console I/O functions like kbhit(), getch() and putch().
-
-- The -u option not only sets the standard I/O streams to unbuffered
-status, but also sets them in binary mode.  (This can also be done
-using msvcrt.setmode(), by the way.)
-
-- The, sys.prefix and sys.exec_prefix variables point to the directory
-where Python is installed, or to the top of the source tree, if it was run
-from there.
-
-- The various os.path modules (posixpath, ntpath, macpath) now support
-passing more than two arguments to the join() function, so
-os.path.join(a, b, c) is the same as os.path.join(a, os.path.join(b,
-c)).
-
-- The ntpath module (normally used as os.path) supports ~ to $HOME
-expansion in expanduser().
-
-- The freeze tool now works on Windows.
-
-- See also the Tkinter category for a sad note on
-_tkinter.createfilehandler().
-
-- The truncate() method for file objects now works on Windows.
-
-- Py_Initialize() is no longer called when the DLL is loaded.  You
-must call it yourself.
-
-- The time module's clock() function now has good precision through
-the use of the Win32 API QueryPerformanceCounter().
-
-- Mark Hammond will release Python 1.5 versions of PythonWin and his
-other Windows specific code: the win32api extensions, COM/ActiveX
-support, and the MFC interface.
-
-
-Mac
----
-
-- As always, the Macintosh port will be done by Jack Jansen.  He will
-make a separate announcement for the Mac specific source code and the
-binary distribution(s) when these are ready.
-
-
-======================================================================
-
-
-=====================================
-==> Release 1.4 (October 25 1996) <==
-=====================================
-
-(Starting in reverse chronological order:)
-
-- Changed disclaimer notice.
-
-- Added SHELL=/bin/sh to Misc/Makefile.pre.in -- some Make versions
-default to the user's login shell.
-
-- In Lib/tkinter/Tkinter.py, removed bogus binding of  in Text
-widget, and bogus bspace() function.
-
-- In Lib/cgi.py, bumped __version__ to 2.0 and restored a truncated
-paragraph.
-
-- Fixed the NT Makefile (PC/vc40.mak) for VC 4.0 to set /MD for all
-subprojects, and to remove the (broken) experimental NumPy
-subprojects.
-
-- In Lib/py_compile.py, cast mtime to long() so it will work on Mac
-(where os.stat() returns mtimes as floats.)
-- Set self.rfile unbuffered (like self.wfile) in SocketServer.py, to
-fix POST in CGIHTTPServer.py.
-
-- Version 2.83 of Misc/python-mode.el for Emacs is included.
-
-- In Modules/regexmodule.c, fixed symcomp() to correctly handle a new
-group starting immediately after a group tag.
-
-- In Lib/SocketServer.py, changed the mode for rfile to unbuffered.
-
-- In Objects/stringobject.c, fixed the compare function to do the
-first char comparison in unsigned mode, for consistency with the way
-other characters are compared by memcmp().
-
-- In Lib/tkinter/Tkinter.py, fixed Scale.get() to support floats.
-
-- In Lib/urllib.py, fix another case where openedurl wasn't set.
-
-(XXX Sorry, the rest is in totally random order.  No time to fix it.)
-
-- SyntaxError exceptions detected during code generation
-(e.g. assignment to an expression) now include a line number.
-
-- Don't leave trailing / or \ in script directory inserted in front of
-sys.path.
-
-- Added a note to Tools/scripts/classfix.py abouts its historical
-importance.
-
-- Added Misc/Makefile.pre.in, a universal Makefile for extensions
-built outside the distribution.
-
-- Rewritten Misc/faq2html.py, by Ka-Ping Yee.
-
-- Install shared modules with mode 555 (needed for performance on some
-platforms).
-
-- Some changes to standard library modules to avoid calling append()
-with more than one argument -- while supported, this should be
-outlawed, and I don't want to set a bad example.
-
-- bdb.py (and hence pdb.py) supports calling run() with a code object
-instead of a code string.
-
-- Fixed an embarrassing bug cgi.py which prevented correct uploading
-of binary files from Netscape (which doesn't distinguish between
-binary and text files).  Also added dormant logging support, which
-makes it easier to debug the cgi module itself.
-
-- Added default writer to constructor of NullFormatter class.
-
-- Use binary mode for socket.makefile() calls in ftplib.py.
-
-- The ihooks module no longer "installs" itself upon import -- this
-was an experimental feature that helped ironing out some bugs but that
-slowed down code that imported it without the need to install it
-(e.g. the rexec module).  Also close the file in some cases and add
-the __file__ attribute to loaded modules.
-
-- The test program for mailbox.py is now more useful.
-
-- Added getparamnames() to Message class in mimetools.py -- it returns
-the names of parameters to the content-type header.
-
-- Fixed a typo in ni that broke the loop stripping "__." from names.
-
-- Fix sys.path[0] for scripts run via pdb.py's new main program.
-
-- profile.py can now also run a script, like pdb.
-
-- Fix a small bug in pyclbr -- don't add names starting with _ when
-emulating from ... import *.
-
-- Fixed a series of embarrassing typos in rexec's handling of standard
-I/O redirection.  Added some more "safe" built-in modules: cmath,
-errno, operator.
-
-- Fixed embarrassing typo in shelve.py.
-
-- Added SliceType and EllipsisType to types.py.
-
-- In urllib.py, added handling for error 301 (same as 302); added
-geturl() method to get the URL after redirection.
-
-- Fixed embarrassing typo in xdrlib.py.  Also fixed typo in Setup.in
-for _xdrmodule.c and removed redundant #include from _xdrmodule.c.
-
-- Fixed bsddbmodule.c to add binary mode indicator on platforms that
-have it.  This should make it working on Windows NT.
-
-- Changed last uses of #ifdef NT to #ifdef MS_WINDOWS or MS_WIN32,
-whatever applies.  Also rationalized some other tests for various MS
-platforms.
-
-- Added the sources for the NT installer script used for Python
-1.4beta3.  Not tested with this release, but better than nothing.
-
-- A compromise in pickle's defenses against Trojan horses: a
-user-defined function is now okay where a class is expected.  A
-built-in function is not okay, to prevent pickling something that
-will execute os.system("rm -f *") when unpickling.
-
-- dis.py will print the name of local variables referenced by local
-load/store/delete instructions.
-
-- Improved portability of SimpleHTTPServer module to non-Unix
-platform.
-
-- The thread.h interface adds an extra argument to down_sema().  This
-only affects other C code that uses thread.c; the Python thread module
-doesn't use semaphores (which aren't provided on all platforms where
-Python threads are supported).  Note: on NT, this change is not
-implemented.
-
-- Fixed some typos in abstract.h; corrected signature of
-PyNumber_Coerce, added PyMapping_DelItem.  Also fixed a bug in
-abstract.c's PyObject_CallMethod().
-
-- apply(classname, (), {}) now works even if the class has no
-__init__() method.
-
-- Implemented complex remainder and divmod() (these would dump core!).
-Conversion of complex numbers to int, long int or float now raises an
-exception, since there is no meaningful way to do it without losing
-information.
-
-- Fixed bug in built-in complex() function which gave the wrong result
-for two real arguments.
-
-- Change the hash algorithm for strings -- the multiplier is now
-1000003 instead of 3, which gives better spread for short strings.
-
-- New default path for Windows NT, the registry structure now supports
-default paths for different install packages.  (Mark Hammond -- the
-next PythonWin release will use this.)
-
-- Added more symbols to the python_nt.def file.
-
-- When using GNU readline, set rl_readline_name to "python".
-
-- The Ellipses built-in name has been renamed to Ellipsis -- this is
-the correct singular form.  Thanks to Ka-Ping Yee, who saved us from
-eternal embarrassment.
-
-- Bumped the PYTHON_API_VERSION to 1006, due to the Ellipses ->
-Ellipsis name change.
-
-- Updated the library reference manual.  Added documentation of
-restricted mode (rexec, Bastion) and the formatter module (for use
-with the htmllib module).  Fixed the documentation of htmllib
-(finally).
-
-- The reference manual is now maintained in FrameMaker.
-
-- Upgraded scripts Doc/partparse.py and Doc/texi2html.py.
-
-- Slight improvements to Doc/Makefile.
-
-- Added fcntl.lockf(). This should be used for Unix file locking
-instead of the posixfile module; lockf() is more portable.
-
-- The getopt module now supports long option names, thanks to Lars
-Wizenius.
-
-- Plenty of changes to Tkinter and Canvas, mostly due to Fred Drake
-and Nils Fischbeck.
-
-- Use more bits of time.time() in whrandom's default seed().
-
-- Performance hack for regex module's regs attribute.
-
-- Don't close already closed socket in socket module.
-
-- Correctly handle separators containing embedded nulls in
-strop.split, strop.find and strop.rfind.  Also added more detail to
-error message for strop.atoi and friends.
-
-- Moved fallback definition for hypot() to Python/hypot.c.
-
-- Added fallback definition for strdup, in Python/strdup.c.
-
-- Fixed some bugs where a function would return 0 to indicate an error
-where it should return -1.
-
-- Test for error returned by time.localtime(), and rationalized its MS
-tests.
-
-- Added Modules/Setup.local file, which is processed after Setup.
-
-- Corrected bug in toplevel Makefile.in -- execution of regen script
-would not use the right PATH and PYTHONPATH.
-
-- Various and sundry NeXT configuration changes (sigh).
-
-- Support systems where libreadline needs neither termcap nor curses.
-
-- Improved ld_so_aix script and python.exp file (for AIX).
-
-- More stringent test for working  in configure script.
-
-- Removed Demo/www subdirectory -- it was totally out of date.
-
-- Improved demos and docs for Fred Drake's parser module; fixed one
-typo in the module itself.
-
-
-=========================================
-==> Release 1.4beta3 (August 26 1996) <==
-=========================================
-
-
-(XXX This is less readable that it should.  I promise to restructure
-it for the final 1.4 release.)
-
-
-What's new in 1.4beta3 (since beta2)?
--------------------------------------
-
-- Name mangling to implement a simple form of class-private variables.
-A name of the form "__spam" can't easily be used outside the class.
-(This was added in 1.4beta3, but left out of the 1.4beta3 release
-message.)
-
-- In urllib.urlopen(): HTTP URLs containing user:passwd@host are now
-handled correctly when using a proxy server.
-
-- In ntpath.normpath(): don't truncate to 8+3 format.
-
-- In mimetools.choose_boundary(): don't die when getuid() or getpid()
-aren't defined.
-
-- Module urllib: some optimizations to (un)quoting.
-
-- New module MimeWriter for writing MIME documents.
-
-- More changes to formatter module.
-
-- The freeze script works once again and is much more robust (using
-sys.prefix etc.).  It also supports a -o option to specify an
-output directory.
-
-- New module whichdb recognizes dbm, gdbm and bsddb/dbhash files.
-
-- The Doc/Makefile targets have been reorganized somewhat to remove the
-insistence on always generating PostScript.
-
-- The texinfo to html filter (Doc/texi2html.py) has been improved somewhat.
-
-- "errors.h" has been renamed to "pyerrors.h" to resolve a long-standing
-name conflict on the Mac.
-
-- Linking a module compiled with a different setting for Py_TRACE_REFS now
-generates a linker error rather than a core dump.
-
-- The cgi module has a new convenience function print_exception(), which
-formats a python exception using HTML.  It also fixes a bug in the
-compatibility code and adds a dubious feature which makes it possible to
-have two query strings, one in the URL and one in the POST data.
-
-- A subtle change in the unpickling of class instances makes it possible
-to unpickle in restricted execution mode, where the __dict__ attribute is
-not available (but setattr() is).
-
-- Documentation for os.path.splitext() (== posixpath.splitext()) has been
-cleared up.  It splits at the *last* dot.
-
-- posixfile locking is now also correctly supported on AIX.
-
-- The tempfile module once again honors an initial setting of tmpdir.  It
-now works on Windows, too.
-
-- The traceback module has some new functions to extract, format and print
-the active stack.
-
-- Some translation functions in the urllib module have been made a little
-less sluggish.
-
-- The addtag_* methods for Canvas widgets in Tkinter as well as in the
-separate Canvas class have been fixed so they actually do something
-meaningful.
-
-- A tiny _test() function has been added to Tkinter.py.
-
-- A generic Makefile for dynamically loaded modules is provided in the Misc
-subdirectory (Misc/gMakefile).
-
-- A new version of python-mode.el for Emacs is provided.  See
-http://www.python.org/ftp/emacs/pmdetails.html for details.  The
-separate file pyimenu.el is no longer needed, imenu support is folded
-into python-mode.el.
-
-- The configure script can finally correctly find the readline library in a
-non-standard location.  The LDFLAGS variable is passed on the Makefiles
-from the configure script.
-
-- Shared libraries are now installed as programs (i.e. with executable
-permission).  This is required on HP-UX and won't hurt on other systems.
-
-- The objc.c module is no longer part of the distribution.  Objective-C
-support may become available as contributed software on the ftp site.
-
-- The sybase module is no longer part of the distribution.  A much
-improved sybase module is available as contributed software from the
-ftp site.
-
-- _tkinter is now compatible with Tcl 7.5 / Tk 4.1 patch1 on Windows and
-Mac (don't use unpatched Tcl/Tk!).  The default line in the Setup.in file
-now links with Tcl 7.5 / Tk 4.1 rather than 7.4/4.0.
-
-- In Setup, you can now write "*shared*" instead of "*noconfig*", and you
-can use *.so and *.sl as shared libraries.
-
-- Some more fidgeting for AIX shared libraries.
-
-- The mpz module is now compatible with GMP 2.x.  (Not tested by me.)
-(Note -- a complete replacement by Niels Mo"ller, called gpmodule, is
-available from the contrib directory on the ftp site.)
-
-- A warning is written to sys.stderr when a __del__ method raises an
-exception (formerly, such exceptions were completely ignored).
-
-- The configure script now defines HAVE_OLD_CPP if the C preprocessor is
-incapable of ANSI style token concatenation and stringification.
-
-- All source files (except a few platform specific modules) are once again
-compatible with K&R C compilers as well as ANSI compilers.  In particular,
-ANSI-isms have been removed or made conditional in complexobject.c,
-getargs.c and operator.c.
-
-- The abstract object API has three new functions, PyObject_DelItem,
-PySequence_DelItem, and PySequence_DelSlice.
-
-- The operator module has new functions delitem and delslice, and the
-functions "or" and "and" are renamed to "or_" and "and_" (since "or" and
-"and" are reserved words).  ("__or__" and "__and__" are unchanged.)
-
-- The environment module is no longer supported; putenv() is now a function
-in posixmodule (also under NT).
-
-- Error in filter(, "") has been fixed.
-
-- Unrecognized keyword arguments raise TypeError, not KeyError.
-
-- Better portability, fewer bugs and memory leaks, fewer compiler warnings,
-some more documentation.
-
-- Bug in float power boundary case (0.0 to the negative integer power)
-fixed.
-
-- The test of negative number to the float power has been moved from the
-built-in pow() function to floatobject.c (so complex numbers can yield the
-correct result).
-
-- The bug introduced in beta2 where shared libraries loaded (using
-dlopen()) from the current directory would fail, has been fixed.
-
-- Modules imported as shared libraries now also have a __file__ attribute,
-giving the filename from which they were loaded.  The only modules without
-a __file__ attribute now are built-in modules.
-
-- On the Mac, dynamically loaded modules can end in either ".slb" or
-"..slb" where  is either "CFM68K" or "ppc".  The ".slb"
-extension should only be used for "fat" binaries.
-
-- C API addition: marshal.c now supports
-PyMarshal_WriteObjectToString(object).
-
-- C API addition: getargs.c now supports
-PyArg_ParseTupleAndKeywords(args, kwdict, format, kwnames, ...)
-to parse keyword arguments.
-
-- The PC versioning scheme (sys.winver) has changed once again.  the
-version number is now "...", where the
-first three s are the Python version (e.g. "1.4.0" for Python 1.4,
-"1.4.1" for Python 1.4.1 -- the beta level is not included) and
- is the four-digit PYTHON_API_VERSION (currently 1005).
-
-- h2py.py accepts whitespace before the # in CPP directives
-
-- On Solaris 2.5, it should now be possible to use either Posix threads or
-Solaris threads (XXX: how do you select which is used???).  (Note: the
-Python pthreads interface doesn't fully support semaphores yet -- anyone
-care to fix this?)
-
-- Thread support should now work on AIX, using either DCE threads or
-pthreads.
-
-- New file Demo/sockets/unicast.py
-
-- Working Mac port, with CFM68K support, with Tk 4.1 support (though not
-both) (XXX)
-
-- New project setup for PC port, now compatible with PythonWin, with
-_tkinter and NumPy support (XXX)
-
-- New module site.py (XXX)
-
-- New module xdrlib.py and optional support module _xdrmodule.c (XXX)
-
-- parser module adapted to new grammar, complete w/ Doc & Demo (XXX)
-
-- regen script fixed (XXX)
-
-- new machdep subdirectories Lib/{aix3,aix4,next3_3,freebsd2,linux2} (XXX)
-
-- testall now also tests math module (XXX)
-
-- string.atoi c.s. now raise an exception for an empty input string.
-
-- At last, it is no longer necessary to define HAVE_CONFIG_H in order to
-have config.h included at various places.
-
-- Unrecognized keyword arguments now raise TypeError rather than KeyError.
-
-- The makesetup script recognizes files with extension .so or .sl as
-(shared) libraries.
-
-- 'access' is no longer a reserved word, and all code related to its
-implementation is gone (or at least #ifdef'ed out).  This should make
-Python a little speedier too!
-
-- Performance enhancements suggested by Sjoerd Mullender.  This includes
-the introduction of two new optional function pointers in type object,
-getattro and setattro, which are like getattr and setattr but take a
-string object instead of a C string pointer.
-
-- New operations in string module: lstrip(s) and rstrip(s) strip whitespace
-only on the left or only on the right, A new optional third argument to
-split() specifies the maximum number of separators honored (so
-splitfields(s, sep, n) returns a list of at most n+1 elements).  (Since
-1.3, splitfields(s, None) is totally equivalent to split(s).)
-string.capwords() has an optional second argument specifying the
-separator (which is passed to split()).
-
-- regsub.split() has the same addition as string.split().  regsub.splitx(s,
-sep, maxsep) implements the functionality that was regsub.split(s, 1) in
-1.4beta2 (return a list containing the delimiters as well as the words).
-
-- Final touch for AIX loading, rewritten Misc/AIX-NOTES.
-
-- In Modules/_tkinter.c, when using Tk 4.1 or higher, use className
-argument to _tkinter.create() to set Tcl's argv0 variable, so X
-resources use the right resource class again.
-
-- Add #undef fabs to Modules/mathmodule.c for macintosh.
-
-- Added some macro renames for AIX in Modules/operator.c.
-
-- Removed spurious 'E' from Doc/liberrno.tex.
-
-- Got rid of some cruft in Misc/ (dlMakefile, pyimenu.el); added new
-Misc/gMakefile and new version of Misc/python-mode.el.
-
-- Fixed typo in Lib/ntpath.py (islink has "return false" which gives a
-NameError).
-
-- Added missing "from types import *" to Lib/tkinter/Canvas.py.
-
-- Added hint about using default args for __init__ to pickle docs.
-
-- Corrected typo in Inclide/abstract.h: PySequence_Lenth ->
-PySequence_Length.
-
-- Some improvements to Doc/texi2html.py.
-
-- In Python/import.c, Cast unsigned char * in struct _frozen to char *
-in calls to rds_object().
-
-- In doc/ref4.tex, added note about scope of lambda bodies.
-
-What's new in 1.4beta2 (since beta1)?
--------------------------------------
-
-- Portability bug in the md5.h header solved.
-
-- The PC build procedure now really works, and sets sys.platform to a
-meaningful value (a few things were botched in beta 1).  Lib/dos_8x3
-is now a standard part of the distribution (alas).
-
-- More improvements to the installation procedure.  Typing "make install"
-now inserts the version number in the pathnames of almost everything
-installed, and creates the machine dependent modules (FCNTL.py etc.) if not
-supplied by the distribution.  (XXX There's still a problem with the latter
-because the "regen" script requires that Python is installed.  Some manual
-intervention may still be required.) (This has been fixed in 1.4beta3.)
-
-- New modules: errno, operator (XXX).
-
-- Changes for use with Numerical Python: built-in function slice() and
-Ellipses object, and corresponding syntax:
-
-	x[lo:hi:stride]		==	x[slice(lo, hi, stride)]
-	x[a, ..., z]		==	x[(a, Ellipses, z)]
-
-- New documentation for errno and cgi modules.
-
-- The directory containing the script passed to the interpreter is
-inserted in from of sys.path; "." is no longer a default path
-component.
-
-- Optional third string argument to string.translate() specifies
-characters to delete.  New function string.maketrans() creates a
-translation table for translate() or for regex.compile().
-
-- Module posix (and hence module os under Unix) now supports putenv().
-Moreover, module os is enhanced so that if putenv() is supported,
-assignments to os.environ entries make the appropriate putenv() call.
-(XXX the putenv() implementation can leak a small amount of memory per
-call.)
-
-- pdb.py can now be invoked from the command line to debug a script:
-python pdb.py